From 4d28f95d8bb56f70fb2507642278975948ad448f Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 17:26:35 -0700 Subject: [PATCH 01/43] includes all the files when ventis build --- ventis/cli.py | 2 ++ ventis/stub_generator.py | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 3aceb18..9ffc149 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -274,6 +274,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + project_dir=project_dir, ) else: @@ -316,6 +317,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + project_dir=project_dir, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..a571ccc 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -263,8 +263,36 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# Directories ventis build itself generates inside a project -- never swept. +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} + + +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + swept = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + for fname in files: + if fname.endswith(".py"): + abs_src = os.path.join(root, fname) + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) + return swept + + +def _stub_destination(stub_file, project_dir): + """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" + basename = os.path.basename(stub_file) + return os.path.join("agents", basename) if project_dir else basename + + def generate_docker( - yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None + yaml_path, + agent_file, + output_dir=None, + grpc_stubs_dir=None, + stub_files=None, + project_dir=None, ): """ Generate a minimal Docker build context for an agent. @@ -278,6 +306,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -301,8 +330,13 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements) + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + # Copy general agent files - files_to_copy = [ + files_to_copy += [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -322,13 +356,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs + # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), os.path.basename(stub_file)) + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -339,7 +373,9 @@ def generate_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") @@ -377,7 +413,12 @@ def generate_docker( def generate_workflow_docker( - workflow_file, stub_files, output_dir=None, grpc_stubs_dir=None, api_port=8080 + workflow_file, + stub_files, + output_dir=None, + grpc_stubs_dir=None, + api_port=8080, + project_dir=None, ): """ Generate a Docker build context for a workflow. @@ -391,6 +432,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -417,7 +459,12 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - files_to_copy = [ + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + + files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -437,9 +484,11 @@ def generate_workflow_docker( ], ] - # Copy stub files + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + files_to_copy.append( + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -449,7 +498,9 @@ def generate_workflow_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") From 23e3928d01bc5996fd1cd4e086d45212c283de0e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 18:23:51 -0700 Subject: [PATCH 02/43] fixed some bugs --- ventis/cli.py | 19 ++++++++++++++ ventis/stub_generator.py | 54 +++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 9ffc149..cb9ee0a 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -220,6 +220,23 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) + # Map each stub's basename to its agent's declared entrypoint, so a stub + # overwrites the exact real file it replaces instead of guessing its path. + stub_entrypoints = {} + for agent_cfg in agents: + entrypoint = agent_cfg.get("entrypoint") + if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: + continue + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_cfg["name"]: + base_name = os.path.splitext(os.path.basename(yaml_path))[0] + stub_entrypoints[f"{base_name}.py"] = entrypoint + break + # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -275,6 +292,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) else: @@ -318,6 +336,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a571ccc..a56080d 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,7 +271,12 @@ def _sweep_py_files(project_dir): """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and not (root == project_dir and d in _GENERATED_DIRS) + ] for fname in files: if fname.endswith(".py"): abs_src = os.path.join(root, fname) @@ -280,10 +285,15 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, project_dir): - """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" +def _stub_destination(stub_file, stub_entrypoints): + """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" basename = os.path.basename(stub_file) - return os.path.join("agents", basename) if project_dir else basename + entrypoint = stub_entrypoints.get(basename) + if entrypoint: + normalized = entrypoint.replace("\\", "/") + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return entrypoint + return basename def generate_docker( @@ -293,6 +303,7 @@ def generate_docker( grpc_stubs_dir=None, stub_files=None, project_dir=None, + stub_entrypoints=None, ): """ Generate a minimal Docker build context for an agent. @@ -301,12 +312,13 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -360,7 +372,10 @@ def generate_docker( if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -419,6 +434,7 @@ def generate_workflow_docker( grpc_stubs_dir=None, api_port=8080, project_dir=None, + stub_entrypoints=None, ): """ Generate a Docker build context for a workflow. @@ -428,11 +444,12 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -487,7 +504,10 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) # Copy gRPC generated stubs if they exist From 95240ca941167a11c6da6cc791ca4a2fb13c7ebe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 19:17:55 -0700 Subject: [PATCH 03/43] ventis build: sweep project .py files into Docker build contexts generate_docker()/generate_workflow_docker() now recursively sweep every .py file under the project directory into the build context, preserving directory structure, so helper files that aren't declared as an agent entrypoint still make it into the image. Generated dirs (docker_container/, stubs/, grpc_stubs/) are excluded at the project root only, not at every depth. Stub files are placed at their agent's declared entrypoint path (mapped from global_controller.yaml) instead of a hardcoded guess, so a stub overwrites the exact real file it replaces. Guards against absolute and '..'-containing entrypoints, symlinked sources, and symlinked-destination escapes, with warnings on unsafe or unmapped stubs. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 45 +++++++++++++++--------------------- ventis/stub_generator.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index cb9ee0a..4c9badf 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -212,6 +212,23 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) + import yaml + + # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} + for yaml_path in yaml_files: + with open(yaml_path) as f: + name = yaml.safe_load(f).get("agent", {}).get("name") + if name: + yaml_by_name[name] = yaml_path + + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -220,23 +237,6 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) - # Map each stub's basename to its agent's declared entrypoint, so a stub - # overwrites the exact real file it replaces instead of guessing its path. - stub_entrypoints = {} - for agent_cfg in agents: - entrypoint = agent_cfg.get("entrypoint") - if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: - continue - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_cfg["name"]: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - stub_entrypoints[f"{base_name}.py"] = entrypoint - break - # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -310,16 +310,7 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = None - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_name: - matching_yaml = yaml_path - break - + matching_yaml = yaml_by_name.get(agent_name) if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a56080d..9ea6c99 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -278,24 +278,43 @@ def _sweep_py_files(project_dir): and not (root == project_dir and d in _GENERATED_DIRS) ] for fname in files: - if fname.endswith(".py"): - abs_src = os.path.join(root, fname) + abs_src = os.path.join(root, fname) + if fname.endswith(".py") and not os.path.islink(abs_src): rel_dst = os.path.relpath(abs_src, project_dir) swept.append((abs_src, rel_dst)) return swept def _stub_destination(stub_file, stub_entrypoints): - """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") if not normalized.startswith("/") and ".." not in normalized.split("/"): - return entrypoint + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + elif stub_entrypoints: + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") return basename +def _copy_files(output_dir, files_to_copy): + """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" + real_output_dir = os.path.realpath(output_dir) + for src, dst in files_to_copy: + if not os.path.isfile(src): + print(f" Warning: source file not found, skipping: {src}") + continue + dest_path = os.path.join(output_dir, dst) + real_dest = os.path.realpath(dest_path) + if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: + print(f" Warning: destination escapes build context, skipping: {dst}") + continue + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) + + def generate_docker( yaml_path, agent_file, @@ -386,13 +405,7 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # Copy the YAML definition too shutil.copy2( @@ -477,9 +490,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -500,7 +511,7 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( @@ -516,13 +527,7 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading From 69d5405c24485ff51db960ad7843e496d91354ce Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:06:31 -0700 Subject: [PATCH 04/43] Fix missing os import in metrics_agent.py Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json From 4af43ae3abb8da914050f9f83c61cbf2badebf17 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Tue, 25 Aug 2026 20:40:54 -0700 Subject: [PATCH 05/43] [Feature] Pass env / secrets into agent containers Users had no way to get API keys (OpenAI, Anthropic, embedding models) into an agent container. Add a top-level `env_file` key to global_controller.yaml pointing at a local .env file, which reaches every container as `docker run --env-file`. - resolve_env_file validates the path before anything launches, so a missing .env fails at deploy time instead of deep inside a container. Relative paths resolve against the project root, matching entrypoint. - env_file_args is a context manager owning the local-vs-remote decision and the cleanup, so both runtimes share one code path. Local containers read the original file; remote containers get a copy that is deleted as soon as `docker run` returns, whether or not it succeeded. - GlobalController._push_file streams the file over ssh under `umask 077` rather than scp, so the copy is never briefly world-readable and the secret never lands in a command line. _run_cmd's ssh options moved to a shared _ssh_args. --env-file is appended after the explicit -e VENTIS_* flags; Docker gives those precedence regardless of order, so a stray VENTIS_* line in someone's .env cannot break agent wiring. Closes #50 --- ventis/cli.py | 10 ++ .../cloud_provider_logic/EC2/_runtime.py | 12 ++- .../cloud_provider_logic/Local/_runtime.py | 12 ++- ventis/controller/global_controller.py | 94 +++++++++++++------ ventis/controller/utils/env_file.py | 92 ++++++++++++++++++ 5 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index b43a6b3..6d85e1f 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,6 +16,8 @@ import subprocess import sys +from ventis.controller.utils.env_file import resolve_env_file + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -397,6 +399,14 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() + # Fail here rather than after a fleet of containers is already up without + # the API keys they need. + try: + resolve_env_file(config, base_dir=project_dir) + except ValueError as e: + logger.error("%s", e) + sys.exit(1) + _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..9955fa2 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,6 +23,7 @@ import boto3 +from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -285,8 +286,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + + # User secrets from `env_file`. Explicit -e flags above still win over + # anything in the file. + with env_file_args( + _controller, host, ssh_user, container_name, is_local=False + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 963eef3..a387f7b 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,6 +8,8 @@ import logging +from ventis.controller.utils.env_file import env_file_args + logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..241daff 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,6 +4,7 @@ import atexit import logging +import shlex import signal import subprocess import threading @@ -16,6 +17,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -64,6 +66,9 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) + # Validate before launching anything: an agent that boots without its + # API keys fails deep inside a container, where it is expensive to debug. + self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -174,6 +179,7 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) + self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -642,6 +648,28 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # + def _ssh_args(self, host, user=None): + """Return the `ssh ... target` prefix used to reach a remote host.""" + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + ) + return [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + f"{user}@{host}" if user else host, + ] + def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -656,41 +684,45 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=180 - ) - else: - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get( - "ssh_private_key_path", "~/.ssh/ventis_ec2" - ) - ) - ssh_target = f"{user}@{host}" if user else host - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - ssh_target, - remote_cmd, - ], + return subprocess.run(cmd, capture_output=True, text=True, timeout=180) + + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + capture_output=True, + text=True, + timeout=180, + ) + + def _push_file(self, local_path, remote_path, host, user=None): + """ + Copy a local file to a remote host over SSH. + + Streams the bytes through `cat` under `umask 077` rather than using + `scp`, so a secrets file is never briefly world-readable on the far + side. + + Returns: + subprocess.CompletedProcess + """ + remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + with open(local_path, "rb") as f: + result = subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + stdin=f, capture_output=True, text=True, timeout=180, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to copy {local_path} to {host}:{remote_path}: " + f"{(result.stderr or result.stdout or '').strip()}" ) + return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py new file mode 100644 index 0000000..c83f770 --- /dev/null +++ b/ventis/controller/utils/env_file.py @@ -0,0 +1,92 @@ +""" +Pass user secrets (API keys and friends) into agent containers. + +The user points `env_file` in `config/global_controller.yaml` at a local +`.env` file. Containers on this machine read that file directly; containers +on a remote host get a short-lived 0600 copy. Either way the file reaches +Docker as `--env-file`. +""" + +import logging +import os +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +REMOTE_ENV_DIR = "/tmp" + + +def resolve_env_file(config, base_dir=None): + """ + Return the absolute path of the configured env file, or None when unset. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + + Raises: + ValueError: the file is configured but unusable. Deploy should fail + here rather than start a fleet of agents with no API keys. + """ + raw = config.get("env_file") + if not raw: + return None + + path = os.path.expanduser(str(raw)) + if not os.path.isabs(path): + path = os.path.join(base_dir or os.getcwd(), path) + path = os.path.abspath(path) + + if not os.path.exists(path): + raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") + if not os.path.isfile(path): + raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") + if not os.access(path, os.R_OK): + raise ValueError(f"env_file is not readable: {path}") + return path + + +def remote_env_path(container_name): + """Where a remote host holds this container's copy of the env file.""" + return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + + +@contextmanager +def env_file_args(controller, host, user, container_name, is_local): + """ + Yield the `docker run` flags that hand the user's env file to a container. + + A container on this machine reads the original file. A container on a + remote host gets a 0600 copy, deleted as soon as the `with` body ends -- + success or failure, since by then the container holds the variables + itself. Keep that body tight around `docker run` so the copy is never + on the host longer than it has to be. + + Yields an empty list when no `env_file` is configured. + """ + env_file_path = getattr(controller, "env_file_path", None) + if not env_file_path: + yield [] + return + + if is_local: + yield ["--env-file", env_file_path] + return + + remote_path = remote_env_path(container_name) + controller._push_file(env_file_path, remote_path, host, user=user) + try: + yield ["--env-file", remote_path] + finally: + _remove_remote_copy(controller, remote_path, host, user) + + +def _remove_remote_copy(controller, remote_path, host, user): + """Delete a remote copy. Best effort -- never masks the caller's error.""" + try: + result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) + if getattr(result, "returncode", 0) != 0: + logger.warning("Failed to delete env file copy %s on %s", remote_path, host) + except Exception as e: + logger.warning( + "Failed to delete env file copy %s on %s: %s", remote_path, host, e + ) From 087ee15b66e967937297580fc551c121c7301a20 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 13:28:52 -0700 Subject: [PATCH 06/43] Harden the remote env file copy against a hostile /tmp Two holes in the remote staging path, both found reviewing the feature commit. `umask 077` only governs files the shell creates, and `>` follows symlinks -- so it did not actually guarantee a 0600 copy. The destination path is fully predictable (`/tmp/ventis-env-ventis-ec2--`), so a local user on the remote host could pre-create it world-readable, or point it at a file of their own, and collect the API keys. Remove whatever sits at the path before writing; `rm -f` unlinks a symlink rather than following it, so `cat >` then creates a fresh file under the umask. `_run_cmd` joins its argv with spaces and hands the result to a remote shell unquoted. `_push_file` quoted its path but the cleanup `rm` did not, so a container name containing a space split the `rm` into two arguments that matched nothing -- it exited 0 while the secrets file stayed on the host, and the returncode check logged nothing. Scrub the name down to [A-Za-z0-9_.-] in remote_env_path, which also closes the same gap in the `--env-file` argument and in any future use of that path. Still open, tracked separately: a push that dies mid-transfer can leave a copy behind, since the cleanup only covers the `docker run` that follows. On EC2 the instance is terminated on that path, which disposes of it. --- ventis/controller/global_controller.py | 9 ++++++++- ventis/controller/utils/env_file.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 241daff..0b30307 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -704,10 +704,17 @@ def _push_file(self, local_path, remote_path, host, user=None): `scp`, so a secrets file is never briefly world-readable on the far side. + Anything already sitting at the destination is removed first: `umask` + only governs files the shell creates, and `>` follows symlinks. Without + the `rm`, a local user on the remote host could pre-create the path + world-readable, or point it at a file of their own, and collect + whatever we write there. + Returns: subprocess.CompletedProcess """ - remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + quoted = shlex.quote(remote_path) + remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" with open(local_path, "rb") as f: result = subprocess.run( self._ssh_args(host, user) + [remote_cmd], diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index c83f770..6cd77b6 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -9,11 +9,13 @@ import logging import os +import re from contextlib import contextmanager logger = logging.getLogger(__name__) REMOTE_ENV_DIR = "/tmp" +_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") def resolve_env_file(config, base_dir=None): @@ -46,8 +48,17 @@ def resolve_env_file(config, base_dir=None): def remote_env_path(container_name): - """Where a remote host holds this container's copy of the env file.""" - return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + """ + Where a remote host holds this container's copy of the env file. + + The name is scrubbed down to a shell-safe alphabet. This path is + interpolated into remote commands that `_run_cmd` joins with spaces and + hands to a shell unquoted, so a container name carrying a space would + split the cleanup `rm` into two harmless arguments -- it would exit 0 + while the secrets stayed on the host, with nothing in the log to say so. + """ + safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) + return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" @contextmanager From 905ef93afcb0ffb28830e8792223d5f503fdce94 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 14:05:05 -0700 Subject: [PATCH 07/43] skill: porting an agent project onto Ventis The skill answers, before anyone writes an adapter, the four questions a port turns on: what class Ventis actually loads, what the adapter has to fix, what has to be declared (import root, requirements, env_file), and what to report rather than fix. `ventis-contract.md` pins each claim to the code that makes it true; `traps.md` maps symptoms back to causes. Its one rule: rewrite orchestration, import everything else. A port that restates a prompt, a tool body or a model call has copied the source instead of reusing it. --- .claude/skills/porting-to-ventis/SKILL.md | 228 ++++++++++++++ .claude/skills/porting-to-ventis/traps.md | 52 ++++ .../porting-to-ventis/ventis-contract.md | 281 ++++++++++++++++++ 3 files changed, 561 insertions(+) create mode 100644 .claude/skills/porting-to-ventis/SKILL.md create mode 100644 .claude/skills/porting-to-ventis/traps.md create mode 100644 .claude/skills/porting-to-ventis/ventis-contract.md diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-ventis/SKILL.md new file mode 100644 index 0000000..3faaa27 --- /dev/null +++ b/.claude/skills/porting-to-ventis/SKILL.md @@ -0,0 +1,228 @@ +--- +name: porting-to-ventis +description: Use when porting an existing agent project (LangChain, LangGraph, CrewAI, AutoGen, or a hand-rolled pipeline) onto Ventis +--- + +# Porting an agent project to Ventis + +## A port is four files beside an untouched source tree + +``` +agents/.yaml declares the callable surface +agents/.py the thinnest class that satisfies Ventis +workflow/_workflow.py entry point; calls deploy() +config/global_controller.yaml deployment manifest +config/policy.yaml optional — only to restrict access + NOT EDITED — copied whole into every image +``` + +The two `agents/` files share one basename, as every example does — the stub the +build generates lands where it cannot collide with either. Pick a basename that +is not a module the adapter imports. + +Everything the source already does — prompts, tools, schemas, parsing, retries, +its LLM client — is reached with an `import`. **If your port contains a prompt +string, a tool body, or a model call that already exists in the source, you are +rewriting the project, not porting it.** + +Mechanism and evidence for every claim here: `ventis-contract.md`. +Symptom-to-cause lookup once something breaks: `traps.md`. + +## Step 1 — Survey the source before writing anything + +Ventis loads an agent by doing exactly this: + +```python +module = +agent = getattr(module, )() # no arguments +result = getattr(agent, )(**args) # synchronous +``` + +Everything below is answerable by reading the source, and expensive to answer +after a green build. + +### What the adapter has to fix + +Only the gap between that contract and what the source exposes. Nothing else +belongs in the file. + +| The source exposes | The adapter | +| ----------------------------------------------------- | ---------------------------------------------------------------------- | +| a no-argument class whose methods are synchronous | none — point `entrypoint` at the file it already lives in | +| module-level functions, or `@tool` objects (`StructuredTool` instances, not methods) | a class whose methods call them | +| a compiled graph, a `Crew`, a `GroupChat` | a class, plus the orchestration rewrite of Rule 1 | +| `async def` | a synchronous signature, with `asyncio.run(...)` inside the body | +| framework objects as results (messages, graph state) | the framework's own serializer — `json.dumps` runs on what you return | +| a model client built at import | nothing; `env_file:` carries the key | + +Most LangChain and LangGraph projects are rows 2–5, and none of those rows is a +reason to touch the source. + +### What has to be declared + +The whole tree is copied into the image at its own relative paths, but the +container starts at `/app`, so only what landed flat imports on its own. + +- **The import root.** A `pyproject.toml`, `setup.py` or `setup.cfg` at the root + is what adds `-e .`, and the project's own packaging metadata is what decides + the import root — Ventis never guesses a directory name. No metadata and the + install is skipped, silently. Say so before writing an adapter that imports + across directories; adding metadata to fix it edits the source tree. +- **`requirements:`** on the config entry, a list of strings — anything else is + warned about and dropped whole. It covers what the source imports beyond the + runtime's base list and beyond whatever `-e .` already installed. +- **`env_file:`** in `config/global_controller.yaml`, a path relative to the + project root pointing at a local `.env`, handed to every container as + `docker run --env-file`. The file never enters the image, and `ventis deploy` + fails on a bad path before launching anything. Credentials are not a wall — + declare the keys the source reads and leave the model stack alone. + +**And one thing to report rather than fix.** `-e .` installs +`[project.dependencies]` in the same resolve as `requirements:`, and workshop +projects routinely put their whole toolchain there so that one install sets a +laptop up. Compare each declared name against the source's imports: + +```bash +grep -rl "import \|from " / +``` + +**Report the mismatch and stop there. Do not move entries, and do not delete +them.** A grep finds names, not requirements: a package loaded from a string at +runtime is imported nowhere and still required. Hand over the list, the cost +(Step 3's protobuf wall, a full image build away), and the two places entries can +move to — `[project.optional-dependencies]`, which `-e .` skips, and +`[dependency-groups]`, which never enters package metadata at all. Then let the +owner decide, including deciding not to. + +## Rule 1 — Rewrite orchestration, import everything else + +One kind of source code genuinely cannot be reused: **control flow owned by a +framework runtime.** Ventis has no runtime to execute a LangGraph `StateGraph`, a +CrewAI `Crew` or an AutoGen `GroupChat`, so their wiring is re-expressed as +ordinary Python — in the workflow when it fans out, in the adapter when it does +not. The nodes those edges connected are imported, unchanged. + +| Source code | Treatment | +| ---------------------------------------------------------- | ------------------------------ | +| `StateGraph` / `add_edge` / `Send` / `Command(goto=...)` | rewrite as Python control flow | +| `Crew(...)` / `GroupChat(...)` assembly | rewrite as Python control flow | +| node functions, prompts, tools, schemas, parsers, clients | **import** | +| the source's model provider and SDK | **keep** | + +## Rule 2 — Split only to scale + +**Splitting into multiple agents is a scaling decision, not a format +requirement.** A single agent holding the whole pipeline is a valid Ventis +project. Start there, and hoist a loop into the workflow only when each iteration +fans out to more than one node: + +- a single-agent ReAct loop **stays whole in one agent** — every turn needs the + full message history, and hoisting pushes a growing message list through Redis + each turn. +- a supervisor handing out N tasks, or a `Send` fan-out, is **hoisted** — N + independent runs per request with no shared state is what replicas pay for. + +An agent with `replicas: 1` and no distinct resource profile is a node Ventis +does nothing for. When you do split, say plainly what it buys. + +## Step 2 — Write the files + +**yaml** — argument `type` is pasted into an AST unchecked, so use `str` `int` +`float` `bool` `dict` `list` and nothing else. Every declared argument is +required. Argument names must equal the Python parameter names character for +character. `returns` is read by nothing — use `type: dict` to mark the call sites +the workflow must `json.loads`. + +**adapter** — class name equals `agent.name`, and the constructor takes no +arguments: configuration comes from environment variables read in `__init__`. +What each method has to do is Step 1's table. + +**workflow** — a top-level function **named `main`, taking a single +`query: str`**, plus `deploy(main, port=...)` at the end. + +Ventis itself is permissive here: it serves `POST /` and splats the +request body in as kwargs, so any name and any arguments run. The deployment +platform's test endpoint is not. It posts to a hardcoded `/main`, and its body +schema is `{query: string}` under a strict validator, so a differently named +workflow is unreachable through it and any other key is rejected with 400 in the +control plane, before the request ever reaches the host. Pack richer input into +`query`; every other parameter needs a default, because nothing will ever send +it. + +The file is `exec`'d rather than imported, so `__name__ == "__main__"` is true +and `if __name__ == "__main__":` blocks fire in production. `deploy()` blocks. + +Dispatch every call before resolving any of them: + +```python +futures = [agent.work(item=i) for i in items] # returns immediately +results = [json.loads(f.value()) for f in futures] # .value() blocks +``` + +Fused into one comprehension the calls run one after another. It does not error; +it is just silently serial, and the fan-out is gone. + +**config** — each entry's `name` must match a yaml's `agent.name`, or the build +warns, skips that image, and still exits 0. Write `provider: local` in +**lowercase**: the port reservation compares `provider == "local"` with no +normalization, so `Local` leaves the port unreserved and deploy dies. + +**policy** — optional. Absent, or present with no rules, every service is +allowed. Write one only to restrict, and then list every service the workflow +reaches; a name left out is not a startup error but an `Unauthorized` response +after the request was accepted. + +## Step 3 — Build, then probe the image twice + +`ventis build` never imports your agent, so a green build proves almost nothing — +it prints `Build complete.` and tags every image for a project whose container +dies on startup. Ventis compounds this: the controller writes `healthy` to Redis +*before* loading the agent and a heartbeat keeps re-asserting it, so a container +with no agent stays `healthy` and keeps receiving requests. + +So run the image — tagged `ventis-` — and do what the +container does. **Both probes, in this order. Neither covers the other.** + +```bash +# 1. The runtime itself. This is what CMD runs, and it fails before your agent +# is ever reached, so probing the entrypoint alone will miss it. +docker run --rm ventis- python -c "import local_controller" + +# 2. The agent, loaded the way _load_agent loads it. +docker run --rm ventis- python -c " +import importlib.util, sys +spec = importlib.util.spec_from_file_location('m', '.py') +m = importlib.util.module_from_spec(spec); sys.modules['m'] = m +spec.loader.exec_module(m); m.(); print('ok')" +``` + +Probe 1 exists because the gRPC stack is unpinned: `ventis build` runs +`grpc_tools.protoc` on the **host** and copies the generated `_pb2.py` in, where +a resolver that knows nothing about them picks the protobuf runtime. Protobuf +refuses gencode newer than its runtime, so a source whose dependencies hold +protobuf back kills the container on `import local_controller`. An image with few +requirements passes by coincidence. Report it — the fix belongs in +`generate_docker`, not in the port — and if Step 1 flagged declared-but-unimported +dependencies, name the culprit here. + +Probe 2 exists because `_load_agent` catches every exception, logs it and returns +`None`: a missing dependency, a wrong class name, a constructor that wants +arguments, or a broken import inside the source tree are all invisible until the +first request answers `"No agent loaded"`. + +Then `ventis deploy`, which needs Docker and an importable `grpc_stubs/` **on +this host** (it aborts if they were cleaned after the build). It starts its own +Redis container — do not run one. + +## Never do these + +Each turns a port into a rewrite. They are not judgment calls, and the middle +column is the thought that gets you there. + +| Move | The rationalization | Why it is wrong | +| ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------- | +| Copy a prompt, tool, or schema into the adapter | "so the adapter stands alone" | It exists in the source. Import it — the whole tree is in the image, and a copy drifts. | +| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider, `env_file:` carries its key. | +| Hardcode a key, or ship it in a file you add | "there is no other way in" | `env_file:` is the way in. Never put a secret in the source tree or the build context. | +| Drop or move a dependency | "this one is obviously dev-only" | Obvious to you, not yours to decide. Declare it under `requirements:`; report the rest and let the owner classify. | +| Edit files in the source tree, or vendor it into `agents/` | "just this one line" | The port must leave `git status` on the source clean, and vendoring is copying. | diff --git a/.claude/skills/porting-to-ventis/traps.md b/.claude/skills/porting-to-ventis/traps.md new file mode 100644 index 0000000..163cc48 --- /dev/null +++ b/.claude/skills/porting-to-ventis/traps.md @@ -0,0 +1,52 @@ +# Traps + +Symptom-to-cause lookup for a port that is already written. The mechanism behind +each row is in `ventis-contract.md`. + +## Before any container starts + + +| Symptom | Cause | +| ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `env_file does not exist` on deploy | the path is resolved against the project root you run from; `cmd_deploy` fails before launching anything | +| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved | +| `generated grpc_stubs are missing or not importable` | `ventis build` has not run on this host, or its output was cleaned | +| An agent missing from the deployment | its config `name` matched no yaml; the build logged a warning and exited 0 | + + +## The container dies or serves nothing + + +| Symptom | Cause | +| -------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| Container exits on `import local_controller` | protobuf gencode newer than the resolved runtime; nothing pins the gRPC stack | +| `"No agent loaded"` on the first request | anything below — the agent container's stdout is the only place the cause exists | +| A replica reports `healthy` but answers nothing | same; `healthy` is written before `_load_agent` runs and is never revised | +| `Missing credentials` loading the agent | no `env_file:`, or the key the source reads is not in it | +| `ModuleNotFoundError` for the source's own modules | the project declares no packaging metadata, so `-e .` was skipped and only flat modules import | +| `ModuleNotFoundError` for a third-party package | an import the source needs is missing from the entry's `requirements:` | +| `NameError` importing a stub | a yaml `type` that is not a builtin | + + +## The request is accepted and then goes wrong + + +| Symptom | Cause | +| --------------------------------------------------- | ------------------------------------------------------------------------------------- | +| `TypeError: unexpected keyword argument` | yaml `arguments[].name` != the Python parameter name | +| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched | +| `.value()` returns a `str` of a dict | expected — `json.loads` it | +| `Object of type ... is not JSON serializable` | the adapter returned framework objects; serialize with the framework's own serializer | +| Redis holds `` | the method is `async def`; keep the signature sync and `asyncio.run` inside | +| No faster than the original | calls fused with `.value()`; dispatch all, then resolve all | +| Debug code runs in production | the workflow is `exec`'d, so `__name__ == "__main__"` | + + + +## Through the deployment platform's test endpoint + +| Symptom | Cause | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| 404 from the test endpoint, container healthy | the workflow function is not named `main`; the platform posts to a hardcoded `/main` | +| 400 before the request reaches the host | the body key is not `query`; the platform's schema is strict and rejects everything else | +| The workflow runs but an argument is missing | only `query` is ever sent; every other parameter needs a default | diff --git a/.claude/skills/porting-to-ventis/ventis-contract.md b/.claude/skills/porting-to-ventis/ventis-contract.md new file mode 100644 index 0000000..56e7132 --- /dev/null +++ b/.claude/skills/porting-to-ventis/ventis-contract.md @@ -0,0 +1,281 @@ +# The Ventis contract + +Mechanism behind every rule in `SKILL.md`. Validate against +[CanyonCodeCoreAI/canyoncodecore](https://github.com/CanyonCodeCoreAI/canyoncodecore). + +## Project layout + +| Path | Where it comes from | +| ------------------------------------------- | ---------------------------------------------------------------------------- | +| `agents/*.yaml` | `cli.py` — `glob(agents_dir/*.yaml)` | +| `stubs/`, `grpc_stubs/` | `cli.py` — generated by `ventis build` | +| `config/global_controller.yaml` | `cli.py` — `DEFAULT_CONFIG_PATH`, overridable with `--config` | +| `config/policy.yaml` | `global_controller.py` `_load_policy_rules` — optional | +| the workflow file | the `workflow_file` key on the `type: workflow` config entry | +| the project root | `cli.py` passes `project_dir=os.getcwd()`; build and deploy run from it | +| `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .` | + +## Agent yaml + +```yaml +agent: + name: # required + functions: # optional; absent -> stub class with only __init__ + - name: # required + description: # optional -> becomes the stub method's docstring + arguments: # optional; absent -> no-arg method + - name: # required + type: # optional -> pasted verbatim as an annotation + returns: + type: # read by nothing +``` + +Nothing else is read. Extra keys are ignored silently. + +- **`type` is pasted, never checked.** `_build_stub_method` does + `ast.Name(id=arg["type"])`, and the generated stub imports only `Future` and + `inspect`. Anything that is not a builtin raises `NameError` when the stub is + imported. Use `str` `int` `float` `bool` `dict` `list` — not `List[str]`, not + `Optional[int]`, not a class name. +- **No default values.** `ast.arguments(..., defaults=[])`. Every declared + argument is required at every call site. Optional configuration belongs in the + agent's `__init__`, read from the environment. +- **Parameter names must match exactly.** The controller invokes `method(**args)`. + Order is irrelevant; spelling is not, or the call raises `TypeError` at request + time. +- **`returns` is documentation.** The stub generator never reads it. Its value is + as a marker: `type: dict` tells whoever writes the workflow that this call site + needs `json.loads`. +- **The filename names the stub, not the agent.** `agents/x.yaml` generates + `stubs/x.py`, which the build copies to `/app/x.py` and `/app/agents/x.py`. + Sharing the entrypoint's basename is therefore fine and is the convention: the + entrypoint is copied last, so it wins `/app/x.py` while the stub keeps + `/app/agents/x.py`. What the basename must **not** match is a source module the + adapter imports — `joke_writer.yaml` beside a `joke_writer.py` puts a stub on + top of the source. + +## The three-way name binding + +``` +config entry `name` == agents/x.yaml `agent.name` == the class inside the .py + | + `entrypoint` on that config entry points at the .py +``` + +`cmd_build` looks up each config entry's `name` among the parsed yamls. No match +means a logged warning and **no image built for that agent** — the build still +exits 0. + +## Agent class + +| Requirement | Enforced by | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------- | +| Class name equals `agent.name` | `generate_docker` writes `ENV VENTIS_AGENT_NAME`; `_load_agent` does `getattr(module, name)` | +| Instantiable with no arguments | `_load_agent` calls `agent_class()` | +| Methods are synchronous | the executor calls `method(**args)` — there is no `await` anywhere on this path | +| Return values survive `json.dumps` / `str()` | `_execute_locally` does `json.dumps(result)` for `dict`/`list`, `str(result)` otherwise | + +`self.tools = [...]` appears throughout `examples/` and is read by **nothing** in +`ventis/`. It is decoration. + +**`.value()` always returns a string.** The result is written into Redis as text +and handed back verbatim; there is no deserialization on the way out. + +## Workflow + +The workflow file is **not imported — it is `exec`'d**. +`generate_workflow_docker` writes a `workflow_launcher.py` whose last line is +`exec(open(".py").read())`, and the Dockerfile's CMD runs that launcher. + +- `__name__ == "__main__"` inside your workflow file, so + `if __name__ == "__main__":` blocks **execute in production**. +- `__file__` points at `workflow_launcher.py`. The `sys.path.insert(..., "..", + "stubs")` lines the examples carry resolve to nonexistent paths; imports work + anyway because the stubs and the runtime are placed flat at `/app`, which is + `sys.path[0]`. What makes the *project* tree importable is the editable + install, not `sys.path[0]`. +- `deploy()` ends in `app.run()` and blocks. Nothing after it runs. +- Module-level code runs **once** at container start; the workflow function runs + **per request**, on a Flask worker thread. +- The REST route is `fn.__name__` — rename the function and the endpoint renames + with it. There is no fixed `/main` **in Ventis**. +- The request body is splatted in as kwargs after `_context` is popped off. Any + shape of body works. + +Both of those are why the platform constraint has to be written down rather than +discovered: the control plane's test endpoint posts to a hardcoded `/main` with a +strictly validated `{query: string}` body, so a port must expose `main(query)` to +be reachable through it. Nothing in this repo enforces that or fails without it — +the constraint lives in the control plane (`deploy.routes.ts`, `deploy.agent.ts`, +`deploy.types.ts`), and the transport layer there is generic +(`Record`) while the route schema is not. + +The workflow container also runs its own `LocalController` on 50051 in a +background thread. That is what dispatches the Futures the workflow creates. + +## The build context + +`generate_docker` takes a `project_dir` and `cmd_build` passes it, so the whole +project reaches the image with its relative paths intact — structure is preserved +rather than flattened because packages need it (`src/tools/__init__.py` and +`src/tools/default/__init__.py` flatten to the same name). + +Copy order decides every collision: the swept tree first, then the shared +runtime, then every stub, then the entrypoint. Later writes land on earlier ones. + +| What | Lands where | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| the project tree | at its own relative paths (`agents/x.py`, `src/pkg/mod.py`) | +| the shared runtime | flat at the context root, winning over the swept tree — `local_controller.py` is the CMD, so a project file of that name breaks the container | +| every stub | **twice**: flat at the root (the copy imports resolve), and at `agents/.py`, landing on the real implementation so a peer's name gives the caller its stub | +| the entrypoint | flat, last, winning the flat name back — `VENTIS_AGENT_FILE` is a **basename**, loaded from `/app` | +| `requirements.txt` | written before anything is copied, so the sweep skips a project's own `requirements.txt` and root `Dockerfile` | + +The sweep takes every file, not only `.py` — the editable install reads packaging +metadata, and that metadata points at a README or a license. It skips hidden +files and directories (`.env` holds credentials and the context is what ships), +`__pycache__`, and the three directories `ventis build` generates: +`docker_container`, `stubs`, `grpc_stubs`. + +**An agent is no longer one file**, and a yaml sharing the entrypoint's basename +no longer eats its own stub. What an agent loses is the ability to import *its +own* stub by name — the entrypoint shadows it flat. It can still reach it at +`agents/.py`, and nothing in `examples/` wants to. + +### The import root + +`_install_step` writes +`RUN uv pip install --system -r requirements.txt -e .` when the project root has +a `pyproject.toml`, `setup.py` or `setup.cfg`. That editable install is what +makes a `src/` layout importable, and the source's own packaging metadata is what +decides it — `[tool.setuptools.package-dir] "" = "src"` is a typical case. Ventis +never guesses a directory name. + +Without packaging metadata the install is skipped — silently, no warning. The +tree is still copied, but `sys.path[0]` is `/app`, so only modules that landed +flat resolve. `examples/helloworld`, `finance` and `text2sql` are all in this +state; they work because their entrypoints import nothing from the project tree, +only stubs, which land flat. + +**One resolve, not two.** Requirements and `-e .` go to a single `uv pip install` +so the runtime's list and the source's own dependencies resolve against each +other; a genuine conflict fails the build instead of the first request. It also +forces `COPY . .` ahead of the install, so the requirements layer no longer +caches on its own. + +## Dependencies + +`generate_docker` and `generate_workflow_docker` both take a `requirements` +argument, and `cmd_build` passes `_normalize_requirements(agent_cfg)`. The +runtime's own list is unconditional and not declarable: + +``` +agent: grpcio grpcio-tools redis pyyaml psutil ipdb ipython boto3 +workflow: the same, plus flask sqlalchemy psycopg[binary] +``` + +The declared list is appended verbatim. `_normalize_requirements` takes only a +list of strings — a bare string, a mapping, or a list with a non-string in it +each logs one warning and becomes `[]`, so a malformed entry costs the whole list +rather than the one item. Nothing is deduplicated against the base either. + +**The source's own `pyproject.toml` is installed in the same resolve**, so +`requirements:` covers only what the adapter imports and the source does not +declare. The whole dependency list comes along, dev extras included — a workshop +project's can carry jupyter, matplotlib and pandas into a 1GB agent image. + +### The gRPC stack is unpinned + +`cmd_build` runs `grpc_tools.protoc` on the **host** and copies the resulting +`_pb2.py` into the image, where a resolver that knows nothing about them picks +the protobuf runtime. Protobuf refuses to load gencode newer than its runtime, so +a source whose own dependencies drag protobuf down produces a container that dies +on `import local_controller` — before the agent is reached, with a green build +behind it: + +``` +google.protobuf.runtime_version.VersionError: Detected incompatible Protobuf +Gencode/Runtime versions ... gencode 7.35.1 runtime 6.33.6. +``` + +An image with few requirements resolves to the newest wheel, which happens to be +at least as new as the host's, and passes by coincidence. A fix means prepending +`grpcio==`, `grpcio-tools==` and `protobuf>=` at the host's own versions +(`importlib.metadata.version`) to the generated requirements — `>=` on protobuf +because the guarantee runs one way: a runtime at or above the gencode. + +**Check this first on any port that installs a large dependency tree.** Probing +the entrypoint module is not enough — it does not import `local_controller`, +which is what the container's CMD actually runs. + +## Credentials: `env_file` + +`_launch_locally` passes exactly five `-e` flags, all `VENTIS_*` +(`AGENT_PORT`, `AGENT_HOST`, `REDIS_HOST`, `REDIS_PORT`, `POLL_INTERVAL`), plus +`VENTIS_DATABASE_URL` and `VENTIS_PROJECT_ID` on a workflow entry when +configured. User secrets travel a separate road. + +`env_file:` in `config/global_controller.yaml` names a local `.env`. +`resolve_env_file` expands `~`, resolves a relative path against the project +root, and raises if the file is missing, is not a file, or is unreadable — +`cmd_deploy` calls it before `GlobalController` exists, so a bad path is one +error line rather than a fleet of agents with no keys. + +`env_file_args` then hands the file to `docker run` as `--env-file`. A container +on this machine reads the original; a container on a remote host gets a 0600 copy +under `/tmp`, deleted as soon as `docker run` returns. The explicit `VENTIS_*` +flags are appended first and still win, so a stray `VENTIS_*` line in someone's +`.env` cannot break agent wiring. + +Consequences for a port: + +- A source that constructs its model client at module scope **loads fine**. The + key is in the environment before the adapter imports the source. +- The file never enters the image — the sweep skips hidden files, and the + variables reach the container at run time. +- A missing key is no longer `"No agent loaded"`; it is a provider error on + `/status` after the request was accepted. +- `load_dotenv(".env")` in the source still does nothing: the file is not in the + image and `load_dotenv` is silent about a missing one. + +## `config/policy.yaml` is optional + +`_load_policy_rules` logs `No policy file found ..., skipping policy setup` and +returns `[]`, which `_load_and_write_policies` publishes to every host Redis. +`LocalController._check_policy` returns `True` when the rule list is empty, so +**no policy file means everything is allowed.** + +When rules exist they are sorted most-specific-first (by number of `match` keys) +and the first rule whose `match` keys all equal the request context decides: +`access: all`, or membership in the `access` list. A service left out of the +matching rule answers `Unauthorized: Policy denied access to service 'X'` in the +`/status` response — after the request was accepted. If no rule matches at all, +access is denied. + +## `provider` is case-sensitive in one direction only + +`InstanceManager.ensure_instances` tests `provider == "local"` to decide whether +to reserve a host port. `Local` fails that test, `reserved_port` stays `None`, +and `Local/_runtime.py`'s +`int(spec.get("host_port", spec.get("port", next_host_port(host))))` raises +`int() argument must be a string, a bytes-like object or a real number, not +'NoneType'`. The EC2 test on the same value is `.upper() == "EC2"` everywhere, so +it accepts any casing. Every example that works writes lowercase `local`. + +## Failures are silent + +`_load_agent` catches every exception, logs it, and returns `None`. + +| Stage | A missing credential / dependency / wrong class name | +| --------------- | ------------------------------------------------------ | +| `ventis build` | passes — it never imports your agent | +| `ventis deploy` | passes — the container starts, gRPC listens | +| first request | `"No agent loaded"` | + +The real cause exists only in that container's stdout. + +Worse, the node still advertises itself as usable. `LocalController.__init__` +writes `healthy` to `controller:::status` **before** calling +`_load_agent`, and `_metrics_loop` re-writes `healthy` on every tick. Nothing +downgrades the status when the agent fails to load, so a replica that can serve +nothing keeps being routed to. From a5e8f4442d0e006e777e6e96be5fdcf25f051a98 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 14:05:05 -0700 Subject: [PATCH 08/43] examples: joke_writer, a LangGraph map-reduce ported to Ventis The skill's worked example. `joke_writer.py` is upstream's map-reduce joke graph, unedited: three prompts, two schemas, three nodes. What could not come across is the graph itself -- `StateGraph`, the `Send` fan-out and the `operator.add` reducer are control flow owned by the LangGraph runtime, and Ventis has no runtime to execute them. So the edges are re-expressed as ordinary Python in the workflow, where the fan-out becomes N calls dispatched across JokeAgent's three replicas, and the nodes those edges connected are imported unchanged. The adapter restates nothing. The workflow entry point is `main(query)` because the deployment platform's test endpoint posts to a hardcoded /main with a strictly validated {query: string} body. Ventis itself would serve any name and any kwargs. --- examples/joke_writer/.env.example | 20 ++ examples/joke_writer/LICENSE | 21 +++ examples/joke_writer/README.md | 177 ++++++++++++++++++ examples/joke_writer/agents/joke_agent.py | 63 +++++++ examples/joke_writer/agents/joke_agent.yaml | 53 ++++++ .../joke_writer/config/global_controller.yaml | 76 ++++++++ examples/joke_writer/config/policy.yaml | 17 ++ examples/joke_writer/joke_writer.py | 151 +++++++++++++++ .../joke_writer/workflow/joke_workflow.py | 59 ++++++ 9 files changed, 637 insertions(+) create mode 100644 examples/joke_writer/.env.example create mode 100644 examples/joke_writer/LICENSE create mode 100644 examples/joke_writer/README.md create mode 100644 examples/joke_writer/agents/joke_agent.py create mode 100644 examples/joke_writer/agents/joke_agent.yaml create mode 100644 examples/joke_writer/config/global_controller.yaml create mode 100644 examples/joke_writer/config/policy.yaml create mode 100644 examples/joke_writer/joke_writer.py create mode 100644 examples/joke_writer/workflow/joke_workflow.py diff --git a/examples/joke_writer/.env.example b/examples/joke_writer/.env.example new file mode 100644 index 0000000..b846149 --- /dev/null +++ b/examples/joke_writer/.env.example @@ -0,0 +1,20 @@ +# Copy this to `.env` and fill in the token. `config/global_controller.yaml` +# points `env_file:` at that copy, and it reaches every container as +# `docker run --env-file`. +# +# Keep the real token out of THIS file. `.env.example` is the one exception to +# the build context's exclusion of `.env*`, so whatever is written here is baked +# into the image; `.env` itself never enters the build and never leaves the host. + +# A Bedrock API key -- the long-term kind generated in the console, or a +# short-term one. botocore matches this exact name against bedrock-runtime's +# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by +# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions +# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and +# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. +AWS_BEARER_TOKEN_BEDROCK= + +# Neither is a secret, and both have defaults in joke_writer.py -- they are here +# to name what the source reads. +BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 +AWS_REGION=us-east-1 diff --git a/examples/joke_writer/LICENSE b/examples/joke_writer/LICENSE new file mode 100644 index 0000000..5600729 --- /dev/null +++ b/examples/joke_writer/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LangChain, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md new file mode 100644 index 0000000..930a410 --- /dev/null +++ b/examples/joke_writer/README.md @@ -0,0 +1,177 @@ +# Joke Writer + +A LangGraph map-reduce, ported to Ventis. Derived from +[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) +at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). + +Unlike the other targets in `examples/`, **the source here is not unmodified**. +`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port +an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked +at the credential wall until the model call was rewritten onto Bedrock. That wall +is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed +anyway, and [What the port cost](#what-the-port-cost) is honest about what that +means. + +## Overview + +Given a topic, the graph splits it into sub-topics, writes one joke per +sub-topic in parallel, then picks the best of them. + +1. `generate_topics` — one LLM call, turns the topic into three sub-topics, + validated into `Subjects`. +2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a + `Send` per subject, so this node runs N times per request with no shared + state between the runs. `jokes` is an `Annotated[list, operator.add]`, which + is how the N results merge back into one state. +3. `best_joke` — one LLM call over every joke, returns the winner by index. + +``` + START + | + generate_topics 1 call + | + continue_to_jokes Send x N + / | \ + joke joke joke N calls, no shared state + \ | / + best_joke 1 call + | + END +``` + +### Why this one + +It is the smallest project in reach whose control flow does something a single +process cannot: `Send` fans out to N independent calls per request. Everything +else about it is deliberately boring — four packages, no tools, no external +service, one API key. + +## The port + +| File | What it holds | +| --- | --- | +| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | +| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | +| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | +| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | +| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | +| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | + +Two decisions worth naming: + +**One agent, not three.** `generate_topics` and `best_joke` run once per request +and have no resource profile of their own. Splitting them out would buy two more +images and two more Redis round trips. What is hoisted is the fan-out, and that +is a workflow concern. + +**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, +operator.add]` reducer are control flow owned by the LangGraph runtime, and +Ventis has no runtime to execute them. The workflow dispatches N +`generate_joke` calls across the three replicas and concatenates the results +itself. Every call is dispatched before any is resolved — `.value()` blocks, so +fusing the two lines into one comprehension would silently serialize the fan-out +and remove the reason to be on Ventis at all. + +## What the port cost + +This is no longer upstream's model stack. `ChatOpenAI` and +`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw +converse API, so each node asks for JSON in its prompt and validates the reply +through the same pydantic schema upstream used. `_extract_json` exists only +because `with_structured_output` used to do that work. + +That rewrite is not something the `porting-to-ventis` skill should do on a +user's project — it is the credential wall, and the skill's instruction is to +report it. It was done here deliberately, so that this example is one that +actually deploys. + +**It would not be necessary today.** The rewrite bought one thing: boto3 builds +no client at import, so the agent could be *loaded* with no secret in the +container, back when `_launch_locally` passed five `-e` flags and all five were +`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches +a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module +scope would import fine. What the rewrite still buys is narrower: a module-scope +client turns a missing key into `"No agent loaded"`, while a per-call one turns +it into a real error on `/status`. Worth knowing, not worth a rewrite. + +The example stays on Bedrock because it is the model call that has been end-to-end +verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call +token telemetry onto the future. + +## Running it + +Copy `.env.example` to `.env` and put a Bedrock API key in it: + +```shell +cp .env.example .env +$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... +``` + +`config/global_controller.yaml` points `env_file:` at that file, and every +container gets it as `docker run --env-file`. Nothing in this project reads the +variable: botocore matches the name against `bedrock-runtime`'s signingName and +switches the client from SigV4 to bearer auth on its own, so +`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. +An IAM access key instead of the bearer token works the same way. + +`.env` is gitignored and excluded from the build context — the key is in the +container's environment and not in the image. Deploy checks the path before it +launches anything, so a missing `.env` is one error line rather than three +replicas that come up and fail every request. + +```shell +ventis build +ventis deploy +``` + +```shell +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' +curl http://localhost:8080/status/ +``` + +```json +{"request_id": "cb6cb62d...", "status": "done", "result": { + "topic": "animals", + "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], + "jokes": ["...", "...", "..."], + "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" +}} +``` + +`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in +`joke_writer.py`; neither is a secret. The region has to match the one the key +was issued for. + +### Running the source outside Ventis + +`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat +`bedrock` copy an agent image gets, so the compiled graph still runs on its own +from a checkout of this repo: + +```shell +pip install -e ../.. # the ventis package +pip install langgraph pydantic typing_extensions boto3 +``` + +```python +from joke_writer import graph + +graph.invoke({"topic": "animals"}) +``` + +## Provenance + +Taken from `module-4/studio/`, which holds four unrelated graphs sharing one +directory. Only `map_reduce.py` and its license are here. + +| Left behind | Why | +| --- | --- | +| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | +| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | +| The module-4 notebooks | Teaching material for the same code. | +| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | + +Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` +or `requirements.txt`, exactly as upstream has none for module-4. That is why +`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/agents/joke_agent.py b/examples/joke_writer/agents/joke_agent.py new file mode 100644 index 0000000..8fa3b93 --- /dev/null +++ b/examples/joke_writer/agents/joke_agent.py @@ -0,0 +1,63 @@ +"""Ventis entrypoint for the map-reduce joke writer. + +Nothing here restates the project. The three prompts, the two schemas and the +Bedrock binding all live in `joke_writer.py` and are reached with an import -- +the whole project tree is in the image. + +What could not be reused is the graph itself. `StateGraph`, the `Send` in +`continue_to_jokes` and the `Annotated[list, operator.add]` reducer are control +flow owned by the LangGraph runtime, and Ventis has no runtime to execute them. +That wiring is re-expressed as ordinary Python in workflow/joke_workflow.py, +where the fan-out becomes N dispatched calls across this agent's replicas. The +nodes those edges connected are imported, unchanged. + +The module is imported whole rather than by name so that +`joke_writer.generate_joke` inside a method named `generate_joke` reads as what +it is: the source's node. +""" + +# The source tree. Importing it reads BEDROCK_MODEL_ID and AWS_REGION, imports +# bedrock.py (which builds a RedisClient at module scope) and compiles the graph +# -- but it constructs no API client, so the import needs no credential. +# +# The credential arrives by a different road: `env_file` in +# config/global_controller.yaml hands the container a .env holding +# AWS_BEARER_TOKEN_BEDROCK, and botocore picks that name up by itself. Nothing +# here or in joke_writer.py names it. +# +# Constructing no client at import is no longer what makes this agent loadable -- +# env_file would carry a key to a module-scope client too. It only changes the +# failure: a missing key is an error on /status rather than "No agent loaded". +import joke_writer + + +class JokeAgent(object): + """The graph's nodes, exposed under the class name `agent.name` declares.""" + + # No constructor arguments -- LocalController does `JokeAgent()`. The model + # id and region are the source's own module-level constants, read from the + # environment there; there is nothing to configure here. + + def generate_topics(self, topic: str) -> dict: + """Split a topic into sub-topics. Returns {"subjects": [...]}. + + Synchronous by signature -- the executor calls this with no `await`, and + returning a coroutine would put `` into Redis. + """ + # The node's own state dict goes in, the node's own return comes out. + # Both hold nothing but str and list, so the executor's json.dumps is + # happy without a serializer -- unlike a graph that hands back messages. + return joke_writer.generate_topics({"topic": topic}) + + def generate_joke(self, subject: str) -> dict: + """Write one joke about one subject. Returns {"jokes": ["..."]}. + + The single-element list is the node's own shape: it is what + `Annotated[list, operator.add]` merged N of. The workflow does that + concatenation now. + """ + return joke_writer.generate_joke({"subject": subject}) + + def best_joke(self, topic: str, jokes: list) -> dict: + """Pick the winner. Returns {"best_selected_joke": "..."}.""" + return joke_writer.best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/agents/joke_agent.yaml b/examples/joke_writer/agents/joke_agent.yaml new file mode 100644 index 0000000..fee8607 --- /dev/null +++ b/examples/joke_writer/agents/joke_agent.yaml @@ -0,0 +1,53 @@ +# The graph's three nodes, exposed as three methods on one agent. +# +# One agent, not three. `generate_topics` and `best_joke` run once per request +# and have no resource profile of their own, so splitting them out would add +# two images, two dependency trees and a Redis round trip to buy nothing. What +# is hoisted is the `Send` fan-out, and that is a workflow concern, not a +# second agent: the workflow dispatches N `generate_joke` calls and the +# routing table spreads them across this agent's replicas. +# +# This file's basename names the generated stub, not the agent. Sharing it with +# joke_agent.py is why both land at /app/joke_agent.py -- the entrypoint is +# copied last and wins it, so the agent container loads the real class while the +# stub keeps /app/agents/joke_agent.py for callers. What the basename must not +# match is a source module: a `joke_writer.yaml` would put a stub at +# /app/joke_writer.py, on top of the module the adapter imports. + +agent: + name: JokeAgent + functions: + # Node 1 of the graph. One LLM call, structured output into `Subjects`. + - name: generate_topics + description: Split a topic into three related sub-topics. + arguments: + # Must equal the Python parameter name character for character -- + # LocalController calls method(**args). + - name: topic + type: str + # dict -> the workflow must json.loads what .value() hands back + returns: + type: dict + + # Node 2. The fan-out: one call per sub-topic, no shared state between + # them. This is the only reason this project is on Ventis. + - name: generate_joke + description: Write one joke about one subject. + arguments: + - name: subject + type: str + returns: + type: dict + + # Node 3. The reduce: one call over every joke the fan-out produced. + - name: best_joke + description: Pick the best joke out of the ones written for a topic. + arguments: + - name: topic + type: str + # `list` is a builtin, so the stub's annotation resolves. `list[str]` + # would be pasted into the AST verbatim and NameError on import. + - name: jokes + type: list + returns: + type: dict diff --git a/examples/joke_writer/config/global_controller.yaml b/examples/joke_writer/config/global_controller.yaml new file mode 100644 index 0000000..0aa650b --- /dev/null +++ b/examples/joke_writer/config/global_controller.yaml @@ -0,0 +1,76 @@ +# Deployment manifest for the map-reduce joke writer. +# +# `entrypoint` is the adapter, which imports the untouched-in-shape source tree. +# +# The source has no pyproject.toml, setup.py or setup.cfg, so the Dockerfile's +# `-e .` is skipped -- silently. It does not matter here: `joke_writer.py` sits +# at the project root, so it lands flat at /app, which is sys.path[0]. A source +# laid out under src/ would need its own packaging metadata to import at all. + +agents: + - name: JokeAgent + # The fan-out. `generate_joke` is stateless, so LocalController picks a + # replica at random per call and the workflow's N dispatched calls spread + # across these three. N is whatever the model returns (the prompt asks for + # three sub-topics); replicas bound how many run at once, not how many run. + replicas: 3 + redis_port: 6379 + resources: + cpu: 1 + memory: 1024 + entrypoint: agents/joke_agent.py + provider: local + # What the source imports beyond the runtime's own list, which the generator + # prepends. boto3 is already in it, which is the whole reason the Bedrock + # call needs nothing declared here. The graph is never executed in this + # container, but `joke_writer.py` imports langgraph at module scope, so it + # still has to be installed. + requirements: + - langgraph + - pydantic + - typing_extensions + + - name: Workflow + type: workflow + replicas: 1 + redis_port: 6379 + api_port: 8080 + workflow_file: workflow/joke_workflow.py + provider: local + +poll_interval: 5 + +redis: + host: localhost + port: 6379 + db: 0 + +# `provider` must be lowercase. InstanceManager.launch_all tests +# `provider == "local"` to decide whether to reserve a host port; `Local` fails +# that test, reserved_port stays None, and Local/_runtime.py raises +# `int() argument must be ... not 'NoneType'` before any container starts. +# +# The credential. `_launch_locally` passes exactly five `-e` flags, all VENTIS_*, +# and .env is excluded from the build context, so for a while the only model call +# that could work here was one that needed no secret in the container: boto3 +# resolving an instance role per call. `env_file` is what changed. It points at a +# local .env, unresolved paths relative to this project root, and every container +# gets it as `docker run --env-file` -- so the key is in the environment without +# ever entering the image. +# +# What lands there is AWS_BEARER_TOKEN_BEDROCK. Nothing in this project reads it: +# botocore matches the name against bedrock-runtime's signingName and switches +# the client to bearer auth on its own, so `ventis/llm/bedrock.py` still builds a +# plain `boto3.client("bedrock-runtime")`. +# +# Deploy fails here rather than in a container: resolve_env_file checks the path +# before InstanceManager launches anything, so a missing .env is one error line +# instead of three replicas that come up and then answer +# {"status": "error", "error": "Unable to locate credentials"} on every request. +# +# What it costs: this is no longer upstream's model stack. See README.md. + +# Relative to this project root, same as `entrypoint` and `workflow_file`. +# .env is gitignored and excluded from the build context; .env.example names +# what belongs in it. +env_file: .env diff --git a/examples/joke_writer/config/policy.yaml b/examples/joke_writer/config/policy.yaml new file mode 100644 index 0000000..e5c049f --- /dev/null +++ b/examples/joke_writer/config/policy.yaml @@ -0,0 +1,17 @@ +# Policy-Based Routing Rules — map-reduce joke writer +# Each rule defines a match condition (key-value pairs checked against the +# request context) and an access list of allowed services. +# Rules are evaluated most-specific-first (most matching keys wins). +# An empty match ({}) acts as a default fallback. +# +# This file is not optional. `_load_policy_rules` returns None when it is +# missing and `_load_and_write_policies` then calls len() on that, so +# `ventis deploy` dies in GlobalController.__init__ before any container starts. + +rules: + # Default fallback: the workflow and the one agent behind it. A service left + # out of this list answers "Unauthorized: Policy denied access to service". + - match: {} + access: + - Workflow + - JokeAgent diff --git a/examples/joke_writer/joke_writer.py b/examples/joke_writer/joke_writer.py new file mode 100644 index 0000000..3ad49d5 --- /dev/null +++ b/examples/joke_writer/joke_writer.py @@ -0,0 +1,151 @@ +"""Map-reduce joke writer. + +Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` +(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are +upstream's. The model call is not: upstream builds a `ChatOpenAI` at module +scope, and when this was ported nothing could carry an OPENAI_API_KEY into an +agent container. Bedrock reaches the model through boto3, which builds no client +at import, so the same code loaded with no secret injected. + +`env_file` has since removed that constraint -- the key now travels to the +container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The +rewrite stayed regardless; README.md says what that costs. + +`with_structured_output` went with it. `call_bedrock` is the raw converse API, so +each node asks for JSON in the prompt and validates the reply through the same +pydantic schema upstream used. +""" + +import json +import operator +import os +import re +from typing import Annotated + +from typing_extensions import TypedDict + +from pydantic import BaseModel, ValidationError + +from langgraph.constants import Send +from langgraph.graph import END, StateGraph, START + +# Ventis copies bedrock.py flat into every agent image; the package path is for +# running this module outside a container. +try: + from ventis.llm.bedrock import call_bedrock +except ImportError: + from bedrock import call_bedrock + +# Prompts we will use. Upstream's, plus the JSON instruction that +# `with_structured_output` used to add on our behalf. +subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. +Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" +joke_prompt = """Generate a joke about {subject}. +Respond with JSON only, no prose: {{"joke": "..."}}""" +best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} +Respond with JSON only, no prose: {{"id": 0}}""" + +# LLM. Both are read once at import; the container gets them from its +# environment, and neither is a secret. +MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") +REGION = os.environ.get("AWS_REGION", "us-east-1") + + +def _extract_json(text): + """Pull the first JSON object out of a model reply. + + Even told to answer with JSON only, a model wraps it in a ```json fence or + prefaces it with a sentence. Upstream never needed this because + `with_structured_output` handled it; the converse API does not. + """ + text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) + try: + return json.loads(text) + except json.JSONDecodeError: + pass + # Fall back to the outermost braced span. + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if not match: + raise ValueError(f"joke_writer: no JSON in model output: {text!r}") + return json.loads(match.group(0)) + + +def _ask(prompt, schema, max_tokens): + """One converse() call, validated into `schema`. + + Raising on a bad reply is deliberate. A node that returned a default would + put a plausible-looking wrong answer into the state, and the reduce step + downstream indexes into the jokes list by an id the model chose -- a silent + default there picks the wrong joke instead of failing. + """ + response = call_bedrock( + model_id=MODEL_ID, + messages=[{"role": "user", "content": [{"text": prompt}]}], + inference_config={"maxTokens": max_tokens, "temperature": 0.0}, + region=REGION, + ) + text = response["output"]["message"]["content"][0]["text"] + if not text: + raise ValueError("joke_writer: LLM returned no output.") + try: + return schema(**_extract_json(text)) + except (ValidationError, TypeError) as exc: + raise ValueError( + f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" + ) from exc + + +# Define the state +class Subjects(BaseModel): + subjects: list[str] + +class BestJoke(BaseModel): + id: int + +class OverallState(TypedDict): + topic: str + subjects: list + jokes: Annotated[list, operator.add] + best_selected_joke: str + +def generate_topics(state: OverallState): + prompt = subjects_prompt.format(topic=state["topic"]) + response = _ask(prompt, Subjects, max_tokens=300) + return {"subjects": response.subjects} + +class JokeState(TypedDict): + subject: str + +class Joke(BaseModel): + joke: str + +def generate_joke(state: JokeState): + prompt = joke_prompt.format(subject=state["subject"]) + response = _ask(prompt, Joke, max_tokens=300) + return {"jokes": [response.joke]} + +def best_joke(state: OverallState): + jokes = "\n\n".join(state["jokes"]) + prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) + response = _ask(prompt, BestJoke, max_tokens=100) + if not 0 <= response.id < len(state["jokes"]): + raise ValueError( + f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." + ) + return {"best_selected_joke": state["jokes"][response.id]} + +def continue_to_jokes(state: OverallState): + return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] + +# Construct the graph: here we put everything together to construct our graph +graph_builder = StateGraph(OverallState) +graph_builder.add_node("generate_topics", generate_topics) +graph_builder.add_node("generate_joke", generate_joke) +graph_builder.add_node("best_joke", best_joke) +graph_builder.add_edge(START, "generate_topics") +graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) +graph_builder.add_edge("generate_joke", "best_joke") +graph_builder.add_edge("best_joke", END) + +# Compile the graph +graph = graph_builder.compile() diff --git a/examples/joke_writer/workflow/joke_workflow.py b/examples/joke_writer/workflow/joke_workflow.py new file mode 100644 index 0000000..4b60c61 --- /dev/null +++ b/examples/joke_writer/workflow/joke_workflow.py @@ -0,0 +1,59 @@ +r"""Ventis workflow for the map-reduce joke writer. + +This file is where the graph went. `generate_topics -> continue_to_jokes -> +generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the +three statements below, and the `Send` fan-out is N calls dispatched across +JokeAgent's replicas. + +The function is `main` and its one argument is `query` because the deployment +platform's test endpoint posts to a hardcoded /main with a strictly validated +{query: string} body. Ventis would serve any name and any kwargs -- the route is +the function's __name__ and the body is splatted in -- so nothing here fails if +you rename it; it just stops being reachable through the platform. + + curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' + curl http://localhost:8080/status/ +""" + +import json + +from deploy import deploy +from joke_agent import JokeAgent + + +def main(query): + """Route: POST /main {"query": ""}""" + agent = JokeAgent() + + # Node 1: one call, and the fan-out width comes out of it. The agent's own + # parameter is still `topic` -- that name is bound by joke_agent.yaml and the + # source's node, and only the workflow's entry point is pinned to `query`. + subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] + + # `continue_to_jokes`, re-expressed. Every call is dispatched before any + # of them is resolved -- .value() blocks, so fusing these two lines into one + # comprehension would run the jokes one after another. It would not error; + # the fan-out would just be gone, and with it the reason to be on Ventis. + futures = [agent.generate_joke(subject=s) for s in subjects] + written = [json.loads(f.value()) for f in futures] + + # `Annotated[list, operator.add]`, re-expressed: the reducer that merged N + # single-joke lists back into one list was part of the graph, not of a node. + written_jokes = [joke for result in written for joke in result["jokes"]] + + # Node 3: the reduce. `list` in the yaml is what lets this argument through. + best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) + + return { + "topic": query, + "subjects": subjects, + "jokes": written_jokes, + "best_selected_joke": best["best_selected_joke"], + } + + +# This file is exec'd, not imported, so __name__ == "__main__" here and any +# `if __name__ == "__main__":` block would run in production. deploy() blocks +# on app.run(); nothing after it executes. +deploy(main, port=8080) From 01a70f27a66376aad350a063850658f66b5f581a Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 21:54:48 -0700 Subject: [PATCH 09/43] Fix PR #51 regression: stubs must also land flat PR #51 moved each generated stub from the context root to its agent's entrypoint path, so the stub overwrites the real implementation the new sweep places there. That part is right, but it was a move rather than an addition, and the flat copy is what every caller actually imports: ModuleNotFoundError: No module named 'joke_agent' File "/app/workflow_launcher.py", line 21, in exec(open("joke_workflow.py").read()) /app is sys.path[0], so `from joke_agent import JokeAgent` needs the stub at /app/joke_agent.py. Every example's workflow does this -- helloworld's `from example_agent import ExampleAgent` breaks the same way. _stub_destination becomes _stub_destinations and returns both paths, flat first. An agent's own entrypoint is still copied afterwards and wins its flat name back, so in an agent image /app/joke_agent.py is the real adapter while /app/agents/joke_agent.py is the stub; in a workflow image both are the stub. Verified end to end on examples/joke_writer against live Bedrock: the three generate_joke calls landed on replicas 0, 1 and 2, one each. --- ventis/stub_generator.py | 44 +++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 9ea6c99..992e3ed 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -285,18 +285,32 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, stub_entrypoints): - """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" +def _stub_destinations(stub_file, stub_entrypoints): + """Every path a stub is copied to, flat name first. + + The flat copy is what callers import -- `from joke_agent import JokeAgent` + resolves against /app, which is sys.path[0]. The entrypoint copy overwrites + the real implementation the sweep placed there, so importing a peer by its + path gives the caller a stub rather than the peer's own code. An agent's own + entrypoint is copied after this and wins its flat name back. + """ basename = os.path.basename(stub_file) + destinations = [basename] + entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") - if not normalized.startswith("/") and ".." not in normalized.split("/"): - return normalized - print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + if normalized.startswith("/") or ".." in normalized.split("/"): + print( + f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat only" + ) + elif normalized != basename: + destinations.append(normalized) elif stub_entrypoints: - print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") - return basename + print( + f" Warning: no entrypoint mapping for stub {basename}, placing flat only" + ) + return destinations def _copy_files(output_dir, files_to_copy): @@ -390,12 +404,8 @@ def generate_docker( # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) + for dst in _stub_destinations(stub_file, stub_entrypoints or {}): + files_to_copy.append((os.path.abspath(stub_file), dst)) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -514,12 +524,8 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) + for dst in _stub_destinations(stub_file, stub_entrypoints or {}): + files_to_copy.append((os.path.abspath(stub_file), dst)) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): From 5d39ab270969f9790bc8aaeb605fe119cd456534 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 15:30:47 -0700 Subject: [PATCH 10/43] Support managed-deployment secrets alongside self-hosted env_file --- README.md | 9 ++++ ventis/controller/utils/env_file.py | 74 +++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5236328..81d944f 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,15 @@ cp -r ../examples/* ./ #### Step 1: Configure the Global Controller Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +#### Step 1.1: Passing secrets to agents (optional) + +Agents that need API keys read them from environment variables. Point `env_file` at a `.env` file to have Ventis inject it into every agent container: + +```yaml +# config/global_controller.yaml +env_file: .env +``` + #### Step 2: Build the project ```bash ventis build diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index 6cd77b6..b1f9381 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -1,10 +1,11 @@ """ Pass user secrets (API keys and friends) into agent containers. -The user points `env_file` in `config/global_controller.yaml` at a local -`.env` file. Containers on this machine read that file directly; containers -on a remote host get a short-lived 0600 copy. Either way the file reaches -Docker as `--env-file`. +Two scopes, never mixed: `env_file` in the config (self-hosted), and +`DEFAULT_SECRETS_FILE` (managed -- the project's `.env` was never uploaded +there). Below `resolve_env_file` both are just a path: containers on this +machine read it directly, remote ones get a short-lived 0600 copy, and either +way it reaches Docker as `--env-file`. """ import logging @@ -17,18 +18,60 @@ REMOTE_ENV_DIR = "/tmp" _UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") +# Where a managed deployment leaves the user's secrets. /var/run is tmpfs, so +# the file dies with the host instead of persisting on disk. +DEFAULT_SECRETS_FILE = "/var/run/ventis/secrets.env" + + +def platform_secrets_file(): + """ + The path a managed deployment leaves the user's secrets at. + + `VENTIS_SECRETS_FILE` overrides it for tests and for deployments that + cannot write under /var/run. Nothing sets it in normal operation. + """ + return os.environ.get("VENTIS_SECRETS_FILE", DEFAULT_SECRETS_FILE) + def resolve_env_file(config, base_dir=None): """ - Return the absolute path of the configured env file, or None when unset. + Return the absolute path of the env file to hand containers, or None. - Relative paths resolve against `base_dir` (default: the current working - directory), matching how `entrypoint` and `workflow_file` are resolved. + A file at `platform_secrets_file()` wins over `env_file`. A convention + beating an explicit setting only makes sense because the two describe + different machines: `env_file: .env` describes the user's own, and + carrying that config to a managed deployment does not make the `.env` + exist there. So it warns rather than raising. + + The platform must create that file unconditionally, empty when no secrets + are configured -- its presence means "managed", not "has secrets". Raises: ValueError: the file is configured but unusable. Deploy should fail here rather than start a fleet of agents with no API keys. """ + platform_path = platform_secrets_file() + if os.path.isfile(platform_path): + if not os.access(platform_path, os.R_OK): + raise ValueError(f"platform secrets file is not readable: {platform_path}") + if config.get("env_file"): + logger.warning( + "env_file %r is ignored in a managed deployment: secrets come " + "from the platform. Configure them on the platform instead.", + config["env_file"], + ) + return platform_path + + return _resolve_project_env_file(config, base_dir) + + +def _resolve_project_env_file(config, base_dir): + """ + Resolve the `env_file` a self-hosted deployment points at, or None. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + """ raw = config.get("env_file") if not raw: return None @@ -72,10 +115,11 @@ def env_file_args(controller, host, user, container_name, is_local): itself. Keep that body tight around `docker run` so the copy is never on the host longer than it has to be. - Yields an empty list when no `env_file` is configured. + Yields an empty list when no env file is configured, and when the one + configured is empty. """ env_file_path = getattr(controller, "env_file_path", None) - if not env_file_path: + if not env_file_path or _has_no_variables(env_file_path): yield [] return @@ -91,6 +135,18 @@ def env_file_args(controller, host, user, container_name, is_local): _remove_remote_copy(controller, remote_path, host, user) +def _has_no_variables(path): + """ + An empty platform file is routine, so skip it and spare a remote host a + copy pushed and deleted per container. A file that vanished since + resolution is not "empty" -- let it through so Docker reports it. + """ + try: + return os.path.getsize(path) == 0 + except OSError: + return False + + def _remove_remote_copy(controller, remote_path, host, user): """Delete a remote copy. Best effort -- never masks the caller's error.""" try: From d1ce61d1e6e1cdea870211dba21763e866e4a9df Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 15:35:34 -0700 Subject: [PATCH 11/43] skill: make every MUST checkable, and add validate.py to check them A skill enforces nothing. A prose MUST is a hope that the model reads and obeys it, and this one had obligation spread through paragraphs -- "has to fix", "needs a default" -- while `must` also sat on things that were merely true. So the word now marks exactly one thing: a rule whose violation breaks the port. Set in capitals, indexed in the MUST list, and nowhere else in the file, so `grep -nE '\b(MUST|NEVER)\b' SKILL.md` returns the rules and only the rules. `validate.py` decides the twenty-two of them a machine can. It parses YAML and Python to an AST and never imports the port, so it runs on a tree whose dependencies are not installed. Errors are provable contract violations and exit 1; the rewrite smells -- a prompt copied out of the source, a hardcoded key, a dirty source tree -- are warnings that exit 0, because a heuristic cannot be allowed to block a correct port. What the review turned up is that a block of this skill's hardest claims is false against the branch it merges into. `env_file:` needs PR #53. `-e .` and the all-file sweep have no PR at all -- they exist only on an abandoned branch. So the script probes the importable ventis for each feature and prints what it found, and a rule whose feature is missing is reported UNAVAILABLE rather than silently skipped. Both docs now name the PR behind every claim that does not hold on main. Two claims were simply wrong. `policy.yaml` is optional, as SKILL.md said -- but past the isfile() guard the read is unguarded, so a present-and-empty file kills `ventis deploy` before any container starts. And the contract's "the sweep takes every file" belongs to the same unproposed branch as `-e .`; `_sweep_py_files` takes `.py` only. Calibrated against all five examples until every remaining finding was true. --- .claude/skills/porting-to-ventis/SKILL.md | 194 +- .claude/skills/porting-to-ventis/traps.md | 38 +- .claude/skills/porting-to-ventis/validate.py | 1766 +++++++++++++++++ .../porting-to-ventis/ventis-contract.md | 67 +- 4 files changed, 1985 insertions(+), 80 deletions(-) create mode 100755 .claude/skills/porting-to-ventis/validate.py diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-ventis/SKILL.md index 3faaa27..4c887cb 100644 --- a/.claude/skills/porting-to-ventis/SKILL.md +++ b/.claude/skills/porting-to-ventis/SKILL.md @@ -5,6 +5,19 @@ description: Use when porting an existing agent project (LangChain, LangGraph, C # Porting an agent project to Ventis +## How to read the rules in this file + +Set in capitals, **MUST** and **NEVER** mark a rule whose violation breaks the +port: the build skips an image, `ventis deploy` dies, or the first request +fails. Every one is indexed in [The MUST list](#the-must-list), and every one a +machine can decide is checked by `validate.py`. Nothing else in this file is +written in capitals, so `grep -nE '\b(MUST|NEVER)\b' SKILL.md` returns the +rules and only the rules. + +Everything else is stated as fact, in the indicative — how Ventis behaves, and +what follows from it. There is no "should", and nothing is left to taste that +does not have to be. + ## A port is four files beside an untouched source tree ``` @@ -21,9 +34,9 @@ build generates lands where it cannot collide with either. Pick a basename that is not a module the adapter imports. Everything the source already does — prompts, tools, schemas, parsing, retries, -its LLM client — is reached with an `import`. **If your port contains a prompt -string, a tool body, or a model call that already exists in the source, you are -rewriting the project, not porting it.** +its LLM client — is reached with an `import`. **A port that contains a prompt +string, a tool body, or a model call that already exists in the source is a +rewrite of the project, not a port of it.** Mechanism and evidence for every claim here: `ventis-contract.md`. Symptom-to-cause lookup once something breaks: `traps.md`. @@ -41,7 +54,7 @@ result = getattr(agent, )(**args) # synchronous Everything below is answerable by reading the source, and expensive to answer after a green build. -### What the adapter has to fix +### What the adapter fixes Only the gap between that contract and what the source exposes. Nothing else belongs in the file. @@ -53,46 +66,56 @@ belongs in the file. | a compiled graph, a `Crew`, a `GroupChat` | a class, plus the orchestration rewrite of Rule 1 | | `async def` | a synchronous signature, with `asyncio.run(...)` inside the body | | framework objects as results (messages, graph state) | the framework's own serializer — `json.dumps` runs on what you return | -| a model client built at import | nothing; `env_file:` carries the key | +| a model client built at import | nothing, once a credential can reach the container | Most LangChain and LangGraph projects are rows 2–5, and none of those rows is a reason to touch the source. -### What has to be declared +### What the config declares The whole tree is copied into the image at its own relative paths, but the container starts at `/app`, so only what landed flat imports on its own. -- **The import root.** A `pyproject.toml`, `setup.py` or `setup.cfg` at the root - is what adds `-e .`, and the project's own packaging metadata is what decides - the import root — Ventis never guesses a directory name. No metadata and the - install is skipped, silently. Say so before writing an adapter that imports - across directories; adding metadata to fix it edits the source tree. -- **`requirements:`** on the config entry, a list of strings — anything else is - warned about and dropped whole. It covers what the source imports beyond the - runtime's base list and beyond whatever `-e .` already installed. -- **`env_file:`** in `config/global_controller.yaml`, a path relative to the +- **`requirements:`** on the config entry, a list of strings. It covers what the + source imports beyond the runtime's own base list. A malformed value costs the + whole list, not the one bad item: `_normalize_requirements` logs one warning + and returns `[]`, and the build still succeeds with none of them installed. + +- **The import root** — *needs a Ventis change that has no PR.* An editable + install (`-e .`) driven by a `pyproject.toml`, `setup.py` or `setup.cfg` at the + root is what makes a `src/` layout importable, and the project's own packaging + metadata is what decides the root. `_install_step` exists only on + `jiajunh/can-228-create-a-skill-to-convert-a-langchain-project-to-ventis`, which + nobody has proposed merging. Until it lands, **only modules that land flat at + `/app` import at all** — an adapter reaching into `src/pkg/` raises + `ModuleNotFoundError` inside `_load_agent`, and the first request answers + `"No agent loaded"`. `validate.py` probes for the feature and reports which + rule is in force. + +- **`env_file:`** — *needs PR #53, open against main.* A path relative to the project root pointing at a local `.env`, handed to every container as `docker run --env-file`. The file never enters the image, and `ventis deploy` - fails on a bad path before launching anything. Credentials are not a wall — - declare the keys the source reads and leave the model stack alone. + fails on a bad path before launching anything. Without that PR, the only + variables reaching a container are five `VENTIS_*` names, and a config that + sets `env_file:` is setting a key nothing reads — the credential is silently + dropped and the failure surfaces as a provider error on the first request. -**And one thing to report rather than fix.** `-e .` installs -`[project.dependencies]` in the same resolve as `requirements:`, and workshop -projects routinely put their whole toolchain there so that one install sets a -laptop up. Compare each declared name against the source's imports: +**And one thing to report rather than fix.** Where the editable install exists, +`-e .` installs `[project.dependencies]` in the same resolve as `requirements:`, +and workshop projects routinely put their whole toolchain there so that one +install sets a laptop up. Compare each declared name against the source's +imports: ```bash grep -rl "import \|from " / ``` -**Report the mismatch and stop there. Do not move entries, and do not delete -them.** A grep finds names, not requirements: a package loaded from a string at -runtime is imported nowhere and still required. Hand over the list, the cost -(Step 3's protobuf wall, a full image build away), and the two places entries can -move to — `[project.optional-dependencies]`, which `-e .` skips, and -`[dependency-groups]`, which never enters package metadata at all. Then let the -owner decide, including deciding not to. +**Report the mismatch and stop there.** A grep finds names, not requirements: a +package loaded from a string at runtime is imported nowhere and still required. +Hand over the list, the cost (Step 4's protobuf wall, a full image build away), +and the two places entries can move to — `[project.optional-dependencies]`, which +`-e .` skips, and `[dependency-groups]`, which never enters package metadata at +all. Then let the owner decide, including deciding not to. ## Rule 1 — Rewrite orchestration, import everything else @@ -127,18 +150,20 @@ does nothing for. When you do split, say plainly what it buys. ## Step 2 — Write the files -**yaml** — argument `type` is pasted into an AST unchecked, so use `str` `int` -`float` `bool` `dict` `list` and nothing else. Every declared argument is -required. Argument names must equal the Python parameter names character for -character. `returns` is read by nothing — use `type: dict` to mark the call sites -the workflow must `json.loads`. +**yaml** — argument `type` is pasted into an AST unchecked, so `str` `int` +`float` `bool` `dict` `list` are the whole vocabulary; the generated stub imports +nothing else, and anything that is not a builtin raises `NameError` when the stub +is imported. Every declared argument is required at every call site — the +generator emits no defaults. `returns` is read by nothing; its value is as a +marker, where `type: dict` tells whoever writes the workflow that this call site +needs `json.loads`. -**adapter** — class name equals `agent.name`, and the constructor takes no +**adapter** — the class name is `agent.name` and the constructor takes no arguments: configuration comes from environment variables read in `__init__`. What each method has to do is Step 1's table. -**workflow** — a top-level function **named `main`, taking a single -`query: str`**, plus `deploy(main, port=...)` at the end. +**workflow** — a top-level function named `main`, taking a single `query: str`, +plus `deploy(main, port=...)` at the end. Ventis itself is permissive here: it serves `POST /` and splats the request body in as kwargs, so any name and any arguments run. The deployment @@ -162,26 +187,89 @@ results = [json.loads(f.value()) for f in futures] # .value() blocks Fused into one comprehension the calls run one after another. It does not error; it is just silently serial, and the fan-out is gone. -**config** — each entry's `name` must match a yaml's `agent.name`, or the build +**config** — each entry's `name` matches a yaml's `agent.name`, or the build warns, skips that image, and still exits 0. Write `provider: local` in **lowercase**: the port reservation compares `provider == "local"` with no -normalization, so `Local` leaves the port unreserved and deploy dies. +normalization, so `Local` leaves the port unreserved and deploy dies. `replicas` +is an integer — the list form that `_get_replica_placements` accepts raises +`TypeError` in `InstanceManager`. + +**policy** — optional. Absent, every service is allowed. Present, it is read +strictly: an empty file, or a null `rules:`, is an `AttributeError` inside +`GlobalController.__init__` that kills `ventis deploy` before a container starts. +Write one only to restrict, and then remember that the first matching rule +decides — a service missing from the rule that matched is not a startup error but +an `Unauthorized` response after the request was accepted. + +## The MUST list + +Every hard rule in this file, and the check that decides it. `--` marks the ones +only a human can judge; they are the reason a clean validator run is a floor and +not a ceiling. + +| # | The rule | Check | +| --- | ------------------------------------------------------------------------- | ---------- | +| M1 | The entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | That class MUST construct with no arguments | V007 | +| M3 | A yaml `arguments[].name` MUST equal the Python parameter name exactly | V008 | +| M4 | A yaml `type` MUST be a bare builtin | V010 | +| M5 | A method backing a yaml function MUST be synchronous | V009 | +| M6 | Every config entry `name` MUST match some yaml `agent.name` | V003, V005 | +| M7 | Two config entry `name`s MUST differ by more than case | V004 | +| M8 | `provider` MUST be lowercase `local` (EC2 takes any casing) | V012 | +| M9 | `replicas` MUST be an integer | V013 | +| M10 | `requirements:` MUST be a list of strings | V014 | +| M11 | The workflow MUST expose `main(query)`; other parameters MUST have defaults | V015, V016 | +| M12 | The workflow MUST NEVER carry an `if __name__ == "__main__":` block | V017 | +| M13 | A fan-out MUST dispatch every call before resolving any | V018 | +| M14 | No project module MUST take the flat name of a runtime file or a stub | V019, V020 | +| M15 | `policy.yaml` MUST be absent, or MUST carry a non-empty `rules:` list | V021 | +| M16 | An EC2 entry MUST declare `instance_type`, and `ec2:` MUST be complete | V022 | +| M17 | NEVER copy a prompt, tool, or schema that exists in the source | W001 | +| M18 | NEVER hardcode a credential, or ship one in the build context | W003 | +| M19 | NEVER edit the source tree, and NEVER vendor it into `agents/` | W002 | +| M20 | NEVER swap the LLM provider the source uses | -- | +| M21 | NEVER move or drop a declared dependency — report it and stop | -- | +| M22 | Framework control flow MUST be rewritten; everything else MUST be imported | -- | + +`validate.py` reports more than this list — V001, V002 and W005, W006 catch +files that do not parse and imports the container cannot satisfy — but every row +here has a check behind it. + +Two more rules apply only where the Ventis you are targeting supports them, +which `validate.py` probes for rather than assumes: + +| # | The rule | Needs | Check | +| --- | ----------------------------------------------------------------- | ----------------------- | ----- | +| M23 | `env_file:` MUST resolve to a readable file, and MUST be the only way a credential enters | PR #53 | V030 | +| M24 | An adapter import from outside the project root MUST have packaging metadata behind it | no PR yet | V031 | + +## Step 3 — Validate + +```bash +python /validate.py . +``` + +Run it before building. `ventis build` never imports your agent, and the +controller writes `healthy` to Redis *before* `_load_agent` runs — so a green +build and a healthy replica are both compatible with a container that can serve +nothing. This script is the only stage that reads what you actually wrote. -**policy** — optional. Absent, or present with no rules, every service is -allowed. Write one only to restrict, and then list every service the workflow -reaches; a name left out is not a startup error but an `Unauthorized` response -after the request was accepted. +It parses; it never imports the port, so it is safe to run on a tree whose +dependencies are not installed. Errors are provable contract violations and exit +1. Warnings are the rewrite smells — M16, M17, M18 — and exit 0 on their own, +because a heuristic cannot be allowed to block a correct port; `--strict` +promotes them for CI. `--json` emits the findings as data. -## Step 3 — Build, then probe the image twice +The header prints which capability-gated rules are in force. A rule whose feature +is missing is reported `UNAVAILABLE`, never silently skipped. -`ventis build` never imports your agent, so a green build proves almost nothing — -it prints `Build complete.` and tags every image for a project whose container -dies on startup. Ventis compounds this: the controller writes `healthy` to Redis -*before* loading the agent and a heartbeat keeps re-asserting it, so a container -with no agent stays `healthy` and keeps receiving requests. +## Step 4 — Build, then probe the image twice -So run the image — tagged `ventis-` — and do what the -container does. **Both probes, in this order. Neither covers the other.** +`ventis build` prints `Build complete.` and tags every image for a project whose +container dies on startup. So run the image — tagged +`ventis-` — and do what the container does. **Both probes, +in this order. Neither covers the other.** ```bash # 1. The runtime itself. This is what CMD runs, and it fails before your agent @@ -222,7 +310,7 @@ column is the thought that gets you there. | Move | The rationalization | Why it is wrong | | ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------- | | Copy a prompt, tool, or schema into the adapter | "so the adapter stands alone" | It exists in the source. Import it — the whole tree is in the image, and a copy drifts. | -| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider, `env_file:` carries its key. | -| Hardcode a key, or ship it in a file you add | "there is no other way in" | `env_file:` is the way in. Never put a secret in the source tree or the build context. | +| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider. | +| Hardcode a key, or ship it in a file you add | "there is no other way in" | The build sweeps the project into every image. Where `env_file:` exists it is the way in; where it does not, say so and stop. | | Drop or move a dependency | "this one is obviously dev-only" | Obvious to you, not yours to decide. Declare it under `requirements:`; report the rest and let the owner classify. | -| Edit files in the source tree, or vendor it into `agents/` | "just this one line" | The port must leave `git status` on the source clean, and vendoring is copying. | +| Edit files in the source tree, or vendor it into `agents/` | "just this one line" | The port leaves `git status` on the source clean, and vendoring is copying. | diff --git a/.claude/skills/porting-to-ventis/traps.md b/.claude/skills/porting-to-ventis/traps.md index 163cc48..4c25940 100644 --- a/.claude/skills/porting-to-ventis/traps.md +++ b/.claude/skills/porting-to-ventis/traps.md @@ -1,17 +1,22 @@ # Traps Symptom-to-cause lookup for a port that is already written. The mechanism behind -each row is in `ventis-contract.md`. +each row is in `ventis-contract.md`. Rows marked with a check id are decided +before any of this happens by `validate.py`. ## Before any container starts | Symptom | Cause | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| `env_file does not exist` on deploy | the path is resolved against the project root you run from; `cmd_deploy` fails before launching anything | -| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved | +| `env_file does not exist` on deploy | the path is resolved against the project root you run from; `cmd_deploy` fails before launching anything (V030) | +| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved (V012) | +| `TypeError: int() argument ...` naming `replicas` | `replicas:` is a list; `_get_replica_placements` accepts that shape but `InstanceManager` calls `int()` on it (V013) | +| `AttributeError` inside `GlobalController.__init__` | `config/policy.yaml` exists but is empty, or its `rules:` is null. Absent would have been fine (V021) | +| `EC2 deploy preflight failed: missing ec2 config keys`| no top-level `ec2:` block, or an incomplete one. `ssh_user` passes the CLI's shorter list and fails later at provision (V022) | | `generated grpc_stubs are missing or not importable` | `ventis build` has not run on this host, or its output was cleaned | -| An agent missing from the deployment | its config `name` matched no yaml; the build logged a warning and exited 0 | +| An agent missing from the deployment | its config `name` matched no yaml, or its entry has no `entrypoint`; the build logged a warning and exited 0 (V003, V005) | +| Two agents, one image | two config `name`s differing only in case — both tag `ventis-` and the second overwrites the first (V004) | ## The container dies or serves nothing @@ -23,9 +28,12 @@ each row is in `ventis-contract.md`. | `"No agent loaded"` on the first request | anything below — the agent container's stdout is the only place the cause exists | | A replica reports `healthy` but answers nothing | same; `healthy` is written before `_load_agent` runs and is never revised | | `Missing credentials` loading the agent | no `env_file:`, or the key the source reads is not in it | -| `ModuleNotFoundError` for the source's own modules | the project declares no packaging metadata, so `-e .` was skipped and only flat modules import | -| `ModuleNotFoundError` for a third-party package | an import the source needs is missing from the entry's `requirements:` | -| `NameError` importing a stub | a yaml `type` that is not a builtin | +| `ModuleNotFoundError` for the source's own modules | only modules that land flat at `/app` import; on a Ventis with `-e .`, the project also declares no packaging metadata (V031) | +| `ModuleNotFoundError` for a third-party package | an import the source needs is missing from the entry's `requirements:` (W006) | +| `ModuleNotFoundError` for nothing in particular | `requirements:` was not a list of strings, so the whole list was dropped with one warning (V014) | +| `NameError` importing a stub | a yaml `type` that is not a builtin (V010) | +| A peer's real code behaves like an empty stub | a project module at the root shares a basename with an `agents/*.yaml`, and the stub is copied over it (V020) | +| The container dies on a Ventis module name | a project module at the root is called `local_controller.py`, `deploy.py`, `future.py` ... — the runtime is copied flat over it (V019) | ## The request is accepted and then goes wrong @@ -33,13 +41,13 @@ each row is in `ventis-contract.md`. | Symptom | Cause | | --------------------------------------------------- | ------------------------------------------------------------------------------------- | -| `TypeError: unexpected keyword argument` | yaml `arguments[].name` != the Python parameter name | -| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched | +| `TypeError: unexpected keyword argument` | yaml `arguments[].name` != the Python parameter name (V008) | +| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched (V021) | | `.value()` returns a `str` of a dict | expected — `json.loads` it | | `Object of type ... is not JSON serializable` | the adapter returned framework objects; serialize with the framework's own serializer | -| Redis holds `` | the method is `async def`; keep the signature sync and `asyncio.run` inside | -| No faster than the original | calls fused with `.value()`; dispatch all, then resolve all | -| Debug code runs in production | the workflow is `exec`'d, so `__name__ == "__main__"` | +| Redis holds `` | the method is `async def`; keep the signature sync and `asyncio.run` inside (V009) | +| No faster than the original | calls fused with `.value()`; dispatch all, then resolve all (V018) | +| Debug code runs in production | the workflow is `exec`'d, so `__name__ == "__main__"` (V017) | @@ -47,6 +55,6 @@ each row is in `ventis-contract.md`. | Symptom | Cause | | --------------------------------------------------- | --------------------------------------------------------------------------------------------- | -| 404 from the test endpoint, container healthy | the workflow function is not named `main`; the platform posts to a hardcoded `/main` | -| 400 before the request reaches the host | the body key is not `query`; the platform's schema is strict and rejects everything else | -| The workflow runs but an argument is missing | only `query` is ever sent; every other parameter needs a default | +| 404 from the test endpoint, container healthy | the workflow function is not named `main`; the platform posts to a hardcoded `/main` (V016) | +| 400 before the request reaches the host | the body key is not `query`; the platform's schema is strict and rejects everything else (V016) | +| The workflow runs but an argument is missing | only `query` is ever sent; every other parameter needs a default (V016) | diff --git a/.claude/skills/porting-to-ventis/validate.py b/.claude/skills/porting-to-ventis/validate.py new file mode 100755 index 0000000..86646d1 --- /dev/null +++ b/.claude/skills/porting-to-ventis/validate.py @@ -0,0 +1,1766 @@ +#!/usr/bin/env python3 +"""Deterministic checks for a Ventis port. + +Every rule in SKILL.md marked MUST or NEVER that a machine can decide is decided +here. Nothing in this file imports the port -- YAML is parsed, Python is parsed +to an AST, and neither is executed. A port that fails here fails at build, at +deploy, or on its first request; `ventis build` will not tell you, because it +never imports your agent, and a replica will not tell you either, because the +controller writes `healthy` to Redis before `_load_agent` runs. + + python validate.py [project_dir] [-c config/global_controller.yaml] + [--json] [--strict] + +Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. + +Some rules depend on Ventis features that are not on `main`. Rather than assume, +this script probes the importable `ventis` package and reports each capability +with the PR that carries it. A check whose capability is absent is reported as +UNAVAILABLE, never silently skipped. +""" + +import argparse +import ast +import builtins +import json +import os +import re +import subprocess +import sys +from typing import ClassVar + +try: + import yaml +except ImportError: # pragma: no cover - pyyaml is a Ventis dependency + sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") + raise SystemExit(2) from None + + +DEFAULT_CONFIG_PATH = "config/global_controller.yaml" + +# Copied flat into every image over the swept project tree, so a project module +# landing flat under one of these names is overwritten. +# ventis/stub_generator.py generate_docker / generate_workflow_docker. +RUNTIME_FLAT_NAMES = frozenset( + { + "future.py", + "ventis_context.py", + "local_controller.py", + "local_controller_frontend.py", + "redis_client.py", + "grpc_options.py", + "gpu_metrics.py", + "bedrock.py", + "deploy.py", + "session_logging.py", + "workflow_launcher.py", + } +) + +# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. +BASE_AGENT_REQUIREMENTS = [ + "grpcio", + "grpcio-tools", + "redis", + "pyyaml", + "psutil", + "ipdb", + "ipython", + "boto3", +] +BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + [ + "flask", + "sqlalchemy", + "psycopg[binary]", +] + +# ventis/cli.py EC2_REQUIRED_CONFIG_KEYS is shorter than what EC2/_runtime.py +# actually demands; the CLI preflight passes and provisioning then fails. +EC2_REQUIRED_CONFIG_KEYS = ( + "ami_id", + "subnet_id", + "security_group_ids", + "region", + "ssh_user", +) + +# Import name -> distribution name, for the handful where they differ and the +# mismatch would otherwise be reported as a missing requirement. +IMPORT_TO_DISTRIBUTION = { + "attr": "attrs", + "bs4": "beautifulsoup4", + "cv2": "opencv-python", + "dateutil": "python-dateutil", + "dotenv": "python-dotenv", + "grpc": "grpcio", + "grpc_tools": "grpcio-tools", + "jwt": "pyjwt", + "PIL": "pillow", + "psycopg": "psycopg", + "psycopg2": "psycopg2-binary", + "pydantic_settings": "pydantic-settings", + "sklearn": "scikit-learn", + "typing_extensions": "typing-extensions", + "yaml": "pyyaml", +} + +SECRET_PATTERNS = [ + (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), + (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), +] +SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) + +MIN_COPIED_LITERAL = 80 + +ERROR = "ERROR" +WARN = "WARN" +INFO = "INFO" + + +# ------------------------------------------------------------------ # +# Capabilities # +# ------------------------------------------------------------------ # +# +# Each entry names what carries the capability. A rule gated on an absent +# capability is reported UNAVAILABLE so the gap is visible rather than assumed. + +CAPABILITY_SOURCE = { + "env_file": "PR #53 (jiajunh/can-232-...), open against main", + "editable_install": "no PR -- only on jiajunh/can-228-create-a-skill-...", + "sweeps_all_files": "no PR -- only on jiajunh/can-228-create-a-skill-...", + "stub_two_destinations": "PR #51 (feature/all-the-files), open against main", +} + + +def probe_capabilities(): + """Ask the importable ventis package what it actually supports.""" + caps = dict.fromkeys(CAPABILITY_SOURCE, False) + caps["ventis"] = False + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a broken install must not crash the check + return caps + + caps["ventis"] = True + caps["editable_install"] = hasattr(stub_generator, "_install_step") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") + + import importlib + + for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): + try: + module = importlib.import_module(module_name) + except Exception: # noqa: BLE001,S112 - the other path is the live one + continue + if hasattr(module, "resolve_env_file"): + caps["env_file"] = True + break + return caps + + +# ------------------------------------------------------------------ # +# YAML with line numbers # +# ------------------------------------------------------------------ # + + +class LineDict(dict): + """A mapping that remembers where it and each of its keys were written.""" + + line = 0 + key_lines: ClassVar[dict] = {} + + +class LineLoader(yaml.SafeLoader): + pass + + +def _construct_mapping(loader, node): + data = LineDict() + yield data + data.update(loader.construct_mapping(node, deep=False)) + data.line = node.start_mark.line + 1 + data.key_lines = { + key.value: key.start_mark.line + 1 + for key, _ in node.value + if isinstance(key, yaml.ScalarNode) + } + + +LineLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping +) + + +def line_of(mapping, key=None): + """The source line of `key` inside `mapping`, or of the mapping itself.""" + if not isinstance(mapping, LineDict): + return 0 + if key is not None: + return mapping.key_lines.get(key, mapping.line) + return mapping.line + + +def load_yaml(path): + """Parse `path`, returning (data, error). Never raises.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return yaml.load(handle, Loader=LineLoader), None + except Exception as exc: # noqa: BLE001 - any parse failure is a finding + return None, str(exc) + + +# ------------------------------------------------------------------ # +# Findings # +# ------------------------------------------------------------------ # + + +class Report: + def __init__(self, project_dir, capabilities): + self.project_dir = project_dir + self.capabilities = capabilities + self.findings = [] + self.reported_ec2_block = False + # A peer agent is imported by the name of its generated stub, which the + # build copies flat into every image. Those are not project modules and + # need no requirement. + self.stub_module_names = set() + + def add(self, check, level, path, line, summary, mechanism): + self.findings.append( + { + "check": check, + "level": level, + "path": self.rel(path) if path else "", + "line": line or 0, + "summary": summary, + "mechanism": mechanism, + } + ) + + def error(self, check, path, line, summary, mechanism): + self.add(check, ERROR, path, line, summary, mechanism) + + def warn(self, check, path, line, summary, mechanism): + self.add(check, WARN, path, line, summary, mechanism) + + def unavailable(self, check, summary): + self.add(check, INFO, "", 0, summary, "") + + def rel(self, path): + try: + return os.path.relpath(path, self.project_dir) + except ValueError: + return path + + def counts(self): + errors = sum(1 for f in self.findings if f["level"] == ERROR) + warnings = sum(1 for f in self.findings if f["level"] == WARN) + return errors, warnings + + +# ------------------------------------------------------------------ # +# Python source helpers # +# ------------------------------------------------------------------ # + + +def parse_python(path): + """AST for `path`, or (None, error). The port is never imported.""" + try: + with open(path, "r", encoding="utf-8") as handle: + source = handle.read() + except OSError as exc: + return None, str(exc) + try: + return ast.parse(source, filename=path), None + except SyntaxError as exc: + return None, f"{exc.msg} (line {exc.lineno})" + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def class_methods(class_node): + return { + node.name: node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def parameter_names(func_node): + """Every parameter a caller can pass by keyword, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + return positional + [a.arg for a in args.kwonlyargs] + + +def required_parameters(func_node): + """Parameters with no default, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + if args.defaults: + positional = positional[: len(positional) - len(args.defaults)] + kwonly = [ + arg.arg + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is None + ] + return positional + kwonly + + +def toplevel_import_names(tree): + """Top-level package name of every import in the module, with line numbers.""" + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.setdefault(alias.name.split(".")[0], node.lineno) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.setdefault(node.module.split(".")[0], node.lineno) + return names + + +# ------------------------------------------------------------------ # +# V001-V002 the files parse at all # +# ------------------------------------------------------------------ # + + +def check_config_loads(report, config_path): + """V001 -- cli.py _load_config, then cmd_build's config.get("agents", []).""" + if not os.path.isfile(config_path): + report.error( + "V001", + config_path, + 0, + "config/global_controller.yaml is missing", + "cli.py cmd_build logs 'Config file not found' and exits 1.", + ) + return None + config, error = load_yaml(config_path) + if error is not None: + report.error("V001", config_path, 0, f"unparseable YAML: {error}", "") + return None + if not isinstance(config, dict): + report.error( + "V001", + config_path, + 0, + "the config is empty or is not a mapping", + "cmd_build calls config.get('agents', []) on it -- AttributeError.", + ) + return None + if "agents" not in config: + report.error( + "V001", + config_path, + line_of(config), + "no `agents:` key", + "Nothing is built and nothing is deployed.", + ) + return config + agents = config.get("agents") + if agents is None or not isinstance(agents, list): + report.error( + "V001", + config_path, + line_of(config, "agents"), + "`agents:` is null or is not a list", + "cmd_build iterates it as a list -- TypeError before any image.", + ) + config["agents"] = [] + return config + + +def check_agent_yaml_loads(report, path): + """V002 -- stub_generator reads agent/name with [], not .get().""" + data, error = load_yaml(path) + if error is not None: + report.error("V002", path, 0, f"unparseable YAML: {error}", "") + return None + if not isinstance(data, dict): + report.error( + "V002", + path, + 0, + "the file is empty or is not a mapping", + "cmd_build does yaml.safe_load(f).get('agent', {}) -- AttributeError.", + ) + return None + agent = data.get("agent") + if not isinstance(agent, dict): + report.error( + "V002", + path, + line_of(data, "agent"), + "`agent:` is missing or null", + "stub_generator does config['agent'] -- KeyError, or AttributeError " + "in cmd_build's name index.", + ) + return None + name = agent.get("name") + if not isinstance(name, str) or not name: + report.error( + "V002", + path, + line_of(agent, "name") or line_of(agent), + "`agent.name` is missing or is not a string", + "It becomes the generated class name and ENV VENTIS_AGENT_NAME.", + ) + return None + if "functions" in agent and agent.get("functions") is None: + report.error( + "V002", + path, + line_of(agent, "functions"), + "`functions:` is present but null", + "stub_generator iterates it -- TypeError: 'NoneType' is not iterable. " + "Omit the key instead.", + ) + for func in agent.get("functions") or []: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + report.error( + "V002", + path, + line_of(agent, "functions"), + "a function entry has no string `name`", + "stub_generator does func_config['name'] -- KeyError.", + ) + continue + if "arguments" in func and func.get("arguments") is None: + report.error( + "V002", + path, + line_of(func, "arguments"), + f"`{func['name']}.arguments:` is present but null", + "stub_generator iterates it -- TypeError. Omit the key instead.", + ) + return data + + +# ------------------------------------------------------------------ # +# V003-V005, V012-V015, V022 the config entries # +# ------------------------------------------------------------------ # + + +def check_config_entries(report, config, config_path, project_dir, yaml_by_name): + """V003 V004 V005 V012 V013 V014 V015 V022.""" + agents = config.get("agents") or [] + seen_lower = {} + workflow_entries = [] + + for entry in agents: + if not isinstance(entry, dict): + report.error( + "V001", + config_path, + line_of(config, "agents"), + f"an `agents:` item is not a mapping: {entry!r}", + "cmd_build does agent_cfg['name'] on it.", + ) + continue + name = entry.get("name") + if not isinstance(name, str) or not name: + report.error( + "V003", + config_path, + line_of(entry), + "an `agents:` entry has no string `name`", + "cmd_build does agent_cfg['name'] -- KeyError.", + ) + continue + + # V004 -- the image tag is ventis-. + previous = seen_lower.get(name.lower()) + if previous is not None: + report.error( + "V004", + config_path, + line_of(entry, "name"), + f"`{name}` and `{previous}` differ only in case", + "Both build to the image tag ventis-" + f"{name.lower()}; the second overwrites the first.", + ) + seen_lower[name.lower()] = name + + entry_type = entry.get("type", "agent") + if entry_type == "workflow": + workflow_entries.append((name, entry)) + check_workflow_entry(report, entry, config_path, project_dir) + else: + check_agent_entry( + report, name, entry, config_path, project_dir, yaml_by_name + ) + + check_provider(report, name, entry, config_path) + check_replicas(report, name, entry, config_path) + check_requirements(report, name, entry, config_path) + check_ec2_entry(report, name, entry, config, config_path) + + # V015 -- without a workflow entry nothing serves HTTP. + if not workflow_entries: + report.error( + "V015", + config_path, + line_of(config, "agents"), + "no entry has `type: workflow`", + "Nothing builds a Flask container, so the port has no HTTP surface.", + ) + elif len(workflow_entries) > 1: + names = ", ".join(name for name, _ in workflow_entries) + report.error( + "V015", + config_path, + line_of(config, "agents"), + f"more than one `type: workflow` entry: {names}", + "Every workflow builds into docker_container/Workflow; the last wins.", + ) + return workflow_entries + + +def check_agent_entry(report, name, entry, config_path, project_dir, yaml_by_name): + """V003 V005.""" + entrypoint = entry.get("entrypoint") + if not entrypoint: + report.error( + "V005", + config_path, + line_of(entry), + f"agent `{name}` has no `entrypoint`", + "cmd_build warns 'Skipping agent', builds no image, and exits 0.", + ) + elif not os.path.isfile(os.path.join(project_dir, entrypoint)): + report.error( + "V005", + config_path, + line_of(entry, "entrypoint"), + f"agent `{name}`: entrypoint `{entrypoint}` does not exist", + "cmd_build logs 'Agent file not found', skips it, and exits 0.", + ) + if name not in yaml_by_name: + report.error( + "V003", + config_path, + line_of(entry, "name"), + f"no agents/*.yaml declares `agent.name: {name}`", + "cmd_build warns 'No YAML definition found', builds no image for it, " + "and exits 0 -- the agent is simply absent from the deployment.", + ) + + +def check_workflow_entry(report, entry, config_path, project_dir): + """V015.""" + workflow_file = entry.get("workflow_file") + if not workflow_file: + report.error( + "V015", + config_path, + line_of(entry), + "the workflow entry has no `workflow_file`", + "cmd_build warns 'Skipping workflow' and exits 0.", + ) + elif not os.path.isfile(os.path.join(project_dir, workflow_file)): + report.error( + "V015", + config_path, + line_of(entry, "workflow_file"), + f"`workflow_file: {workflow_file}` does not exist", + "cmd_build logs 'Workflow file not found', skips it, and exits 0.", + ) + + +def check_provider(report, name, entry, config_path): + """V012 -- provider == "local" is compared case-sensitively; EC2 is not.""" + provider = entry.get("provider", "local") + if not isinstance(provider, str): + report.error( + "V012", + config_path, + line_of(entry, "provider"), + f"`{name}`: provider must be a string, got {provider!r}", + "InstanceManager compares it to the literal 'local'.", + ) + return + if provider == "local" or provider.upper() == "EC2": + return + if provider.lower() == "local": + report.error( + "V012", + config_path, + line_of(entry, "provider"), + f"`{name}`: `provider: {provider}` must be lowercase `local`", + "InstanceManager.ensure_instances tests provider == 'local' to " + "reserve a host port. Any other casing leaves reserved_port None and " + "Local/_runtime.py dies on int(None) before a container starts.", + ) + else: + report.error( + "V012", + config_path, + line_of(entry, "provider"), + f"`{name}`: unknown `provider: {provider}`", + "Only 'local' (exact) and 'EC2' (any casing) are recognised.", + ) + + +def check_replicas(report, name, entry, config_path): + """V013 -- InstanceManager does int(replicas).""" + if "replicas" not in entry: + return + replicas = entry.get("replicas") + if isinstance(replicas, bool) or not isinstance(replicas, int): + report.error( + "V013", + config_path, + line_of(entry, "replicas"), + f"`{name}`: `replicas` must be an int, got {replicas!r}", + "InstanceManager.ensure_instances does range(int(replicas)); the " + "list form GlobalController._get_replica_placements accepts raises " + "TypeError here.", + ) + elif replicas < 1: + report.error( + "V013", + config_path, + line_of(entry, "replicas"), + f"`{name}`: `replicas: {replicas}` launches nothing", + "range(0) -- the agent is deployed with no instances.", + ) + + +def check_requirements(report, name, entry, config_path): + """V014 -- one bad item drops the whole list, with only a warning.""" + if "requirements" not in entry: + return + requirements = entry.get("requirements") + if requirements is None: + return + if not isinstance(requirements, list) or not all( + isinstance(item, str) for item in requirements + ): + report.error( + "V014", + config_path, + line_of(entry, "requirements"), + f"`{name}`: `requirements` must be a list of strings", + "_normalize_requirements logs one warning and returns [] -- the " + "whole list is dropped, not the bad item, and the build still " + "succeeds with none of them installed.", + ) + + +def check_ec2_entry(report, name, entry, config, config_path): + """V022 -- the CLI preflight list is shorter than what provisioning needs.""" + provider = entry.get("provider", "local") + if not isinstance(provider, str) or provider.upper() != "EC2": + return + if not entry.get("instance_type"): + report.error( + "V022", + config_path, + line_of(entry), + f"`{name}`: EC2 entry has no `instance_type`", + "EC2/_runtime.py does spec['instance_type'] -- KeyError at provision.", + ) + if report.reported_ec2_block: + return + report.reported_ec2_block = True + ec2 = config.get("ec2") or {} + missing = [key for key in EC2_REQUIRED_CONFIG_KEYS if not ec2.get(key)] + if missing: + report.error( + "V022", + config_path, + line_of(config, "ec2") or line_of(config), + f"top-level `ec2:` is missing {', '.join(missing)}", + "cli.py's preflight checks only four of these; ssh_user is demanded " + "later by EC2/_runtime.py, after preflight has already passed.", + ) + + +# ------------------------------------------------------------------ # +# V006-V010, W005 the adapter against its yaml # +# ------------------------------------------------------------------ # + +BUILTIN_TYPE_NAMES = frozenset( + name + for name in dir(builtins) + if isinstance(getattr(builtins, name), type) and not name[0].isupper() +) + + +def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): + """V006 V007 V008 V009 V010 W005.""" + name = agent_block["name"] + functions = agent_block.get("functions") or [] + check_argument_types(report, agent_yaml_path, functions) + + entrypoint = entry.get("entrypoint") + if not entrypoint: + return + entrypoint_path = os.path.join(project_dir, entrypoint) + if not os.path.isfile(entrypoint_path): + return + + tree, error = parse_python(entrypoint_path) + if tree is None: + report.error( + "V006", + entrypoint_path, + 0, + f"the entrypoint does not parse: {error}", + "_load_agent exec_module's it and swallows the exception; the first " + "request answers 'No agent loaded'.", + ) + return + + class_node = find_class(tree, name) + if class_node is None: + classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] + found = ", ".join(classes) if classes else "no classes at all" + report.error( + "V006", + entrypoint_path, + 1, + f"no class named `{name}` at module level (found: {found})", + "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " + "the AttributeError. The class name must equal agent.name exactly.", + ) + return + + methods = class_methods(class_node) + check_constructor(report, entrypoint_path, name, methods) + + for func in functions: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + continue + check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) + + +def check_argument_types(report, agent_yaml_path, functions): + """V010 -- the type string is pasted into an ast.Name, never checked.""" + for func in functions: + if not isinstance(func, dict): + continue + for arg in func.get("arguments") or []: + if not isinstance(arg, dict) or "type" not in arg: + continue + declared = arg.get("type") + if not isinstance(declared, str): + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared!r}` is not a string", + "ast.Name(id=) then ast.unparse -- a non-string raises " + "while the stub is generated.", + ) + continue + if declared in BUILTIN_TYPE_NAMES: + continue + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared}` is not a builtin", + "stub_generator pastes it verbatim into the generated " + "annotation, and the stub module imports only Future and " + "inspect. Anything else raises NameError when the stub is " + "imported -- after a green build. Use str int float bool dict " + "list.", + ) + + +def check_constructor(report, entrypoint_path, name, methods): + """V007 -- _load_agent calls agent_class() with no arguments.""" + init = methods.get("__init__") + if init is None: + return + required = required_parameters(init) + if required: + report.error( + "V007", + entrypoint_path, + init.lineno, + f"`{name}.__init__` requires {', '.join(required)}", + "_load_agent calls agent_class() with no arguments; the TypeError is " + "swallowed and the first request answers 'No agent loaded'. Read " + "configuration from the environment inside __init__ instead.", + ) + + +def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): + """V008 V009 W005.""" + func_name = func["name"] + method = methods.get(func_name) + if method is None: + report.error( + "V008", + entrypoint_path, + 0, + f"`{class_name}` has no method `{func_name}`", + "The yaml declares it, so callers get a stub for it; the controller " + f"then answers \"Agent {class_name} has no method '{func_name}'\".", + ) + return + + # V009 -- nothing on the execution path awaits. + if isinstance(method, ast.AsyncFunctionDef): + report.error( + "V009", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` is `async def`", + "The executor calls method(**args) with no await, so Redis receives " + "''. Keep the signature synchronous and call " + "asyncio.run(...) inside the body.", + ) + + # V008 -- the controller calls method(**args) with the yaml's names. + declared = [ + arg["name"] + for arg in func.get("arguments") or [] + if isinstance(arg, dict) and isinstance(arg.get("name"), str) + ] + actual = parameter_names(method) + required = required_parameters(method) + + missing = [arg for arg in declared if arg not in actual] + if missing: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` has no parameter " + f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", + "LocalController does method(**args) with the yaml's argument names. " + "A mismatch is TypeError: unexpected keyword argument, at request " + f"time. See {os.path.basename(agent_yaml_path)}.", + ) + + unfilled = [arg for arg in required if arg not in declared] + if unfilled: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " + "the yaml does not declare", + "Only declared arguments are ever sent, and the generated stub gives " + "none of them a default. Declare them in the yaml or default them " + "in the signature.", + ) + + # W005 -- `returns` is read by nothing, but it is how the workflow author + # learns the call site needs json.loads. + returns = func.get("returns") + declared_return = returns.get("type") if isinstance(returns, dict) else None + annotation = method.returns + annotated = annotation.id if isinstance(annotation, ast.Name) else None + if annotated in ("dict", "list") and declared_return != annotated: + report.warn( + "W005", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` returns {annotated} but the yaml " + f"declares `returns.type: {declared_return}`", + "`returns` is read by nothing; its only job is telling whoever " + "writes the workflow that .value() hands back a string to json.loads.", + ) + + +# ------------------------------------------------------------------ # +# V016-V018 the workflow # +# ------------------------------------------------------------------ # + + +def check_workflow(report, workflow_path): + """V016 V017 V018.""" + tree, error = parse_python(workflow_path) + if tree is None: + report.error("V016", workflow_path, 0, f"does not parse: {error}", "") + return + + main = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "main" + ): + main = node + break + + if main is None: + defined = [ + n.name + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + found = ", ".join(defined) if defined else "no top-level functions" + report.error( + "V016", + workflow_path, + 1, + f"no top-level function named `main` (found: {found})", + "Ventis serves POST /, but the deployment platform's " + "test endpoint posts to a hardcoded /main. A differently named " + "workflow builds, deploys and stays unreachable -- 404, container " + "healthy.", + ) + else: + check_main_signature(report, workflow_path, main) + + # V016 -- deploy() is what starts Flask. + if not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "deploy" + for node in ast.walk(tree) + ): + report.error( + "V016", + workflow_path, + 1, + "the workflow never calls `deploy(...)`", + "workflow_launcher.py exec's this file and nothing else starts the " + "HTTP server; the container comes up serving nothing.", + ) + + check_main_guard(report, workflow_path, tree) + check_fused_fanout(report, workflow_path, tree) + + +def check_main_signature(report, workflow_path, main): + """V016 -- the platform sends exactly {"query": ...}.""" + if isinstance(main, ast.AsyncFunctionDef): + report.error( + "V016", + workflow_path, + main.lineno, + "`main` is `async def`", + "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " + "no await; the response body would be a coroutine repr.", + ) + params = parameter_names(main) + if not params: + report.error( + "V016", + workflow_path, + main.lineno, + "`main` takes no arguments", + 'The platform posts {"query": "..."} and deploy() splats the ' + "body in as kwargs -- TypeError on every request.", + ) + return + if params[0] != "query": + report.error( + "V016", + workflow_path, + main.lineno, + f"`main`'s first parameter is `{params[0]}`, not `query`", + "The platform's body schema is strictly validated as " + "{query: string}; any other key is rejected with 400 in the control " + "plane, before the request reaches the host.", + ) + extra = [p for p in required_parameters(main) if p != "query"] + if extra: + report.error( + "V016", + workflow_path, + main.lineno, + f"`main` requires {', '.join(extra)} beyond `query`", + "Only `query` is ever sent, so every other parameter needs a " + "default or the call raises on every request. Pack richer input " + "into `query`.", + ) + + +def check_main_guard(report, workflow_path, tree): + """V017 -- the workflow is exec'd, so __name__ == "__main__".""" + for node in tree.body: + if not isinstance(node, ast.If): + continue + test = node.test + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and any( + isinstance(c, ast.Constant) and c.value == "__main__" + for c in test.comparators + ) + ): + report.error( + "V017", + workflow_path, + node.lineno, + '`if __name__ == "__main__":` block in the workflow', + "workflow_launcher.py runs exec(open().read()), so " + "__name__ IS '__main__' here and this block executes in " + "production, at container start.", + ) + + +def check_fused_fanout(report, workflow_path, tree): + """V018 -- .value() blocks, so dispatching and resolving in one + comprehension runs the fan-out one call at a time.""" + comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) + for node in ast.walk(tree): + if not isinstance(node, comprehensions + (ast.DictComp,)): + continue + elements = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for element in elements: + for inner in ast.walk(element): + if ( + isinstance(inner, ast.Call) + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "value" + and isinstance(inner.func.value, ast.Call) + ): + report.error( + "V018", + workflow_path, + node.lineno, + "one comprehension both dispatches a call and resolves " + "it with .value()", + ".value() blocks, so each call completes before the " + "next is dispatched. It does not error -- the fan-out " + "is just silently serial, and with it the reason to be " + "on Ventis. Dispatch every call first, then resolve: " + "futures = [a.work(i) for i in items] then " + "[f.value() for f in futures].", + ) + return + + +# ------------------------------------------------------------------ # +# V019-V020 what the copy order overwrites # +# ------------------------------------------------------------------ # + + +def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): + """V019 V020 -- later copies land on earlier ones at the context root.""" + for entry in sorted(os.listdir(project_dir)): + path = os.path.join(project_dir, entry) + if not os.path.isfile(path) or not entry.endswith(".py"): + continue + + # V019 -- the shared runtime is copied flat, after the project sweep. + if entry in RUNTIME_FLAT_NAMES: + report.error( + "V019", + path, + 1, + f"a project module named `{entry}` sits at the project root", + "The shared Ventis runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by Ventis's own " + f"{entry}. Rename it or move it into a package directory.", + ) + + # V020 -- a stub is copied flat under its yaml's basename. + entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} + for yaml_path in yaml_paths: + stem = os.path.splitext(os.path.basename(yaml_path))[0] + module = f"{stem}.py" + if module in entrypoint_basenames: + continue # the entrypoint is copied last and wins its flat name back + candidate = os.path.join(project_dir, module) + if os.path.isfile(candidate): + report.error( + "V020", + candidate, + 1, + f"`{os.path.basename(yaml_path)}` generates a stub that lands on " + f"`{module}`", + "The yaml's basename names the stub, and the stub is copied flat " + "over the swept tree. Anything importing this module inside the " + "container gets the generated stub instead of the real code. " + "Rename the yaml to match its own entrypoint.", + ) + + +# ------------------------------------------------------------------ # +# V021 policy.yaml # +# ------------------------------------------------------------------ # + + +def check_policy(report, config, config_path): + """V021 -- absent is fine; present-but-empty kills deploy.""" + policy_path = os.path.join( + os.path.dirname(os.path.abspath(config_path)), "policy.yaml" + ) + if not os.path.isfile(policy_path): + # _load_policy_rules logs and returns [], and _check_policy allows + # everything when the rule list is empty. Nothing to check. + return + + policy, error = load_yaml(policy_path) + if error is not None: + report.error("V021", policy_path, 0, f"unparseable YAML: {error}", "") + return + if not isinstance(policy, dict): + report.error( + "V021", + policy_path, + 0, + "policy.yaml exists but is empty", + "_load_policy_rules does policy_config.get('rules', []) on None -- " + "AttributeError inside GlobalController.__init__, so `ventis deploy` " + "dies before any container starts. Delete the file instead; absent " + "means everything is allowed.", + ) + return + + rules = policy.get("rules") + if rules is None or not isinstance(rules, list): + report.error( + "V021", + policy_path, + line_of(policy, "rules") or line_of(policy), + "`rules:` is null or is not a list", + "_load_policy_rules calls .sort() on it -- AttributeError inside " + "GlobalController.__init__, before any container starts.", + ) + return + + declared = [ + entry.get("name") + for entry in config.get("agents") or [] + if isinstance(entry, dict) and isinstance(entry.get("name"), str) + ] + fallback = None + for rule in rules: + if not isinstance(rule, dict): + report.error( + "V021", + policy_path, + line_of(policy, "rules"), + f"a rule is not a mapping: {rule!r}", + "_check_policy does rule.get('match', {}) on it.", + ) + continue + match = rule.get("match") + if match is None or (isinstance(match, dict) and not match): + fallback = rule + + if fallback is None: + report.error( + "V021", + policy_path, + line_of(policy, "rules"), + "no rule with an empty `match: {}`", + "_check_policy denies access when no rule matches the request " + "context, so every service answers Unauthorized after its request " + "was already accepted with a 202.", + ) + return + + if not isinstance(fallback.get("access"), (list, str)): + report.error( + "V021", + policy_path, + line_of(fallback, "access") or line_of(fallback), + "the `match: {}` rule's `access` is neither a list nor `all`", + "_check_policy does `service in access`.", + ) + return + + # Reachable under *some* context. A service deliberately restricted to one + # caller -- text2sql keeps ProductionExecutorAgent out of the fallback and + # reaches it only through an `access: all` rule -- is correct policy, not a + # defect, so only a service no rule can ever reach is worth reporting. + reachable = set() + for rule in rules: + if not isinstance(rule, dict): + continue + access = rule.get("access") + if access == "all": + reachable.update(declared) + elif isinstance(access, list): + reachable.update(item for item in access if isinstance(item, str)) + + unreachable = [name for name in declared if name not in reachable] + if unreachable: + report.warn( + "V021", + policy_path, + line_of(policy, "rules"), + f"no rule grants access to {', '.join(unreachable)}", + "The first matching rule decides, and a service named in none of " + "them answers Unauthorized on /status after the request was " + "already accepted with a 202. Intentional if the service is meant " + "to be unreachable.", + ) + + +# ------------------------------------------------------------------ # +# V030-V031 capability-gated rules # +# ------------------------------------------------------------------ # + + +def check_env_file(report, config, config_path, project_dir): + """V030 -- gated on the env_file support that PR #53 carries.""" + declared = config.get("env_file") + supported = report.capabilities.get("env_file") + + if not supported: + if declared: + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is set, but this Ventis never reads it", + "No resolve_env_file in the importable ventis package, so the " + "key is silently dropped and the container answers a provider " + "credential error on the first request. It arrives with " + f"{CAPABILITY_SOURCE['env_file']}.", + ) + else: + report.unavailable( + "V030", + "env_file is not supported by the importable Ventis " + f"({CAPABILITY_SOURCE['env_file']}). Credentials have no " + "declared path into a container on this tree.", + ) + return + + if not declared: + report.warn( + "V030", + config_path, + line_of(config), + "no `env_file:` in the config", + "Only five VENTIS_* variables reach a container without it. If the " + "source reads any credential from the environment, the first " + "request fails on a provider error.", + ) + return + + resolved = os.path.expanduser(str(declared)) + if not os.path.isabs(resolved): + resolved = os.path.join(project_dir, resolved) + if not os.path.isfile(resolved): + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` does not resolve to a file", + "resolve_env_file raises before GlobalController exists, so " + "`ventis deploy` fails with one error line. The path is resolved " + "against the project root you run from.", + ) + elif not os.access(resolved, os.R_OK): + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is not readable", + "resolve_env_file raises on an unreadable file.", + ) + + +def check_import_root(report, project_dir, entrypoint_paths): + """V031 -- gated on the editable install, which no PR carries today.""" + supported = report.capabilities.get("editable_install") + has_metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + + non_flat = [] + for path in entrypoint_paths: + tree, _ = parse_python(path) + if tree is None: + continue + for name, lineno in toplevel_import_names(tree).items(): + if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: + continue + if _resolves_flat(project_dir, name): + continue + location = _resolves_nested(project_dir, name) + if location: + non_flat.append((path, lineno, name, location)) + + if not supported: + report.unavailable( + "V031", + "the editable install (`-e .`) is not supported by the importable " + f"Ventis ({CAPABILITY_SOURCE['editable_install']}). Only modules " + "that land flat at /app import inside a container.", + ) + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, which is not at the " + "project root", + "sys.path[0] is /app and this Ventis runs no editable install, " + "so only modules swept to the root import. The adapter raises " + "ModuleNotFoundError inside _load_agent and the first request " + "answers 'No agent loaded'.", + ) + return + + if non_flat and not has_metadata: + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, and the project root " + "has no packaging metadata", + "A pyproject.toml, setup.py or setup.cfg at the root is what " + "adds `-e .`, and the project's own metadata is what decides the " + "import root. Without it the install is skipped silently and " + "only flat modules import.", + ) + + +def _resolves_flat(project_dir, name): + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isfile( + os.path.join(project_dir, name, "__init__.py") + ) + + +def _resolves_nested(project_dir, name): + """Where inside the tree `name` lives, if it is a project module at all.""" + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] + if root == project_dir: + continue + if f"{name}.py" in files: + return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) + if name in dirs and os.path.isfile(os.path.join(root, name, "__init__.py")): + return os.path.relpath(os.path.join(root, name), project_dir) + return None + + +# ------------------------------------------------------------------ # +# W001-W006 the rewrite smells # +# ------------------------------------------------------------------ # +# +# Warnings, not errors: each is a heuristic, and a false positive must never +# block a correct port. --strict promotes them for CI. + + +def _string_literals(tree, minimum): + for node in ast.walk(tree): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and len(node.value.strip()) >= minimum + ): + yield node.value, node.lineno + + +def check_copied_literals(report, project_dir, port_paths, source_paths): + """W001 -- a prompt that exists in the source and again in the adapter.""" + source_text = {} + for path in source_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + source_text[path] = handle.read() + except OSError: + continue + if not source_text: + return + + for port_path in port_paths: + tree, _ = parse_python(port_path) + if tree is None: + continue + for literal, lineno in _string_literals(tree, MIN_COPIED_LITERAL): + for source_path, text in source_text.items(): + if literal in text: + report.warn( + "W001", + port_path, + lineno, + f"a {len(literal)}-character string literal also appears " + f"in {report.rel(source_path)}", + "It exists in the source. Import it -- the whole tree is " + "in the image, and a copy drifts the moment the source " + "changes. A port that restates a prompt has rewritten " + "the project, not ported it.", + ) + break + + +def check_secrets(report, port_paths): + """W003 -- env_file is the way in; nothing else is.""" + for path in port_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError: + continue + for number, line in enumerate(lines, start=1): + for pattern, description in SECRET_PATTERNS: + if pattern.search(line): + report.warn( + "W003", + path, + number, + f"this line looks like {description}", + "Never put a secret in the source tree or the build " + "context. The build sweeps the project into every " + "image.", + ) + break + + tree, _ = parse_python(path) + if tree is None: + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not ( + isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() + ): + continue + for target in node.targets: + if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): + report.warn( + "W003", + path, + node.lineno, + f"`{target.id}` is assigned a literal string", + "Read it from the environment instead; the build sweeps " + "this file into every image.", + ) + + +def check_source_tree_clean(report, project_dir): + """W002 -- the port must leave `git status` on the source clean.""" + + def git(*args): + try: + result = subprocess.run( + ["git", *args], + cwd=project_dir, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout if result.returncode == 0 else None + + # `git status` prints paths relative to the repo root, so a port nested + # inside a larger repo needs that prefix stripped before the port's own + # directories can be recognised. + prefix = git("rev-parse", "--show-prefix") + if prefix is None: + return + prefix = prefix.strip() + status = git("status", "--porcelain", "--", ".") + if status is None: + return + + port_prefixes = ("agents/", "workflow/", "config/") + generated = ("docker_container/", "stubs/", "grpc_stubs/") + for line in status.splitlines(): + path = line[3:].strip().strip('"') + if prefix and path.startswith(prefix): + path = path[len(prefix) :] + if not path or path.startswith(port_prefixes) or path.startswith(generated): + continue + report.warn( + "W002", + os.path.join(project_dir, path), + 0, + f"`{path}` is modified or untracked outside the port's own files", + "A port adds agents/, workflow/ and config/ beside an untouched " + "source tree. If this is an edit to the source, it is a rewrite; if " + "it is unrelated local work, ignore this line.", + ) + + +def check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path +): + """W006 -- an import the container cannot satisfy.""" + tree, _ = parse_python(entrypoint_path) + if tree is None: + return + + declared = { + _normalize_distribution(item) + for item in (entry.get("requirements") or []) + if isinstance(item, str) + } + base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} + stdlib = getattr(sys, "stdlib_module_names", frozenset()) + + for name, lineno in sorted(toplevel_import_names(tree).items()): + if name in stdlib or name == "ventis": + continue + # Provided by the image itself: the shared runtime is copied flat, and + # every agents/*.yaml generates a stub that is copied flat too. + if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: + continue + if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): + continue + distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) + if distribution in base or distribution in declared: + continue + report.warn( + "W006", + entrypoint_path, + lineno, + f"`import {name}` is in neither the runtime's base list nor this " + "entry's `requirements:`", + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside _load_agent " + "and 'No agent loaded' on the first request. If the distribution is " + f"named something other than `{name}`, declare that name in " + f"{report.rel(config_path)}.", + ) + + +def _normalize_distribution(name): + return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( + "_", "-" + ) + + +# ------------------------------------------------------------------ # +# Driver # +# ------------------------------------------------------------------ # + + +def validate(project_dir, config_path, capabilities): + report = Report(project_dir, capabilities) + + config = check_config_loads(report, config_path) + if config is None: + return report + + import glob + + yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) + if not yaml_paths: + report.error( + "V002", + os.path.join(project_dir, "agents"), + 0, + "no agents/*.yaml files", + "cmd_build warns 'No agent YAML files found'; no stubs are generated " + "and no agent image is built.", + ) + + report.stub_module_names = { + os.path.splitext(os.path.basename(path))[0] for path in yaml_paths + } + + agents_by_name = {} + for path in yaml_paths: + data = check_agent_yaml_loads(report, path) + if data is not None: + agents_by_name[data["agent"]["name"]] = (path, data["agent"]) + + entries = config.get("agents") or [] + check_config_entries(report, config, config_path, project_dir, set(agents_by_name)) + + entrypoints = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": + continue + name = entry.get("name") + entrypoint = entry.get("entrypoint") + if entrypoint: + entrypoints.append(entrypoint) + if name in agents_by_name: + yaml_path, agent_block = agents_by_name[name] + check_adapter(report, yaml_path, agent_block, entry, project_dir) + entrypoint_path = os.path.join(project_dir, entrypoint or "") + if entrypoint and os.path.isfile(entrypoint_path): + check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path + ) + + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": + continue + workflow_file = entry.get("workflow_file") + if not workflow_file: + continue + workflow_path = os.path.join(project_dir, workflow_file) + if os.path.isfile(workflow_path): + check_workflow(report, workflow_path) + + check_flat_collisions(report, project_dir, yaml_paths, entrypoints) + check_policy(report, config, config_path) + check_env_file(report, config, config_path, project_dir) + + entrypoint_paths = [ + os.path.join(project_dir, e) + for e in entrypoints + if os.path.isfile(os.path.join(project_dir, e)) + ] + check_import_root(report, project_dir, entrypoint_paths) + + port_paths = list(entrypoint_paths) + for entry in entries: + if isinstance(entry, dict) and entry.get("workflow_file"): + candidate = os.path.join(project_dir, entry["workflow_file"]) + if os.path.isfile(candidate): + port_paths.append(candidate) + + source_paths = _source_paths(project_dir, port_paths) + check_copied_literals(report, project_dir, port_paths, source_paths) + check_secrets(report, port_paths) + check_source_tree_clean(report, project_dir) + return report + + +def _source_paths(project_dir, port_paths): + """Every project .py that is not one of the port's own four files.""" + excluded = {os.path.abspath(p) for p in port_paths} + found = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and d != "__pycache__" + and not ( + root == project_dir and d in ("docker_container", "stubs", "grpc_stubs") + ) + ] + for name in files: + if not name.endswith(".py"): + continue + path = os.path.join(root, name) + if os.path.abspath(path) not in excluded: + found.append(path) + return found + + +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + +LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} + + +def _wrap(text, width, indent): + words = text.split() + lines = [] + current = "" + for word in words: + candidate = f"{current} {word}".strip() + if len(candidate) + len(indent) > width and current: + lines.append(indent + current) + current = word + else: + current = candidate + if current: + lines.append(indent + current) + return lines + + +def print_report(report, project_dir): + caps = report.capabilities + if not caps.get("ventis"): + print("ventis is not importable here -- capability-gated rules are") + print("reported UNAVAILABLE rather than checked.\n") + else: + print("Ventis capabilities detected:") + for key, source in CAPABILITY_SOURCE.items(): + mark = "yes" if caps.get(key) else "no " + print(f" {mark} {key:<22} {source}") + print() + + findings = sorted( + report.findings, + key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), + ) + for finding in findings: + where = finding["path"] + if where and finding["line"]: + where = f"{where}:{finding['line']}" + header = f"{finding['check']} {finding['level']:<5}" + print(f"{header} {where}" if where else header) + for line in _wrap(finding["summary"], 78, " "): + print(line) + if finding["mechanism"]: + for line in _wrap(finding["mechanism"], 78, " "): + print(line) + print() + + errors, warnings = report.counts() + if not findings: + print(f"{project_dir}: clean.") + return + print(f"{errors} error(s), {warnings} warning(s).") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Check a Ventis port against the rules in SKILL.md." + ) + parser.add_argument( + "project_dir", nargs="?", default=".", help="the port's project root" + ) + parser.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument("--json", action="store_true", help="emit findings as JSON") + parser.add_argument( + "--strict", action="store_true", help="fail on warnings as well as errors" + ) + args = parser.parse_args(argv) + + project_dir = os.path.abspath(args.project_dir) + config_path = ( + args.config + if os.path.isabs(args.config) + else os.path.join(project_dir, args.config) + ) + + capabilities = probe_capabilities() + report = validate(project_dir, config_path, capabilities) + errors, warnings = report.counts() + + if args.json: + print( + json.dumps( + { + "project_dir": project_dir, + "capabilities": capabilities, + "errors": errors, + "warnings": warnings, + "findings": report.findings, + }, + indent=2, + ) + ) + else: + print_report(report, report.rel(project_dir) or project_dir) + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/porting-to-ventis/ventis-contract.md b/.claude/skills/porting-to-ventis/ventis-contract.md index 56e7132..3d83165 100644 --- a/.claude/skills/porting-to-ventis/ventis-contract.md +++ b/.claude/skills/porting-to-ventis/ventis-contract.md @@ -3,6 +3,18 @@ Mechanism behind every rule in `SKILL.md`. Validate against [CanyonCodeCoreAI/canyoncodecore](https://github.com/CanyonCodeCoreAI/canyoncodecore). +**Which Ventis this describes.** Two sections below hold for a branch rather than +for `main`, and each says so where it starts. `validate.py` probes the importable +`ventis` package for them instead of assuming: + +| Behaviour | Carried by | +| ---------------------------------------------- | ------------------------------------------------------------------- | +| a stub lands at **two** paths | **PR #51** (`feature/all-the-files`), open against main | +| `env_file:` carries credentials to a container | **PR #53** (`jiajunh/can-232-...`), open against main | +| `-e .`, and a sweep that takes non-`.py` files | **no PR** — only on `jiajunh/can-228-create-a-skill-...` | + +Everything not marked holds on `main` today. + ## Project layout | Path | Where it comes from | @@ -13,7 +25,7 @@ Mechanism behind every rule in `SKILL.md`. Validate against | `config/policy.yaml` | `global_controller.py` `_load_policy_rules` — optional | | the workflow file | the `workflow_file` key on the `type: workflow` config entry | | the project root | `cli.py` passes `project_dir=os.getcwd()`; build and deploy run from it | -| `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .` | +| `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .`. **No PR carries this** | ## Agent yaml @@ -127,15 +139,20 @@ runtime, then every stub, then the entrypoint. Later writes land on earlier ones | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | | the project tree | at its own relative paths (`agents/x.py`, `src/pkg/mod.py`) | | the shared runtime | flat at the context root, winning over the swept tree — `local_controller.py` is the CMD, so a project file of that name breaks the container | -| every stub | **twice**: flat at the root (the copy imports resolve), and at `agents/.py`, landing on the real implementation so a peer's name gives the caller its stub | +| every stub (PR #51) | **twice**: flat at the root (the copy imports resolve), and at `agents/.py`, landing on the real implementation so a peer's name gives the caller its stub | | the entrypoint | flat, last, winning the flat name back — `VENTIS_AGENT_FILE` is a **basename**, loaded from `/app` | | `requirements.txt` | written before anything is copied, so the sweep skips a project's own `requirements.txt` and root `Dockerfile` | -The sweep takes every file, not only `.py` — the editable install reads packaging -metadata, and that metadata points at a README or a license. It skips hidden -files and directories (`.env` holds credentials and the context is what ships), -`__pycache__`, and the three directories `ventis build` generates: -`docker_container`, `stubs`, `grpc_stubs`. +`_sweep_py_files` takes **`.py` files only**, skipping symlinks, hidden files +and directories (`.env` holds credentials and the context is what ships), +`__pycache__`, and the three directories `ventis build` generates at the project +root: `docker_container`, `stubs`, `grpc_stubs`. + +That matters for a source whose packaging metadata points at a README or a +license: those files do not reach the image, so an editable install of it would +fail on the missing file. `_sweep_project_files`, which takes every file, exists +only on `jiajunh/can-228-create-a-skill-...` and has no PR — the same branch that +carries `_install_step`, which is the only reason the wider sweep is needed. **An agent is no longer one file**, and a yaml sharing the entrypoint's basename no longer eats its own stub. What an agent loses is the ability to import *its @@ -144,6 +161,13 @@ own* stub by name — the entrypoint shadows it flat. It can still reach it at ### The import root +> **No PR carries this.** `_install_step` lives only on +> `jiajunh/can-228-create-a-skill-...`. On `main`, and on both open PRs, the +> agent Dockerfile is `COPY requirements.txt` → `uv pip install -r +> requirements.txt` → `COPY . .`, with no `-e .` and no packaging detection — +> so **only modules that land flat at `/app` import at all**, whatever metadata +> the project declares. `validate.py` V031 enforces whichever rule is in force. + `_install_step` writes `RUN uv pip install --system -r requirements.txt -e .` when the project root has a `pyproject.toml`, `setup.py` or `setup.cfg`. That editable install is what @@ -157,11 +181,13 @@ flat resolve. `examples/helloworld`, `finance` and `text2sql` are all in this state; they work because their entrypoints import nothing from the project tree, only stubs, which land flat. -**One resolve, not two.** Requirements and `-e .` go to a single `uv pip install` -so the runtime's list and the source's own dependencies resolve against each -other; a genuine conflict fails the build instead of the first request. It also -forces `COPY . .` ahead of the install, so the requirements layer no longer -caches on its own. +**One resolve, not two.** Where `_install_step` exists, requirements and `-e .` +go to a single `uv pip install` so the runtime's list and the source's own +dependencies resolve against each other; a genuine conflict fails the build +instead of the first request. It also forces `COPY . .` ahead of the install, so +the requirements layer no longer caches on its own. Without that branch the two +are separate layers, requirements first, and the source's own dependency list is +never installed at all. ## Dependencies @@ -210,6 +236,13 @@ which is what the container's CMD actually runs. ## Credentials: `env_file` +> **PR #53** (`jiajunh/can-232-...`) carries this, open against main. Without it +> nothing reads the key: `grep -rn env_file ventis/` finds no hits on `main`, so +> an `env_file:` line in the config is inert, the credential never reaches the +> container, and the failure surfaces as a provider error on the first request +> rather than as a config error at deploy. `validate.py` V030 probes for +> `resolve_env_file` and reports which of the two situations you are in. + `_launch_locally` passes exactly five `-e` flags, all `VENTIS_*` (`AGENT_PORT`, `AGENT_HOST`, `REDIS_HOST`, `REDIS_PORT`, `POLL_INTERVAL`), plus `VENTIS_DATABASE_URL` and `VENTIS_PROJECT_ID` on a workflow entry when @@ -245,6 +278,16 @@ returns `[]`, which `_load_and_write_policies` publishes to every host Redis. `LocalController._check_policy` returns `True` when the rule list is empty, so **no policy file means everything is allowed.** +**Absent is safe; present-and-empty is not.** Past the `os.path.isfile` guard the +read is unguarded: `policy_config.get("rules", [])` on an empty file's `None` is +an `AttributeError`, and a null `rules:` reaches `rules.sort()` as `None`. Either +one raises inside `GlobalController.__init__`, so `ventis deploy` dies before a +single container starts. Deleting the file is the safe state; a half-written one +is not. + +The path is derived from the **config file's own directory**, not from the +project root — `-c foo/gc.yaml` looks for `foo/policy.yaml`. + When rules exist they are sorted most-specific-first (by number of `match` keys) and the first rule whose `match` keys all equal the request context decides: `access: all`, or membership in the `access` list. A service left out of the From b7e2f2f07cba123eae8d3d42b7b13fa518c67463 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 15:35:34 -0700 Subject: [PATCH 12/43] examples/joke_writer: correct the policy comment, gate env_file on its PR policy.yaml claimed the file was mandatory and that a missing one crashes deploy. It is the reverse: `_load_policy_rules` returns [] when the file is absent and `_check_policy` then allows everything. The real hazard is a half-written file -- an empty one, or a null `rules:`, raises inside GlobalController.__init__ before a single container starts. `env_file: .env` needs PR #53, still open. On main nothing reads the key, so the config line is inert and every request answers a Bedrock credential error. Said so where a reader meets it: the config, and the README's setup steps. --- examples/joke_writer/README.md | 6 ++++++ examples/joke_writer/config/global_controller.yaml | 6 ++++++ examples/joke_writer/config/policy.yaml | 9 ++++++--- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md index 930a410..7161d28 100644 --- a/examples/joke_writer/README.md +++ b/examples/joke_writer/README.md @@ -107,6 +107,12 @@ cp .env.example .env $EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... ``` +> **`env_file:` needs PR #53** (`jiajunh/can-232-...`), still open against main. +> Until it merges nothing in `ventis/` reads the key, so the steps below leave +> the container without a credential and every request answers a Bedrock +> credential error. `python ../../.claude/skills/porting-to-ventis/validate.py .` +> reports this as V030 and stops reporting it the day the PR lands. + `config/global_controller.yaml` points `env_file:` at that file, and every container gets it as `docker run --env-file`. Nothing in this project reads the variable: botocore matches the name against `bedrock-runtime`'s signingName and diff --git a/examples/joke_writer/config/global_controller.yaml b/examples/joke_writer/config/global_controller.yaml index 0aa650b..eb26090 100644 --- a/examples/joke_writer/config/global_controller.yaml +++ b/examples/joke_writer/config/global_controller.yaml @@ -73,4 +73,10 @@ redis: # Relative to this project root, same as `entrypoint` and `workflow_file`. # .env is gitignored and excluded from the build context; .env.example names # what belongs in it. +# +# NOTE: this key needs PR #53 (jiajunh/can-232-...), which is still open against +# main. On main nothing reads it -- `grep -rn env_file ventis/` finds no hits -- +# so the key is inert, no credential reaches the container, and every request +# answers a Bedrock credential error. `validate.py` reports that as V030 until +# the PR lands. env_file: .env diff --git a/examples/joke_writer/config/policy.yaml b/examples/joke_writer/config/policy.yaml index e5c049f..2cb9cb3 100644 --- a/examples/joke_writer/config/policy.yaml +++ b/examples/joke_writer/config/policy.yaml @@ -4,9 +4,12 @@ # Rules are evaluated most-specific-first (most matching keys wins). # An empty match ({}) acts as a default fallback. # -# This file is not optional. `_load_policy_rules` returns None when it is -# missing and `_load_and_write_policies` then calls len() on that, so -# `ventis deploy` dies in GlobalController.__init__ before any container starts. +# This file IS optional -- `_load_policy_rules` logs "No policy file found" and +# returns [], and `_check_policy` allows everything when the rule list is empty. +# What is not safe is a half-written one: past the isfile() guard the read is +# unguarded, so an empty file (`.get("rules")` on None) or a null `rules:` +# (`None.sort()`) raises inside GlobalController.__init__ and `ventis deploy` +# dies before any container starts. Delete it or fill it; do not leave it empty. rules: # Default fallback: the workflow and the one agent behind it. A service left From c34e2a433b9019725ea8f477f4aef2f7cfd7e7fd Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 16:02:37 -0700 Subject: [PATCH 13/43] docs: design for testing porting-to-ventis across 100 repositories CAN-238. Eight-stage pipeline where only the port stage runs an agent, so failures at validate/build/deploy/serve are attributable to the port itself. Two constraints from the skill's own rules shape the design: the source tree stays read-only (M19/M20), so Bedrock is reached by env var against its native OpenAI- and Anthropic-compatible surfaces plus a model-id rewriting shim; and the skill is pinned per run rather than auto-edited, so results within a run are comparable and results across runs are diffable. --- skill_harness/DESIGN.md | 204 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 skill_harness/DESIGN.md diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md new file mode 100644 index 0000000..0224783 --- /dev/null +++ b/skill_harness/DESIGN.md @@ -0,0 +1,204 @@ +# Testing `porting-to-ventis` across 100 repositories + +CAN-238 design. Written 2026-08-27. + +## What this is for + +`.claude/skills/porting-to-ventis/` claims that an arbitrary agent project can be +moved onto Ventis by writing four files beside an untouched source tree. The claim +has been checked against one project (`examples/joke_writer`). This harness checks +it against a hundred, and produces per-repo evidence of where the claim broke. + +The output is not a pass rate on its own. It is a table of *how far each repo got* +and *what stopped it*, partitioned by whether the blame lies with the skill, with +Ventis, or with the repo. + +## The measurement problem, and what follows from it + +Two constraints shape everything below. Both come from the skill's own rules. + +**The source tree may not be edited.** M19 (`NEVER edit the source tree`) and M20 +(`NEVER swap the LLM provider the source uses`) are rules the skill is being +tested on. A harness that rewrites each repo's model calls onto Bedrock before +running the skill is not testing the skill — it is testing the rewrite, and every +downstream failure becomes unattributable. So the source tree is read-only for +the entire pipeline, and the LLM problem is solved outside it (§3). + +**The skill may not change mid-run.** If the skill is edited between repo 1 and +repo 100, the two were not given the same test, the pass rate has no denominator, +and a fix that merely relocates a failure looks like a fix. So `tests.skill_sha` +is pinned on every row and the harness never writes to the skill. Fixes happen +between runs, as a new pinned version, with the affected repos re-run. Which +corner cases a new version closed is then a diff between two runs — which is what +CAN-237 wants anyway. + +## 1. Pipeline + +Eight stages. `tests.farthest_step` is the last one that passed. + +| # | Stage | What runs | Fails when | +|---|-------|-----------|-----------| +| 1 | `fetched` | `git clone --depth 1`, record SHA | repo gone, too large, no license | +| 2 | `screened` | static scan: framework, LLM provider, hardcoded model ids, dependency shape | repo is out of scope for this run | +| 3 | `wired` | write `.env`, ensure model shim is up | no Bedrock credential, unmappable model | +| 4 | `ported` | **`claude -p`** running `porting-to-ventis` | agent gives up, budget exhausted, timeout | +| 5 | `validated` | `validate.py ` | contract violation the agent introduced | +| 6 | `built` | `ventis build` + both probes from SKILL.md Step 4 | image builds but container cannot import | +| 7 | `deployed` | `ventis deploy` | port/config/policy failure | +| 8 | `served` | `POST /main` → `GET /status/` | `"No agent loaded"`, provider error, wrong shape | + +**Only stage 4 uses an agent.** Everything else is a deterministic subprocess with +a timeout. This is the property that makes failures attributable: a stage 6 failure +is a fact about the port, not about how the agent happened to behave that day. + +Stage 6 runs *both* probes from SKILL.md Step 4, in order, because neither covers +the other — probe 1 (`import local_controller`) catches the protobuf/gRPC wall +before the agent is ever reached; probe 2 (`_load_agent`-shaped import) catches +the failures that otherwise surface only as `"No agent loaded"` at stage 8. + +## 2. Driving Claude Code + +A `claude -p` subprocess per repo, concurrency 2 until the pipeline is proven. + +``` +claude -p "" \ + --bare \ + --setting-sources "" \ + --permission-mode bypassPermissions \ + --output-format stream-json --verbose \ + --model --effort \ + --max-budget-usd \ + --no-session-persistence +``` + +Every flag above was checked against `claude --help` on the machine that will run +it, not recalled. + +- **`--bare` is not optional.** It suppresses hooks, auto-memory, plugin sync and + CLAUDE.md auto-discovery. Without it the operator's personal `~/.claude/CLAUDE.md` + and accumulated auto-memory enter all 100 runs, vary between them, and are + invisible in the results. Under `--bare` auth is strictly `ANTHROPIC_API_KEY`. +- **`--setting-sources ""`** keeps user/project/local settings out for the same + reason. +- **`--max-budget-usd`** is the containment mechanism; this CLI has no `--max-turns`. + A budget-exhausted run is recorded as its own failure mode, not as a crash. +- **The skill is delivered explicitly**, by copying + `.claude/skills/porting-to-ventis/` into each repo working directory, so the + version under test is the version recorded — never whatever is globally installed. +- **Tool restriction is unresolved and must be measured.** There are reports that + under `bypassPermissions`, `--allowedTools` is ignored and only `--disallowedTools` + constrains the tool set. This is verified on the first repo before the run scales; + it is not assumed in either direction. + +`--output-format stream-json` is written to `tests.trace_path`. The trace is the +only record of *how* the agent reached its result and is what makes a skill defect +diagnosable after the fact. + +## 3. Reaching Bedrock without touching the source + +Verified against AWS documentation on 2026-08-27: + +| Source SDK | Base URL | Auth header | +|---|---|---| +| `openai` / `ChatOpenAI` | `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` | `Authorization: Bearer $AWS_BEARER_TOKEN_BEDROCK` | +| `anthropic` / `ChatAnthropic` | `https://bedrock-runtime.{region}.amazonaws.com/anthropic` | `x-api-key: $AWS_BEARER_TOKEN_BEDROCK` | + +Both surfaces support client-side tool use. Both are reachable by environment +variable alone, which is why the source tree never needs an edit. + +**Model coverage is asymmetric and constrains repo selection.** Counted from +Bedrock's API-compatibility tables: 40 models serve Chat Completions (OpenAI, Qwen, +Mistral, Google, Z.AI, NVIDIA, DeepSeek, MiniMax, xAI, Moonshot); 7 serve the +Messages API, all Anthropic Claude. No Claude model serves Chat Completions, and +Meta / Amazon / Cohere / AI21 serve neither. A `ChatOpenAI` repo therefore lands on +a gpt-oss / Qwen / Mistral class model, never on Claude. + +**The model id is the one thing an env var cannot reach.** A repo writes +`ChatOpenAI(model="gpt-4o-mini")`; the id travels in the request body, and Bedrock +rejects it. The fix is a shim in front of Bedrock that **rewrites the `model` field +and forwards everything else unchanged**. No protocol translation is involved — +Bedrock speaks both wire formats natively — so this is a small addition to the +`llm_proxy` skeleton on PR #54 (`core.proxy_request`, `providers/base.HttpProvider`), +not a new subsystem. Its `hooks.py` seam yields per-repo token accounting for free. + +The mapping from observed model id to Bedrock model id is shim configuration, +recorded per run, so a result can always be read against the model that produced it. + +Credentials reach the containers through `env_file:` (PR #53, merged into this +branch), which is the only sanctioned path — M18 forbids baking a key into the +build context. + +**Prerequisite:** no AWS credential exists on the target machine today +(`~/.aws/` holds no credentials file; neither `AWS_BEARER_TOKEN_BEDROCK` nor +`AWS_ACCESS_KEY_ID` is set). Stage 3 cannot run until a Bedrock API key exists. + +## 4. Storage + +SQLite. The two tables from the ticket, plus the fields the two constraints above require. + +```sql +CREATE TABLE repos ( + id INTEGER PRIMARY KEY, + repo TEXT UNIQUE NOT NULL, -- github url + stars INTEGER, + framework TEXT, -- langchain|langgraph|crewai|autogen|plain|adk + is_multiagent INTEGER, + description TEXT +); + +CREATE TABLE tests ( + id INTEGER PRIMARY KEY, + repo TEXT NOT NULL REFERENCES repos(repo), + repo_sha TEXT NOT NULL, -- pins the source under test + skill_sha TEXT NOT NULL, -- pins the skill under test + farthest_step TEXT NOT NULL, -- the stage enum of §1 + core_issue TEXT, -- json: Ventis defects + skill_issue TEXT, -- json: skill defects + analysis TEXT, -- AI recap + status TEXT NOT NULL, -- passed|failed|blocked|budget_exhausted|timeout + cost_usd REAL, + duration_s REAL, + trace_path TEXT +); +``` + +`core_issue` and `skill_issue` are separate columns on purpose: "Ventis cannot do +this" and "the skill fails to say this" are different findings with different +owners, and collapsing them loses the distinction the run exists to produce. + +`(repo_sha, skill_sha)` is what makes two runs comparable and two *different* runs +diffable. + +## 5. Scope of the first version + +Stages 1–8 straight through, concurrency fixed at 2, repo list supplied by hand — +two repos from `langchain-samples`. + +Deliberately excluded until the pipeline is proven: GitHub search and automated +repo selection, retry policy, parallelism above 2, and any cross-repo aggregation +beyond the raw table. These are worth writing once the failure modes are known and +not before. + +## 6. Rejected alternatives + +**Rewriting each repo's LLM calls onto Bedrock before the port** — the literal +reading of the ticket plan. Rejected: it violates M19 and M20, which are rules +under test, and it contaminates every downstream stage. `examples/joke_writer`'s +README already records this conclusion for the one project where the rewrite was +done deliberately: it "is not something the `porting-to-ventis` skill should do on +a user's project — it is the credential wall, and the skill's instruction is to +report it." + +**A protocol-translating proxy** (OpenAI/Anthropic wire format → Bedrock Converse). +Rejected as unnecessary: Bedrock serves both wire formats natively, so only the +model id needs rewriting. + +**The Claude Agent SDK as the driver.** Considered for its structured event stream. +Rejected for the first version: a `claude -p` subprocess gives the same trace via +`--output-format stream-json` with one less dependency, and the pipeline's +attribution comes from stages 5–8 being deterministic rather than from finer +introspection of stage 4. + +**Restricting the run to repos already on Bedrock.** Rejected: too few exist to +reach 100, and selecting for them would bias the sample toward projects that never +exercise the credential wall the skill has the most to say about. From be72dba26abaa594c4ad70849ac204c58dcbc288 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 16:43:13 -0700 Subject: [PATCH 14/43] docs: store artifacts rather than analysis; let validate not gate the build The schema was carrying tables for things derivable after the fact from the run's artifacts. Cut back to the ticket's two tables plus the three SHAs that cannot be reconstructed once a run is over, and an artifacts directory that every later analysis reads from. Stage 5 no longer halts the pipeline. A validation that was wrong to block is only observable if the build runs anyway, and that observation cannot be recovered later. --- skill_harness/DESIGN.md | 55 ++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md index 0224783..e91be37 100644 --- a/skill_harness/DESIGN.md +++ b/skill_harness/DESIGN.md @@ -34,7 +34,9 @@ CAN-237 wants anyway. ## 1. Pipeline -Eight stages. `tests.farthest_step` is the last one that passed. +Eight stages. `tests.farthest_step` is the furthest one reached. Stage 5 is the +one exception: it does not gate what follows (see below), so its verdict is +recorded in `tests.validate_ok` rather than by halting the pipeline. | # | Stage | What runs | Fails when | |---|-------|-----------|-----------| @@ -56,6 +58,14 @@ the other — probe 1 (`import local_controller`) catches the protobuf/gRPC wall before the agent is ever reached; probe 2 (`_load_agent`-shaped import) catches the failures that otherwise surface only as `"No agent loaded"` at stage 8. +**Stage 5 does not gate stages 6–8.** A failed `validate.py` is recorded and the +pipeline continues. This is the only way to observe a validation that was wrong to +block — validate says no, the port would have served anyway — and that observation +cannot be recovered later, because the build never ran. Together with the opposite +case (validate passes, a later stage fails, which is visible by default) it gives +`validate.py` a confusion matrix, which is the only quantitative basis on which the +script can be improved. The cost is a few wasted builds. + ## 2. Driving Claude Code A `claude -p` subprocess per repo, concurrency 2 until the pipeline is proven. @@ -90,9 +100,9 @@ it, not recalled. constrains the tool set. This is verified on the first repo before the run scales; it is not assumed in either direction. -`--output-format stream-json` is written to `tests.trace_path`. The trace is the -only record of *how* the agent reached its result and is what makes a skill defect -diagnosable after the fact. +`--output-format stream-json` is written into the run's `artifacts/` directory. The +trace is the only record of *how* the agent reached its result, and it is what makes +a skill defect diagnosable after the fact. ## 3. Reaching Bedrock without touching the source @@ -134,7 +144,8 @@ build context. ## 4. Storage -SQLite. The two tables from the ticket, plus the fields the two constraints above require. +SQLite, holding the two tables from the ticket. The database stores **artifacts and +versions, not analysis.** ```sql CREATE TABLE repos ( @@ -149,25 +160,39 @@ CREATE TABLE repos ( CREATE TABLE tests ( id INTEGER PRIMARY KEY, repo TEXT NOT NULL REFERENCES repos(repo), - repo_sha TEXT NOT NULL, -- pins the source under test - skill_sha TEXT NOT NULL, -- pins the skill under test + repo_sha TEXT NOT NULL, -- which source + skill_sha TEXT NOT NULL, -- which skill + ventis_sha TEXT NOT NULL, -- which core farthest_step TEXT NOT NULL, -- the stage enum of §1 + status TEXT NOT NULL, -- passed|failed|blocked|budget_exhausted|timeout + validate_ok INTEGER, -- stage 5's verdict, kept apart from the outcome core_issue TEXT, -- json: Ventis defects skill_issue TEXT, -- json: skill defects analysis TEXT, -- AI recap - status TEXT NOT NULL, -- passed|failed|blocked|budget_exhausted|timeout cost_usd REAL, - duration_s REAL, - trace_path TEXT + artifacts TEXT NOT NULL -- directory: trace, the four written files, + -- validate output, per-stage stderr ); ``` -`core_issue` and `skill_issue` are separate columns on purpose: "Ventis cannot do -this" and "the skill fails to say this" are different findings with different -owners, and collapsing them loses the distinction the run exists to produce. +The three SHAs are the only things that cannot be reconstructed afterwards — once +the run is over, which skill and which core produced a result is unrecoverable. +Everything else about *why* a repo failed is computed later by reading `artifacts/`. + +`validate_ok` is a separate column from `farthest_step` so the two can be joined: +that join is the confusion matrix of §1. + +`core_issue` and `skill_issue` stay separate columns: "Ventis cannot do this" and +"the skill fails to say this" are findings with different owners, and collapsing +them loses the distinction the run exists to produce. -`(repo_sha, skill_sha)` is what makes two runs comparable and two *different* runs -diffable. +Deliberately **not** in the schema: per-check validation statistics, agent-behaviour +counts (which skill files were read, how many edits were retried), and the +deterministic audits of the MUST rules a machine can decide (M19's clean +`git status` on the source, M20's unchanged provider imports). All of these are +derivable from `artifacts/` by a script, at any time, without re-running anything — +and running the pipeline is the expensive part. Write those scripts when there is a +corpus worth aggregating and it is clear what to aggregate. ## 5. Scope of the first version From 5f77b1038491a550f8bd565ae0e6f8514fa26183 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 16:51:11 -0700 Subject: [PATCH 15/43] skill_harness: the eight-stage pipeline Runs porting-to-ventis against a repo list and records how far each got. Only stage 4 runs an agent; the rest are deterministic subprocesses, which is what makes a build or deploy failure a fact about the port. The shim rewrites the model id on the way to Bedrock and forwards everything else unchanged, so no repo's source is edited to reach a provider. Stage 5 records validate.py's verdict without gating the build, so the runs where it was wrong to block are observable; report prints the confusion matrix. Stages 6-8 hold a global lock: they collide on image tags, the workflow port, and the Redis container ventis deploy starts. --- .gitignore | 4 +- skill_harness/README.md | 52 +++++ skill_harness/__init__.py | 0 skill_harness/__main__.py | 135 +++++++++++++ skill_harness/db.py | 116 +++++++++++ skill_harness/repos.yaml | 35 ++++ skill_harness/runner.py | 146 ++++++++++++++ skill_harness/screen.py | 141 ++++++++++++++ skill_harness/shim.py | 195 +++++++++++++++++++ skill_harness/stages.py | 400 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 1223 insertions(+), 1 deletion(-) create mode 100644 skill_harness/README.md create mode 100644 skill_harness/__init__.py create mode 100644 skill_harness/__main__.py create mode 100644 skill_harness/db.py create mode 100644 skill_harness/repos.yaml create mode 100644 skill_harness/runner.py create mode 100644 skill_harness/screen.py create mode 100644 skill_harness/shim.py create mode 100644 skill_harness/stages.py diff --git a/.gitignore b/.gitignore index de8776e..1b1c177 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ AWSCLIV2.pkg uv.lock Agent Artifacts -docs/ \ No newline at end of file +docs/ +# skill_harness working tree: clones, artifacts, results db +.harness/ diff --git a/skill_harness/README.md b/skill_harness/README.md new file mode 100644 index 0000000..b658dff --- /dev/null +++ b/skill_harness/README.md @@ -0,0 +1,52 @@ +# skill_harness + +Runs `porting-to-ventis` against a list of repositories and records how far each +one got. `DESIGN.md` is why it is shaped this way; this file is how to run it. + +## Setup + +```shell +uv venv --python 3.12 .venv +uv pip install -e . +export AWS_BEARER_TOKEN_BEDROCK=... # required; stage 3 cannot run without it +export ANTHROPIC_API_KEY=... # required; `claude --bare` reads only this +``` + +## Run + +```shell +.venv/bin/python -m skill_harness run --repos skill_harness/repos.yaml +.venv/bin/python -m skill_harness report +``` + +Results land in `.harness/results.sqlite`; each repo's artifacts — the agent +trace, the four files it wrote, every stage's log — in `.harness//artifacts/`. +Nothing is analysed at write time, so the aggregations come later, from those +directories. + +## What each module does + +| File | Stage | Job | +|---|---|---| +| `runner.py` | — | sequences the pipeline; concurrency 2, with 6–8 serialised | +| `screen.py` | 2 | reads the repo without running it; finds the hardcoded model ids | +| `shim.py` | 3 | rewrites the `model` field on the way to Bedrock; nothing else | +| `stages.py` | 1–8 | one function per stage, each writing its own log | +| `db.py` | — | schema, and `confusion()` — validate.py's accuracy | + +## Two things to know before reading a result + +**Stages 6–8 hold a global lock.** They build images tagged `ventis-`, +bind the workflow's `api_port`, and `ventis deploy` starts its own Redis +container. Two repos cannot be in those stages at once no matter how wide +`--concurrency` is, so raising it past 2 buys less than it looks like it should. + +**Stage 5 does not gate.** `validate.py` failing does not stop the build. That is +deliberate — it is the only way to find out that a validation was wrong to block — +so `farthest_step` can read `served` on a repo whose `validate_ok` is 0. Those +rows are the interesting ones. `report` prints the resulting confusion matrix. + +## Not done yet + +GitHub search and repo selection (the list is hand-written), retry policy, and +any aggregation over `artifacts/`. Deliberately — see `DESIGN.md` section 5. diff --git a/skill_harness/__init__.py b/skill_harness/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py new file mode 100644 index 0000000..67e26bb --- /dev/null +++ b/skill_harness/__main__.py @@ -0,0 +1,135 @@ +"""CLI. + + python -m skill_harness run --repos skill_harness/repos.yaml + python -m skill_harness report +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +import yaml + +from . import db, runner, shim +from .stages import Config + +HARNESS_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_WORK = HARNESS_ROOT / ".harness" +DEFAULT_DB = DEFAULT_WORK / "results.sqlite" + + +def _load_repos(path: Path) -> list[str]: + doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return [r["repo"] if isinstance(r, dict) else r for r in doc.get("repos", [])] + + +def _model_map(path: Path) -> shim.ModelMap: + doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + m = doc.get("models", {}) + return shim.ModelMap( + exact=m.get("exact", {}), + prefixes=[(p["prefix"], p["to"]) for p in m.get("prefixes", [])], + defaults=m["defaults"], + ) + + +def cmd_run(args: argparse.Namespace) -> int: + work = Path(args.work).resolve() + work.mkdir(parents=True, exist_ok=True) + repos_file = Path(args.repos).resolve() + repos = _load_repos(repos_file) + if not repos: + print(f"no repos listed in {repos_file}", file=sys.stderr) + return 2 + + key = os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "") + if not key: + # Stage 3 would report this per repo, but failing here says it once and + # avoids cloning a hundred repos to learn it. + print("AWS_BEARER_TOKEN_BEDROCK is not set; stage 3 cannot run.", file=sys.stderr) + return 2 + + shim.start(region=args.region, key=key, model_map=_model_map(repos_file), + port=args.shim_port) + + cfg = Config( + harness_root=HARNESS_ROOT, + work_root=work, + region=args.region, + shim_base=f"{args.shim_host}:{args.shim_port}", + model=args.model, + effort=args.effort, + budget_usd=args.budget, + port_timeout=args.port_timeout, + stage_timeout=args.stage_timeout, + skill_sha=runner._tree_sha(HARNESS_ROOT, ".claude/skills/porting-to-ventis"), + ventis_sha=runner._tree_sha(HARNESS_ROOT, "ventis"), + disallowed_tools=args.disallowed_tools, + ) + logging.info("skill %s | core %s | model %s/%s", + cfg.skill_sha[:12], cfg.ventis_sha[:12], cfg.model, cfg.effort) + + conn = db.connect(args.db) + records = runner.run_all(repos, cfg, conn, concurrency=args.concurrency) + + failed = sum(1 for r in records if r["status"] != "passed") + print(f"\n{len(records)} repos, {len(records) - failed} served, {failed} did not") + return 0 + + +def cmd_report(args: argparse.Namespace) -> int: + conn = db.connect(args.db) + rows = db.summary(conn) + if not rows: + print("no results yet") + return 0 + width = max(len(r["repo"]) for r in rows) + for r in rows: + v = {None: "-", 1: "pass", 0: "FAIL"}[r["validate_ok"]] + print(f"{r['repo']:<{width}} {r['farthest_step']:<10} {r['status']:<18} validate={v}") + print("\nvalidate.py against the eventual outcome:") + print(json.dumps(db.confusion(conn), indent=2)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="skill_harness") + parser.add_argument("--db", default=str(DEFAULT_DB)) + parser.add_argument("-v", "--verbose", action="store_true") + sub = parser.add_subparsers(dest="command", required=True) + + run_p = sub.add_parser("run", help="run the pipeline over a repo list") + run_p.add_argument("--repos", default=str(HARNESS_ROOT / "skill_harness" / "repos.yaml")) + run_p.add_argument("--work", default=str(DEFAULT_WORK)) + run_p.add_argument("--concurrency", type=int, default=2) + run_p.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) + run_p.add_argument("--shim-port", type=int, default=8300) + # Containers reach the host by a different name than the harness does. + run_p.add_argument("--shim-host", default="http://host.docker.internal") + run_p.add_argument("--model", default="opus") + run_p.add_argument("--effort", default="high") + run_p.add_argument("--budget", type=float, default=8.0) + run_p.add_argument("--port-timeout", type=int, default=3600) + run_p.add_argument("--stage-timeout", type=int, default=900) + run_p.add_argument("--disallowed-tools", default="") + run_p.set_defaults(func=cmd_run) + + rep_p = sub.add_parser("report", help="print the results table") + rep_p.set_defaults(func=cmd_report) + + args = parser.parse_args(argv) + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-5s %(name)-8s %(message)s", + datefmt="%H:%M:%S", + ) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skill_harness/db.py b/skill_harness/db.py new file mode 100644 index 0000000..20924f0 --- /dev/null +++ b/skill_harness/db.py @@ -0,0 +1,116 @@ +"""SQLite storage for harness runs. + +The database holds artifacts and versions, not analysis. The three SHAs are the +only things that cannot be reconstructed once a run is over; everything about +*why* a repo failed is recomputed later by reading its artifacts directory. +See DESIGN.md section 4. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +# The gating stages, in order. Stage 5 (`validated`) is deliberately absent: it +# does not halt the pipeline, so it cannot be the "furthest step reached" — its +# verdict lives in tests.validate_ok instead. See DESIGN.md section 1. +STAGES = ["fetched", "screened", "wired", "ported", "built", "deployed", "served"] + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS repos ( + id INTEGER PRIMARY KEY, + repo TEXT UNIQUE NOT NULL, + stars INTEGER, + framework TEXT, + is_multiagent INTEGER, + description TEXT +); + +CREATE TABLE IF NOT EXISTS tests ( + id INTEGER PRIMARY KEY, + repo TEXT NOT NULL REFERENCES repos(repo), + repo_sha TEXT NOT NULL, + skill_sha TEXT NOT NULL, + ventis_sha TEXT NOT NULL, + farthest_step TEXT NOT NULL, + status TEXT NOT NULL, + validate_ok INTEGER, + core_issue TEXT, + skill_issue TEXT, + analysis TEXT, + cost_usd REAL, + artifacts TEXT NOT NULL, + started_at TEXT NOT NULL, + ended_at TEXT +); +""" + + +def connect(path: str | Path) -> sqlite3.Connection: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path, check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + conn.commit() + return conn + + +def upsert_repo(conn: sqlite3.Connection, repo: str, **fields) -> None: + """Insert the repo if new, then update whichever columns were supplied. + + Stage 2 learns most of these, so a repo row is written twice: once empty at + fetch time, once populated after the screen. + """ + with conn: + conn.execute("INSERT OR IGNORE INTO repos (repo) VALUES (?)", (repo,)) + known = {"stars", "framework", "is_multiagent", "description"} + cols = {k: v for k, v in fields.items() if k in known and v is not None} + if cols: + assigns = ", ".join(f"{k} = ?" for k in cols) + conn.execute( + f"UPDATE repos SET {assigns} WHERE repo = ?", + (*cols.values(), repo), + ) + + +def record_test(conn: sqlite3.Connection, **fields) -> int: + for key in ("core_issue", "skill_issue"): + if isinstance(fields.get(key), (list, dict)): + fields[key] = json.dumps(fields[key]) + cols = ", ".join(fields) + marks = ", ".join("?" for _ in fields) + with conn: + cur = conn.execute( + f"INSERT INTO tests ({cols}) VALUES ({marks})", tuple(fields.values()) + ) + return cur.lastrowid + + +def summary(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return conn.execute( + """ + SELECT repo, farthest_step, status, validate_ok, cost_usd, artifacts + FROM tests ORDER BY id + """ + ).fetchall() + + +def confusion(conn: sqlite3.Connection) -> dict[str, int]: + """validate.py's confusion matrix — the point of not letting stage 5 gate. + + A false negative is a check validate.py is missing. A false positive is a + check that was wrong to block, and is only observable because the build ran + anyway. + """ + rows = conn.execute( + "SELECT validate_ok, farthest_step FROM tests WHERE validate_ok IS NOT NULL" + ).fetchall() + served = lambda r: r["farthest_step"] == "served" # noqa: E731 + return { + "true_positive": sum(1 for r in rows if not r["validate_ok"] and not served(r)), + "false_positive": sum(1 for r in rows if not r["validate_ok"] and served(r)), + "false_negative": sum(1 for r in rows if r["validate_ok"] and not served(r)), + "true_negative": sum(1 for r in rows if r["validate_ok"] and served(r)), + } diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml new file mode 100644 index 0000000..a99fdcc --- /dev/null +++ b/skill_harness/repos.yaml @@ -0,0 +1,35 @@ +# The repos under test, and the model mapping the shim applies to them. +# +# Hand-supplied while the pipeline is being proven. GitHub search and automated +# selection are deliberately out of scope until the failure modes are known — +# see DESIGN.md section 5. + +repos: + - https://github.com/langchain-ai/langchain-academy + - https://github.com/langchain-ai/rag-from-scratch + +# A repo's model id is hardcoded in its source, and the source is never edited, +# so the shim rewrites the id on the way past. Anything not matched here falls to +# the per-surface default. +# +# Note the asymmetry, which is a property of Bedrock and not a choice: no Claude +# model serves Chat Completions, so an OpenAI-SDK repo lands on a gpt-oss class +# model and never on Claude. See DESIGN.md section 3. +models: + defaults: + openai: openai.gpt-oss-120b + anthropic: us.anthropic.claude-sonnet-5 + + exact: {} + + prefixes: + - prefix: gpt- + to: openai.gpt-oss-120b + - prefix: o1 + to: openai.gpt-oss-120b + - prefix: o3 + to: openai.gpt-oss-120b + - prefix: o4 + to: openai.gpt-oss-120b + - prefix: claude- + to: us.anthropic.claude-sonnet-5 diff --git a/skill_harness/runner.py b/skill_harness/runner.py new file mode 100644 index 0000000..903090a --- /dev/null +++ b/skill_harness/runner.py @@ -0,0 +1,146 @@ +"""Orchestration. + +Stages 1-5 run concurrently across repos. Stages 6-8 take a global lock: they +build images tagged `ventis-`, bind the workflow's api_port, and +`ventis deploy` starts its own Redis container, so two repos cannot be in them at +once regardless of how wide the pool is. +""" + +from __future__ import annotations + +import logging +import re +import subprocess +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +from . import db, shim, stages +from .stages import Config, Ctx, Result + +log = logging.getLogger("runner") + +# (stage name, function, gating). Stage 5 is the one non-gating stage: a failed +# validate.py is recorded and the pipeline continues, which is the only way to +# observe a validation that was wrong to block. See DESIGN.md section 1. +PIPELINE = [ + ("fetched", stages.fetch, True), + ("screened", stages.screen, True), + ("wired", stages.wire, True), + ("ported", stages.port, True), + ("validated", stages.validate, False), + ("built", stages.build, True), + ("deployed", stages.deploy, True), + ("served", stages.serve, True), +] + +DOCKER_STAGES = {"built", "deployed", "served"} + +_SLUG = re.compile(r"[^a-z0-9]+") + + +def slug_for(repo: str) -> str: + return _SLUG.sub("-", repo.rstrip("/").split("/")[-1].removesuffix(".git").lower()).strip("-") + + +def _tree_sha(root: Path, path: str) -> str: + """The git tree hash of a subdirectory — it changes when that subtree changes + and not when anything else in the repo does, which is exactly what pinning + the skill and the core each require.""" + try: + out = subprocess.run(["git", "rev-parse", f"HEAD:{path}"], cwd=root, + capture_output=True, text=True, timeout=30) + return out.stdout.strip() or "unknown" + except Exception: + return "unknown" + + +def _classify(stage: str, result: Result, ctx: Ctx) -> str: + if stage == "screened": + return "blocked" # out of scope for this run, not a skill failure + if stage == "wired": + return "blocked" # missing credential, nothing was tested + if stage == "ported": + trace = ctx.log_path("4-port.log") + text = trace.read_text(encoding="utf-8", errors="replace") if trace.is_file() else "" + if "budget" in text.lower() and "exceed" in text.lower(): + return "budget_exhausted" + if "timed out" in result.detail or "timed out" in text: + return "timeout" + return "failed" + + +def run_repo(repo: str, cfg: Config, conn, docker_lock: threading.Lock) -> dict: + slug = slug_for(repo) + artifacts = cfg.work_root / slug / "artifacts" + artifacts.mkdir(parents=True, exist_ok=True) + ctx = Ctx(repo=repo, slug=slug, root=cfg.work_root / slug / "src", + artifacts=artifacts, cfg=cfg) + + started = datetime.now(timezone.utc).isoformat(timespec="seconds") + farthest, status = "none", "passed" + began = time.time() + + try: + for stage, fn, gating in PIPELINE: + if stage in DOCKER_STAGES: + docker_lock.acquire() + try: + result = fn(ctx) + except Exception as e: # a harness bug, not a port failure + log.exception("%s: %s crashed", slug, stage) + result = Result(False, f"harness error: {type(e).__name__}: {e}") + finally: + if stage in DOCKER_STAGES: + docker_lock.release() + + mark = "ok " if result.ok else "FAIL" + log.info("%-24s %-10s %s %s", slug, stage, mark, result.detail) + + if result.ok: + if gating: + farthest = stage + elif gating: + status = _classify(stage, result, ctx) + break + finally: + stages.teardown(ctx) + + usage = shim.usage_for(slug) + (artifacts / "usage.json").write_text(str(usage), encoding="utf-8") + + if ctx.screen: + db.upsert_repo(conn, repo, framework=ctx.screen.framework, + is_multiagent=int(ctx.screen.is_multiagent), + description=ctx.screen.description) + else: + db.upsert_repo(conn, repo) + + record = dict( + repo=repo, + repo_sha=ctx.repo_sha or "unknown", + skill_sha=cfg.skill_sha, + ventis_sha=cfg.ventis_sha, + farthest_step=farthest, + status=status, + validate_ok=None if ctx.validate_ok is None else int(ctx.validate_ok), + core_issue=ctx.core_issue or None, + skill_issue=ctx.skill_issue or None, + analysis=None, + cost_usd=None, + artifacts=str(artifacts), + started_at=started, + ended_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + ) + db.record_test(conn, **record) + log.info("%-24s => %s at %s (%.0fs)", slug, status, farthest, time.time() - began) + return record + + +def run_all(repos: list[str], cfg: Config, conn, concurrency: int = 2) -> list[dict]: + docker_lock = threading.Lock() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(run_repo, r, cfg, conn, docker_lock) for r in repos] + return [f.result() for f in futures] diff --git a/skill_harness/screen.py b/skill_harness/screen.py new file mode 100644 index 0000000..cadc905 --- /dev/null +++ b/skill_harness/screen.py @@ -0,0 +1,141 @@ +"""Stage 2 — read the repo without running it. + +Two jobs. It decides whether the repo is in scope for this run, and it finds the +hardcoded model ids that stage 3 has to teach the shim about, because that is the +one thing an environment variable cannot reach (DESIGN.md section 3). +""" + +from __future__ import annotations + +import ast +import re +from dataclasses import dataclass, field +from pathlib import Path + +SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox", "build", "dist"} + +FRAMEWORK_MARKERS = [ + ("langgraph", ("langgraph",)), + ("langchain", ("langchain", "langchain_core", "langchain_community")), + ("crewai", ("crewai",)), + ("autogen", ("autogen", "autogen_agentchat")), + ("adk", ("google.adk", "google_adk")), +] + +# Ordered: the first two decide which base URL stage 3 writes, so they must be +# recognised by more than their own package name — a repo commonly reaches a +# provider through a wrapper, and matching only `openai`/`anthropic` misses it. +SDK_MARKERS = [ + ("openai", ("openai", "langchain_openai", "llama_index.llms.openai")), + ("anthropic", ("anthropic", "langchain_anthropic", "llama_index.llms.anthropic")), + ("bedrock", ("boto3", "botocore", "langchain_aws", "ventis.llm", "ventis")), + # Reaches a model, but not through a provider SDK we can redirect by env var. + ("other", ("litellm", "instructor", "google.generativeai", "google.genai", + "cohere", "mistralai", "ollama", "langchain.chat_models")), +] + +# Model ids as they appear in source. Deliberately broad: a literal this matches +# is a candidate for the shim's mapping table, and a human reads the list before +# the run. Missing one is a stage 8 provider error; over-matching costs nothing. +MODEL_LITERAL = re.compile( + r"\b(gpt-[\w.\-]+|o[134](?:-[\w.\-]+)?|claude-[\w.\-]+|" + r"(?:meta|mistral|amazon|cohere|anthropic|openai|qwen|deepseek)\.[\w.\-:]+)\b" +) + +MULTIAGENT_MARKERS = ("Send(", "StateGraph", "Crew(", "GroupChat", "add_edge", "Command(") + + +@dataclass +class Screen: + py_files: int = 0 + loc: int = 0 + framework: str = "plain" + llm_sdk: str = "none" + model_ids: list[str] = field(default_factory=list) + is_multiagent: bool = False + layout: str = "flat" + packaging: str = "none" + description: str = "" + reject: str | None = None + + +def _imports(tree: ast.AST) -> set[str]: + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(a.name for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + return names + + +def _matches(imports: set[str], markers: tuple[str, ...]) -> bool: + return any(i == m or i.startswith(m + ".") for i in imports for m in markers) + + +def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000) -> Screen: + out = Screen() + imports: set[str] = set() + models: set[str] = set() + + for path in root.rglob("*.py"): + if SKIP_DIRS & set(path.relative_to(root).parts): + continue + out.py_files += 1 + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + out.loc += text.count("\n") + models.update(MODEL_LITERAL.findall(text)) + if any(m in text for m in MULTIAGENT_MARKERS): + out.is_multiagent = True + try: + imports |= _imports(ast.parse(text)) + except SyntaxError: + # Not fatal to the screen. validate.py's V001/V002 is where a file + # that does not parse becomes a finding about the port. + continue + + out.model_ids = sorted(models) + + for name, markers in FRAMEWORK_MARKERS: + if _matches(imports, markers): + out.framework = name + break + + hits = [name for name, markers in SDK_MARKERS if _matches(imports, markers)] + if {"openai", "anthropic"} <= set(hits): + out.llm_sdk = "both" + elif hits: + out.llm_sdk = hits[0] + + if (root / "src").is_dir(): + out.layout = "src" + for candidate in ("pyproject.toml", "setup.py", "setup.cfg"): + if (root / candidate).is_file(): + out.packaging = candidate + break + + readme = next((p for p in root.glob("README*") if p.is_file()), None) + if readme: + body = readme.read_text(encoding="utf-8", errors="replace").strip().splitlines() + out.description = " ".join(line for line in body[:12] if line.strip())[:500] + + # Rejections. Each is a fact about scope, not a failure of the skill, so the + # test row records `screened` as the furthest step and stops there. + if out.py_files == 0: + out.reject = "no python files" + elif out.py_files > max_py_files or out.loc > max_loc: + out.reject = f"too large: {out.py_files} files, {out.loc} loc" + elif out.llm_sdk == "none" and not out.model_ids: + # Both signals absent. One alone is not enough to reject on: a wrapper + # hides the SDK, and a model id read from config leaves no literal. + out.reject = "no LLM call found" + elif out.layout == "src" and out.packaging == "none": + # M24: without packaging metadata there is no editable install, and the + # Ventis change that would make a src/ layout importable has no PR. The + # port cannot succeed, and that is a finding about Ventis, not the skill. + out.reject = "src/ layout with no packaging metadata (M24, no PR)" + + return out diff --git a/skill_harness/shim.py b/skill_harness/shim.py new file mode 100644 index 0000000..9f83d4e --- /dev/null +++ b/skill_harness/shim.py @@ -0,0 +1,195 @@ +"""A model-id rewriting proxy in front of Bedrock. + +Bedrock serves both the OpenAI Chat Completions wire format and the Anthropic +Messages wire format natively, so a ported repo needs no protocol translation — +only its base URL changed, which an environment variable can do. The one thing an +environment variable cannot reach is the model id, which travels in the request +body: a repo asks for `gpt-4o-mini` and Bedrock rejects it. + +So this rewrites the `model` field and forwards everything else unchanged. That +is the whole job. See DESIGN.md section 3. + +Requests are buffered, not streamed, matching the scope llm_proxy already set. +When PR #54 lands this should fold into `llm_proxy/providers/` as another +provider rather than continuing to live here. +""" + +from __future__ import annotations + +import json +import logging +import re +import threading +import urllib.error +import urllib.request +from collections import defaultdict +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +log = logging.getLogger("shim") + +# Per-repo token accounting, keyed by the slug in the request path. This is the +# seam llm_proxy's hooks.py exists for; here it is four lines. +USAGE: dict[str, dict[str, int]] = defaultdict(lambda: {"input": 0, "output": 0, "calls": 0}) +_USAGE_LOCK = threading.Lock() + +# Surface -> (upstream path prefix, auth header name, auth value template). +SURFACES = { + "openai": ("/openai/v1", "Authorization", "Bearer {key}"), + "anthropic": ("/anthropic", "x-api-key", "{key}"), +} + +_PATH = re.compile(r"^/r/(?P[\w.\-]+)/(?Popenai/v1|anthropic)(?P/.*)$") + +# Headers that describe the hop, not the request. Forwarding them corrupts the +# upstream call. +_HOP_BY_HOP = {"host", "content-length", "connection", "authorization", "x-api-key", + "accept-encoding", "transfer-encoding"} + + +class ModelMap: + """Resolves a source model id to a Bedrock one. + + Exact matches first, then prefix rules, then a per-surface default. The + resolved mapping is logged for every distinct source id so a result can + always be read against the model that actually produced it. + """ + + def __init__(self, exact: dict[str, str], prefixes: list[tuple[str, str]], + defaults: dict[str, str]): + self.exact = exact + self.prefixes = prefixes + self.defaults = defaults + self.seen: dict[str, str] = {} + + def resolve(self, model: str, surface: str) -> str: + if model in self.exact: + target = self.exact[model] + else: + target = next( + (dst for pre, dst in self.prefixes if model.startswith(pre)), + self.defaults[surface], + ) + if self.seen.get(model) != target: + self.seen[model] = target + log.info("model map: %s -> %s (%s)", model, target, surface) + return target + + +def _handler_class(upstream_host: str, key: str, model_map: ModelMap, timeout: float): + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): # quieter than the stdlib default + log.debug(fmt, *args) + + def _fail(self, code: int, detail: str): + body = json.dumps({"error": "shim_error", "detail": detail}).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path == "/healthz": + body = b'{"status":"ok"}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + self._proxy(b"") + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + self._proxy(self.rfile.read(length) if length else b"") + + def _proxy(self, body: bytes): + match = _PATH.match(self.path) + if not match: + return self._fail(404, f"unroutable path {self.path!r}") + slug, surface_path, rest = match["slug"], match["surface"], match["rest"] + surface = "openai" if surface_path.startswith("openai") else "anthropic" + prefix, auth_header, auth_template = SURFACES[surface] + + if body: + try: + payload = json.loads(body) + except json.JSONDecodeError: + payload = None + if isinstance(payload, dict) and "model" in payload: + payload["model"] = model_map.resolve(payload["model"], surface) + # Bedrock buffers; the repo may have asked to stream. + payload.pop("stream", None) + body = json.dumps(payload).encode() + + headers = { + k: v for k, v in self.headers.items() if k.lower() not in _HOP_BY_HOP + } + headers[auth_header] = auth_template.format(key=key) + if surface == "anthropic": + headers.setdefault("anthropic-version", "2023-06-01") + + req = urllib.request.Request( + f"{upstream_host}{prefix}{rest}", + data=body or None, + headers=headers, + method=self.command, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + status, raw = resp.status, resp.read() + except urllib.error.HTTPError as e: + status, raw = e.code, e.read() + except Exception as e: # network, DNS, timeout + return self._fail(502, f"{type(e).__name__}: {e}") + + self._account(slug, raw) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + @staticmethod + def _account(slug: str, raw: bytes): + try: + usage = json.loads(raw).get("usage") or {} + except (json.JSONDecodeError, AttributeError): + return + # OpenAI names them prompt/completion; Anthropic input/output. + inp = usage.get("input_tokens") or usage.get("prompt_tokens") or 0 + out = usage.get("output_tokens") or usage.get("completion_tokens") or 0 + with _USAGE_LOCK: + bucket = USAGE[slug] + bucket["input"] += inp + bucket["output"] += out + bucket["calls"] += 1 + + return Handler + + +def start(region: str, key: str, model_map: ModelMap, host: str = "0.0.0.0", + port: int = 8300, timeout: float = 600.0, + upstream: str | None = None) -> ThreadingHTTPServer: + """Start the shim on a daemon thread and return the server. + + It binds 0.0.0.0 because the callers are agent containers, which reach the + host by a different address than the harness does. `upstream` is injectable + so the shim can be tested without Bedrock, and pointed at `bedrock-mantle` + without a code change. + """ + upstream = upstream or f"https://bedrock-runtime.{region}.amazonaws.com" + server = ThreadingHTTPServer( + (host, port), _handler_class(upstream, key, model_map, timeout) + ) + threading.Thread(target=server.serve_forever, daemon=True, name="shim").start() + log.info("shim listening on %s:%s -> %s", host, port, upstream) + return server + + +def usage_for(slug: str) -> dict[str, int]: + with _USAGE_LOCK: + return dict(USAGE[slug]) diff --git a/skill_harness/stages.py b/skill_harness/stages.py new file mode 100644 index 0000000..74164c9 --- /dev/null +++ b/skill_harness/stages.py @@ -0,0 +1,400 @@ +"""The eight stages. + +Only stage 4 runs an agent. Everything else is a deterministic subprocess with a +timeout, which is what makes a stage 6 failure a fact about the port rather than +about how the agent happened to behave that day. See DESIGN.md section 1. + +Every stage writes its own log into the test's artifacts directory. Those logs, +plus the agent trace and the four written files, are what every later analysis +reads — the database deliberately stores none of it. +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil +import signal +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from . import screen as screen_mod + +log = logging.getLogger("stages") + +SKILL_REL = ".claude/skills/porting-to-ventis" + +PORT_PROMPT = """\ +Port the project in this directory onto Ventis. + +Use the porting-to-ventis skill. Follow it; it is the thing under test. + +Do not ask for confirmation — there is nobody to answer. When the skill tells you +to report something and stop rather than fix it, write the report to +PORT_REPORT.md in this directory and stop, which counts as following it. +""" + + +@dataclass +class Result: + ok: bool + detail: str = "" + + +@dataclass +class Ctx: + repo: str + slug: str + root: Path + artifacts: Path + cfg: "Config" + repo_sha: str = "" + screen: screen_mod.Screen | None = None + validate_ok: bool | None = None + core_issue: list = field(default_factory=list) + skill_issue: list = field(default_factory=list) + _procs: list = field(default_factory=list) + + def log_path(self, name: str) -> Path: + return self.artifacts / name + + +@dataclass +class Config: + harness_root: Path + work_root: Path + region: str + shim_base: str # what a *container* uses to reach the shim + model: str + effort: str + budget_usd: float + port_timeout: int + stage_timeout: int + # The two pins that cannot be reconstructed after a run: which skill and + # which core produced the result. Git tree hashes, so each moves only when + # its own subtree does. + skill_sha: str = "unknown" + ventis_sha: str = "unknown" + disallowed_tools: str = "" + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # + +def run(ctx: Ctx, name: str, cmd: list[str], *, cwd: Path | None = None, + timeout: int | None = None, env: dict | None = None) -> tuple[int, str]: + """Run a subprocess, tee its output into the artifacts directory, return it.""" + timeout = timeout or ctx.cfg.stage_timeout + full_env = {**os.environ, **(env or {})} + log.debug("%s: %s", name, " ".join(cmd)) + try: + proc = subprocess.run( + cmd, cwd=cwd or ctx.root, env=full_env, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + rc, out = proc.returncode, proc.stdout + except subprocess.TimeoutExpired as e: + rc, out = 124, (e.output or "") + f"\n[harness] timed out after {timeout}s\n" + except FileNotFoundError as e: + rc, out = 127, f"[harness] {e}\n" + ctx.log_path(f"{name}.log").write_text(out or "", encoding="utf-8") + return rc, out or "" + + +def _config_path(root: Path) -> Path: + return root / "config" / "global_controller.yaml" + + +def _load_yaml(path: Path) -> dict: + try: + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + return {} + + +def _agent_entries(root: Path) -> list[dict]: + return _load_yaml(_config_path(root)).get("agents") or [] + + +def _api_port(root: Path, default: int = 8080) -> int: + for entry in _agent_entries(root): + if entry.get("type") == "workflow": + return int(entry.get("api_port", default)) + return default + + +# --------------------------------------------------------------------------- # +# 1. fetched +# --------------------------------------------------------------------------- # + +def fetch(ctx: Ctx) -> Result: + if ctx.root.exists(): + shutil.rmtree(ctx.root) + ctx.root.parent.mkdir(parents=True, exist_ok=True) + rc, out = run(ctx, "1-fetch", ["git", "clone", "--depth", "1", ctx.repo, str(ctx.root)], + cwd=ctx.root.parent) + if rc != 0: + return Result(False, f"clone failed: {out.strip()[-300:]}") + rc, sha = run(ctx, "1-sha", ["git", "rev-parse", "HEAD"]) + ctx.repo_sha = sha.strip() if rc == 0 else "unknown" + return Result(True, ctx.repo_sha[:12]) + + +# --------------------------------------------------------------------------- # +# 2. screened +# --------------------------------------------------------------------------- # + +def screen(ctx: Ctx) -> Result: + ctx.screen = screen_mod.screen(ctx.root) + ctx.log_path("2-screen.json").write_text( + json.dumps(ctx.screen.__dict__, indent=2, default=str), encoding="utf-8" + ) + if ctx.screen.reject: + return Result(False, ctx.screen.reject) + return Result(True, f"{ctx.screen.framework}/{ctx.screen.llm_sdk}, {ctx.screen.loc} loc") + + +# --------------------------------------------------------------------------- # +# 3. wired +# --------------------------------------------------------------------------- # + +def wire(ctx: Ctx) -> Result: + """Write the .env the port will point `env_file:` at. + + The source tree is never edited; this adds a file beside it, which is what + the skill's own credential path expects (M18, M23). + """ + key = os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "") + if not key: + return Result(False, "AWS_BEARER_TOKEN_BEDROCK is not set") + + base = f"{ctx.cfg.shim_base}/r/{ctx.slug}" + env = "\n".join([ + "# Written by skill_harness. The source tree is untouched; this file is", + "# what `env_file:` in the port's config points at.", + f"OPENAI_BASE_URL={base}/openai/v1", + f"OPENAI_API_KEY=shim-not-used", + f"ANTHROPIC_BASE_URL={base}/anthropic", + f"ANTHROPIC_API_KEY=shim-not-used", + f"AWS_BEARER_TOKEN_BEDROCK={key}", + f"AWS_REGION={ctx.cfg.region}", + "", + ]) + (ctx.root / ".env").write_text(env, encoding="utf-8") + ctx.log_path("3-wire.log").write_text( + env.replace(key, "***"), encoding="utf-8" + ) + return Result(True, f"shim base {base}") + + +# --------------------------------------------------------------------------- # +# 4. ported — the only stage that runs an agent +# --------------------------------------------------------------------------- # + +def port(ctx: Ctx) -> Result: + """Copy the skill in, then run `claude -p` against the repo. + + The skill is delivered explicitly rather than relied on globally, so the + version under test is the version recorded in tests.skill_sha. + """ + src = ctx.cfg.harness_root / SKILL_REL + dst = ctx.root / SKILL_REL + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst, dirs_exist_ok=True) + + cmd = [ + "claude", "-p", PORT_PROMPT, + "--bare", + "--setting-sources", "", + "--permission-mode", "bypassPermissions", + "--output-format", "stream-json", "--verbose", + "--model", ctx.cfg.model, + "--effort", ctx.cfg.effort, + "--max-budget-usd", str(ctx.cfg.budget_usd), + "--no-session-persistence", + ] + if ctx.cfg.disallowed_tools: + cmd += ["--disallowedTools", ctx.cfg.disallowed_tools] + + rc, out = run(ctx, "4-port", cmd, timeout=ctx.cfg.port_timeout) + # The stream-json trace is the only record of *how* the agent got there. + ctx.log_path("4-port.trace.jsonl").write_text(out, encoding="utf-8") + + report = ctx.root / "PORT_REPORT.md" + if report.is_file(): + ctx.skill_issue.append({"kind": "reported_and_stopped", + "text": report.read_text(encoding="utf-8")[:2000]}) + + if rc != 0: + return Result(False, f"claude exited {rc}") + missing = [p for p in ("config/global_controller.yaml",) if not (ctx.root / p).is_file()] + if missing: + return Result(False, f"port produced no {missing[0]}") + return Result(True, "port written") + + +# --------------------------------------------------------------------------- # +# 5. validated — records a verdict, does not gate +# --------------------------------------------------------------------------- # + +def validate(ctx: Ctx) -> Result: + script = ctx.cfg.harness_root / SKILL_REL / "validate.py" + rc, out = run(ctx, "5-validate", ["python3", str(script), ".", "--json"]) + ctx.validate_ok = rc == 0 + if rc == 127: + ctx.validate_ok = None + return Result(False, "validate.py could not run") + try: + findings = json.loads(out) + ctx.log_path("5-validate.json").write_text(json.dumps(findings, indent=2), + encoding="utf-8") + except json.JSONDecodeError: + pass + return Result(True, "pass" if ctx.validate_ok else "fail (not gating)") + + +# --------------------------------------------------------------------------- # +# 6. built — build, then both probes from SKILL.md Step 4 +# --------------------------------------------------------------------------- # + +def build(ctx: Ctx) -> Result: + rc, out = run(ctx, "6-build", ["ventis", "build", "-c", "config/global_controller.yaml"]) + if rc != 0: + return Result(False, f"ventis build exited {rc}") + + # `ventis build` prints "Build complete." and exits 0 for a project whose + # container dies on startup, so the build result is not evidence on its own. + for entry in _agent_entries(ctx.root): + if entry.get("type") == "workflow": + continue + name = entry.get("name", "") + image = f"ventis-{name.lower()}" + + rc, out = run(ctx, f"6-probe1-{name}", + ["docker", "run", "--rm", image, "python", "-c", + "import local_controller"]) + if rc != 0: + # The gRPC/protobuf stack is unpinned and resolved on the host; this + # failure belongs to Ventis, not to the port. + ctx.core_issue.append({"kind": "runtime_import", "agent": name, + "detail": out.strip()[-500:]}) + return Result(False, f"{image}: import local_controller failed") + + entrypoint = Path(entry.get("entrypoint", "")).name + probe = ( + "import importlib.util, sys\n" + f"spec = importlib.util.spec_from_file_location('m', '{entrypoint}')\n" + "m = importlib.util.module_from_spec(spec); sys.modules['m'] = m\n" + f"spec.loader.exec_module(m); m.{name}(); print('ok')" + ) + rc, out = run(ctx, f"6-probe2-{name}", + ["docker", "run", "--rm", image, "python", "-c", probe]) + if rc != 0: + # _load_agent swallows every exception, so without this probe the + # symptom would only appear as "No agent loaded" at stage 8. + ctx.skill_issue.append({"kind": "agent_unloadable", "agent": name, + "detail": out.strip()[-500:]}) + return Result(False, f"{image}: agent would not load") + + return Result(True, "built, both probes pass") + + +# --------------------------------------------------------------------------- # +# 7. deployed +# --------------------------------------------------------------------------- # + +def deploy(ctx: Ctx) -> Result: + """`ventis deploy` blocks in a health-monitoring loop, so it runs detached + and the harness waits on the workflow's port instead.""" + logfile = ctx.log_path("7-deploy.log").open("w", encoding="utf-8") + proc = subprocess.Popen( + ["ventis", "deploy", "-c", "config/global_controller.yaml"], + cwd=ctx.root, stdout=logfile, stderr=subprocess.STDOUT, text=True, + start_new_session=True, + ) + ctx._procs.append(proc) + + port_no = _api_port(ctx.root) + deadline = time.time() + ctx.cfg.stage_timeout + while time.time() < deadline: + if proc.poll() is not None: + return Result(False, f"ventis deploy exited early ({proc.returncode})") + try: + urllib.request.urlopen(f"http://localhost:{port_no}/", timeout=2) + return Result(True, f"workflow answering on :{port_no}") + except urllib.error.HTTPError: + return Result(True, f"workflow answering on :{port_no}") # 404 is an answer + except Exception: + time.sleep(2) + return Result(False, f"workflow never answered on :{port_no}") + + +# --------------------------------------------------------------------------- # +# 8. served +# --------------------------------------------------------------------------- # + +def serve(ctx: Ctx, query: str = "animals") -> Result: + port_no = _api_port(ctx.root) + body = json.dumps({"query": query}).encode() + req = urllib.request.Request( + f"http://localhost:{port_no}/main", data=body, + headers={"Content-Type": "application/json"}, method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + accepted = json.loads(resp.read()) + except Exception as e: + return Result(False, f"POST /main failed: {type(e).__name__}: {e}") + + request_id = accepted.get("request_id") + if not request_id: + return Result(False, f"POST /main returned no request_id: {accepted}") + + deadline = time.time() + ctx.cfg.stage_timeout + last = {} + while time.time() < deadline: + try: + with urllib.request.urlopen( + f"http://localhost:{port_no}/status/{request_id}", timeout=30 + ) as resp: + last = json.loads(resp.read()) + except Exception as e: + last = {"status": "poll_failed", "detail": str(e)} + ctx.log_path("8-serve.json").write_text(json.dumps(last, indent=2), encoding="utf-8") + status = last.get("status") + if status == "done": + return Result(True, "served") + if status == "error": + ctx.skill_issue.append({"kind": "request_error", "detail": last}) + return Result(False, f"request errored: {str(last)[:300]}") + time.sleep(3) + return Result(False, f"request never completed: {str(last)[:300]}") + + +# --------------------------------------------------------------------------- # +# teardown +# --------------------------------------------------------------------------- # + +def teardown(ctx: Ctx) -> None: + """Containers and the controller outlive the pipeline unless killed.""" + for proc in ctx._procs: + if proc.poll() is None: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + proc.wait(timeout=30) + except Exception: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except Exception: + pass + ctx._procs.clear() + if _config_path(ctx.root).is_file(): + run(ctx, "9-clean", ["ventis", "clean"], timeout=120) From c1531bc45d941e43d85259804db24bf05d64e99e Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:12:56 -0700 Subject: [PATCH 16/43] skill_harness: gate on the surfaces the credential can actually reach Measured against a real Bedrock key rather than assumed. The OpenAI Chat Completions surface works; the Anthropic Messages surface is closed on this account -- every Messages-capable Claude answers permission_error, and Claude 3 Haiku names the cause as an unsubmitted use case form. Claude is reachable via Converse, but that is a third wire format and routing to it would mean the protocol translation this design avoids. Two ids in the model map were wrong: openai.gpt-oss-120b lacks the version suffix Bedrock requires, and claude-sonnet-5 does not exist on this account. A surface with no configured target is now closed, and the screen rejects repos whose SDK needs it instead of spending an agent budget on a port that cannot reach a model. Reopening it is one line of repos.yaml. --- skill_harness/DESIGN.md | 35 ++++++++++++++++++++++++++++++++--- skill_harness/__main__.py | 13 ++++++++++--- skill_harness/repos.yaml | 29 ++++++++++++++++++----------- skill_harness/screen.py | 14 +++++++++++++- skill_harness/stages.py | 5 ++++- 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md index e91be37..db6b9ae 100644 --- a/skill_harness/DESIGN.md +++ b/skill_harness/DESIGN.md @@ -123,6 +123,32 @@ Messages API, all Anthropic Claude. No Claude model serves Chat Completions, and Meta / Amazon / Cohere / AI21 serve neither. A `ChatOpenAI` repo therefore lands on a gpt-oss / Qwen / Mistral class model, never on Claude. +**What the credential can reach is narrower still, and is an account property +rather than a property of Bedrock.** Measured on 2026-08-28 against the key in +use: + +| Surface | Result | +|---|---| +| OpenAI Chat Completions | works — `openai.gpt-oss-120b-1:0`, and gpt-oss-20b, qwen3-32b, mistral-large-3, deepseek-v3.2 all answer | +| Anthropic Messages | closed — every Messages-capable Claude answers `permission_error` | + +The control plane lists 121 models, which is what the platform offers and not +what the account may call: a model can appear there and still be refused. Claude 3 +Haiku gives the reason — *"Model use case details have not been submitted for this +account"* — so this is an entitlement, reopened by submitting the Anthropic use +case form rather than by any change here. + +Claude is reachable on this account through **Converse**, which was confirmed. It +is not a way around the closed surface: Converse is a third wire format, so +routing an Anthropic SDK call to it means the protocol translation this design +exists to avoid. + +The consequence is a scope limit that must be stated with any result from this +run: **repos using the Anthropic SDK are rejected at stage 2, not tested.** The +harness expresses this as data rather than in code — a surface whose entry in +`repos.yaml` is empty is a surface the screen refuses to route to — so the day +the entitlement lands, one line of configuration brings those repos back. + **The model id is the one thing an env var cannot reach.** A repo writes `ChatOpenAI(model="gpt-4o-mini")`; the id travels in the request body, and Bedrock rejects it. The fix is a shim in front of Bedrock that **rewrites the `model` field @@ -138,9 +164,12 @@ Credentials reach the containers through `env_file:` (PR #53, merged into this branch), which is the only sanctioned path — M18 forbids baking a key into the build context. -**Prerequisite:** no AWS credential exists on the target machine today -(`~/.aws/` holds no credentials file; neither `AWS_BEARER_TOKEN_BEDROCK` nor -`AWS_ACCESS_KEY_ID` is set). Stage 3 cannot run until a Bedrock API key exists. +**Credential shape.** The key in use is a short-term Bedrock bearer token: an +`ASIA...` STS credential scoped to one region, valid 12 hours. That is ample for +proving the pipeline on two repos and too short for a hundred, so a run at full +size needs either a long-term key or a refresh step. The harness reads the token +from `AWS_BEARER_TOKEN_BEDROCK` on each `wire`, so a refreshed token is picked up +by repos that have not started yet, but not by containers already running. ## 4. Storage diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py index 67e26bb..29a0568 100644 --- a/skill_harness/__main__.py +++ b/skill_harness/__main__.py @@ -31,10 +31,11 @@ def _load_repos(path: Path) -> list[str]: def _model_map(path: Path) -> shim.ModelMap: doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} m = doc.get("models", {}) + defaults = {k: v for k, v in (m.get("defaults") or {}).items() if v} return shim.ModelMap( exact=m.get("exact", {}), prefixes=[(p["prefix"], p["to"]) for p in m.get("prefixes", [])], - defaults=m["defaults"], + defaults=defaults, ) @@ -54,8 +55,13 @@ def cmd_run(args: argparse.Namespace) -> int: print("AWS_BEARER_TOKEN_BEDROCK is not set; stage 3 cannot run.", file=sys.stderr) return 2 - shim.start(region=args.region, key=key, model_map=_model_map(repos_file), - port=args.shim_port) + model_map = _model_map(repos_file) + surfaces = frozenset(model_map.defaults) + if not surfaces: + print("no usable surface: every models.defaults entry is empty", file=sys.stderr) + return 2 + logging.info("reachable surfaces: %s", ", ".join(sorted(surfaces))) + shim.start(region=args.region, key=key, model_map=model_map, port=args.shim_port) cfg = Config( harness_root=HARNESS_ROOT, @@ -70,6 +76,7 @@ def cmd_run(args: argparse.Namespace) -> int: skill_sha=runner._tree_sha(HARNESS_ROOT, ".claude/skills/porting-to-ventis"), ventis_sha=runner._tree_sha(HARNESS_ROOT, "ventis"), disallowed_tools=args.disallowed_tools, + surfaces=surfaces, ) logging.info("skill %s | core %s | model %s/%s", cfg.skill_sha[:12], cfg.ventis_sha[:12], cfg.model, cfg.effort) diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml index a99fdcc..0e790d6 100644 --- a/skill_harness/repos.yaml +++ b/skill_harness/repos.yaml @@ -12,24 +12,31 @@ repos: # so the shim rewrites the id on the way past. Anything not matched here falls to # the per-surface default. # -# Note the asymmetry, which is a property of Bedrock and not a choice: no Claude -# model serves Chat Completions, so an OpenAI-SDK repo lands on a gpt-oss class -# model and never on Claude. See DESIGN.md section 3. +# An empty default closes that surface: the screen then rejects any repo whose +# SDK would need it, instead of letting an agent spend its budget porting a repo +# that cannot reach a model. models: defaults: - openai: openai.gpt-oss-120b - anthropic: us.anthropic.claude-sonnet-5 + # Measured against this credential on 2026-08-28, not assumed. + openai: openai.gpt-oss-120b-1:0 + + # Closed. Every Messages-API-capable Claude on this account answers + # `permission_error`, and Claude 3 Haiku says why: "Model use case details + # have not been submitted for this account." Claude *is* reachable here, but + # only through Converse — a different wire format, so routing an Anthropic + # SDK call to it would need the protocol translation this design avoids. + # Submitting the Anthropic use case form is what reopens this; then put + # a model id back here and anthropic-SDK repos come back into scope. + anthropic: "" exact: {} prefixes: - prefix: gpt- - to: openai.gpt-oss-120b + to: openai.gpt-oss-120b-1:0 - prefix: o1 - to: openai.gpt-oss-120b + to: openai.gpt-oss-120b-1:0 - prefix: o3 - to: openai.gpt-oss-120b + to: openai.gpt-oss-120b-1:0 - prefix: o4 - to: openai.gpt-oss-120b - - prefix: claude- - to: us.anthropic.claude-sonnet-5 + to: openai.gpt-oss-120b-1:0 diff --git a/skill_harness/screen.py b/skill_harness/screen.py index cadc905..614e4cd 100644 --- a/skill_harness/screen.py +++ b/skill_harness/screen.py @@ -73,7 +73,15 @@ def _matches(imports: set[str], markers: tuple[str, ...]) -> bool: return any(i == m or i.startswith(m + ".") for i in imports for m in markers) -def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000) -> Screen: +def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, + surfaces: frozenset[str] = frozenset({"openai", "anthropic"})) -> Screen: + """`surfaces` is which Bedrock wire formats the credential can actually reach. + + It is an account property, not a property of Bedrock: a Messages-API model + can be listed by the control plane and still answer `permission_error`. A + repo whose SDK needs a closed surface is rejected here rather than after an + agent has spent its budget porting it. + """ out = Screen() imports: set[str] = set() models: set[str] = set() @@ -132,6 +140,10 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000) -> Screen # Both signals absent. One alone is not enough to reject on: a wrapper # hides the SDK, and a model id read from config leaves no literal. out.reject = "no LLM call found" + elif out.llm_sdk in ("openai", "anthropic") and out.llm_sdk not in surfaces: + out.reject = f"{out.llm_sdk} surface unavailable on this credential" + elif out.llm_sdk == "both" and not {"openai", "anthropic"} <= surfaces: + out.reject = "needs both surfaces; only " + ",".join(sorted(surfaces)) elif out.layout == "src" and out.packaging == "none": # M24: without packaging metadata there is no editable install, and the # Ventis change that would make a src/ layout importable has no PR. The diff --git a/skill_harness/stages.py b/skill_harness/stages.py index 74164c9..9c4746e 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -83,6 +83,9 @@ class Config: skill_sha: str = "unknown" ventis_sha: str = "unknown" disallowed_tools: str = "" + # Which Bedrock wire formats this credential can actually reach. An account + # property, measured rather than assumed — see README. + surfaces: frozenset = frozenset({"openai"}) # --------------------------------------------------------------------------- # @@ -153,7 +156,7 @@ def fetch(ctx: Ctx) -> Result: # --------------------------------------------------------------------------- # def screen(ctx: Ctx) -> Result: - ctx.screen = screen_mod.screen(ctx.root) + ctx.screen = screen_mod.screen(ctx.root, surfaces=ctx.cfg.surfaces) ctx.log_path("2-screen.json").write_text( json.dumps(ctx.screen.__dict__, indent=2, default=str), encoding="utf-8" ) From 459f7029417f810d02392cfa868c687531094761 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:22:00 -0700 Subject: [PATCH 17/43] skill_harness: give subprocesses an interpreter, and stop scoring the skill working as the skill failing Two faults the first real run exposed. The harness never put its own venv on PATH. The ventis CLI was simply not found, so stage 6 would have exited 127 and recorded a harness setup fault as a defect in the port; and the agent in stage 4, having no interpreter that could import ventis, went three directories up and out of the tree under test to find one. Every subprocess now inherits the harness interpreter's bin, and validate.py runs on sys.executable -- on any other interpreter its capability probe reports every capability absent. Stage 4 also scored report-and-stop as a port failure. The skill's report-rather- than-fix paths are all triggered by something Ventis cannot do, so an agent that takes one has followed the skill exactly. Those runs are now status=blocked with the report filed as a core issue, which is where a finding with a Ventis owner belongs; a run that writes neither a port nor a report is still a failure. --- skill_harness/runner.py | 5 ++++ skill_harness/screen.py | 12 ++++++--- skill_harness/stages.py | 54 +++++++++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/skill_harness/runner.py b/skill_harness/runner.py index 903090a..99346f1 100644 --- a/skill_harness/runner.py +++ b/skill_harness/runner.py @@ -63,6 +63,11 @@ def _classify(stage: str, result: Result, ctx: Ctx) -> str: if stage == "wired": return "blocked" # missing credential, nothing was tested if stage == "ported": + if ctx.reported_and_stopped: + # The skill told the agent to report rather than fix, and it did. + # Scoring this as a failure would count the skill working as the + # skill failing, and would bury the Ventis gap that caused it. + return "blocked" trace = ctx.log_path("4-port.log") text = trace.read_text(encoding="utf-8", errors="replace") if trace.is_file() else "" if "budget" in text.lower() and "exceed" in text.lower(): diff --git a/skill_harness/screen.py b/skill_harness/screen.py index 614e4cd..9c9ee73 100644 --- a/skill_harness/screen.py +++ b/skill_harness/screen.py @@ -34,12 +34,16 @@ "cohere", "mistralai", "ollama", "langchain.chat_models")), ] -# Model ids as they appear in source. Deliberately broad: a literal this matches -# is a candidate for the shim's mapping table, and a human reads the list before -# the run. Missing one is a stage 8 provider error; over-matching costs nothing. +# Model ids as they appear in source. A literal this matches is a candidate for +# the shim's mapping table, which a human reads before the run — so a miss costs +# a stage 8 provider error, and a false match costs that human's attention. MODEL_LITERAL = re.compile( r"\b(gpt-[\w.\-]+|o[134](?:-[\w.\-]+)?|claude-[\w.\-]+|" - r"(?:meta|mistral|amazon|cohere|anthropic|openai|qwen|deepseek)\.[\w.\-:]+)\b" + # A vendor-prefixed Bedrock id ends in a version marker (`-v1:0`, `-1:0`, + # `-2507`). Requiring one keeps `meta.com` and other hostnames out; without + # it the list a human reads to build the model map fills with domains. + r"(?:meta|mistral|amazon|cohere|anthropic|openai|qwen|deepseek)" + r"\.[\w.\-]*(?:v?\d+(?::\d+)?|\d{4}))\b" ) MULTIAGENT_MARKERS = ("Send(", "StateGraph", "Crew(", "GroupChat", "add_edge", "Command(") diff --git a/skill_harness/stages.py b/skill_harness/stages.py index 9c4746e..cee3d9b 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -17,6 +17,7 @@ import shutil import signal import subprocess +import sys import time import urllib.error import urllib.request @@ -60,6 +61,11 @@ class Ctx: validate_ok: bool | None = None core_issue: list = field(default_factory=list) skill_issue: list = field(default_factory=list) + # The skill's "report rather than fix" paths -- the credential wall, the + # import root, a dependency mismatch -- are all Ventis limitations. An agent + # that takes one has followed the skill, so this is not a port failure and + # must not be scored as one. + reported_and_stopped: bool = False _procs: list = field(default_factory=list) def log_path(self, name: str) -> Path: @@ -92,11 +98,28 @@ class Config: # helpers # --------------------------------------------------------------------------- # +# The harness runs under its own virtualenv, but launching it does not put that +# venv's bin on PATH. Without this the `ventis` CLI is simply not found -- stage 6 +# exits 127 and the run records a harness setup fault as a defect in the port -- +# and the agent in stage 4 has no interpreter that can import ventis, so it goes +# looking for one outside the tree under test. +_BIN = str(Path(sys.executable).parent) + + +def subprocess_env(extra: dict | None = None) -> dict: + env = {**os.environ, **(extra or {})} + path = env.get("PATH", "") + if _BIN not in path.split(os.pathsep): + env["PATH"] = _BIN + os.pathsep + path + env.setdefault("VIRTUAL_ENV", str(Path(_BIN).parent)) + return env + + def run(ctx: Ctx, name: str, cmd: list[str], *, cwd: Path | None = None, timeout: int | None = None, env: dict | None = None) -> tuple[int, str]: """Run a subprocess, tee its output into the artifacts directory, return it.""" timeout = timeout or ctx.cfg.stage_timeout - full_env = {**os.environ, **(env or {})} + full_env = subprocess_env(env) log.debug("%s: %s", name, " ".join(cmd)) try: proc = subprocess.run( @@ -233,24 +256,39 @@ def port(ctx: Ctx) -> Result: report = ctx.root / "PORT_REPORT.md" if report.is_file(): - ctx.skill_issue.append({"kind": "reported_and_stopped", - "text": report.read_text(encoding="utf-8")[:2000]}) + # Filed as a core issue: every "report and stop" the skill defines is + # triggered by something Ventis cannot do, so it is a finding with a + # Ventis owner. The full text stays in artifacts; this is the grouping key. + ctx.reported_and_stopped = True + ctx.core_issue.append({"kind": "reported_and_stopped", + "text": report.read_text(encoding="utf-8")[:4000]}) if rc != 0: return Result(False, f"claude exited {rc}") - missing = [p for p in ("config/global_controller.yaml",) if not (ctx.root / p).is_file()] - if missing: - return Result(False, f"port produced no {missing[0]}") + + if not _config_path(ctx.root).is_file(): + if ctx.reported_and_stopped: + return Result(False, "reported and stopped: " + _report_headline(report)) + return Result(False, "no port written, and no report explaining why") return Result(True, "port written") +def _report_headline(report: Path) -> str: + """The report's first non-empty, non-heading line, for the run log.""" + for line in report.read_text(encoding="utf-8", errors="replace").splitlines(): + line = line.strip().lstrip("#").strip() + if line and not line.startswith(("**Date", "**Skill", "---")): + return line[:160] + return "(no summary line)" + + # --------------------------------------------------------------------------- # # 5. validated — records a verdict, does not gate # --------------------------------------------------------------------------- # def validate(ctx: Ctx) -> Result: script = ctx.cfg.harness_root / SKILL_REL / "validate.py" - rc, out = run(ctx, "5-validate", ["python3", str(script), ".", "--json"]) + rc, out = run(ctx, "5-validate", [sys.executable, str(script), ".", "--json"]) ctx.validate_ok = rc == 0 if rc == 127: ctx.validate_ok = None @@ -321,7 +359,7 @@ def deploy(ctx: Ctx) -> Result: proc = subprocess.Popen( ["ventis", "deploy", "-c", "config/global_controller.yaml"], cwd=ctx.root, stdout=logfile, stderr=subprocess.STDOUT, text=True, - start_new_session=True, + start_new_session=True, env=subprocess_env(), ) ctx._procs.append(proc) From f89c12ff4d62580db28aa85b4c8104cccd2f3ffe Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:25:44 -0700 Subject: [PATCH 18/43] skill_harness: make the layout check mean what M24 means, and add a screen-only command The screen called a repo flat when it merely had no src/ directory, so langchain-academy -- whose 17 modules all sit under module-N/studio/ -- was passed to an agent that spent four minutes rediscovering statically that no port of it can be loaded. Flat now means what the port needs it to mean: a module the adapter can import from the project root. The threshold follows the Ventis under test rather than being fixed. Without an editable install M24 holds strictly and only root-flat modules import, whatever the packaging says; with one, packaging metadata decides. The harness asks the code, the way validate.py does. The new screen subcommand clones and screens candidates without porting any of them, which is how the repo list gets assembled -- a stage 2 verdict costs a shallow clone and no agent budget. --- skill_harness/__main__.py | 42 ++++++++++++++++++++++++++++++++ skill_harness/screen.py | 50 +++++++++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py index 29a0568..5334379 100644 --- a/skill_harness/__main__.py +++ b/skill_harness/__main__.py @@ -89,6 +89,43 @@ def cmd_run(args: argparse.Namespace) -> int: return 0 +def cmd_screen(args: argparse.Namespace) -> int: + """Clone and screen candidates without porting anything. + + Stage 2 is a static read, so answering "is this repo in scope" costs a + shallow clone and no agent budget. This is how the repo list gets built. + """ + import shutil + import subprocess + import tempfile + + from .screen import editable_install_available, screen as do_screen + + repos = _load_repos(Path(args.repos).resolve()) if args.repos else [] + repos += args.repo + surfaces = frozenset(_model_map(Path(args.repos).resolve()).defaults) if args.repos \ + else frozenset({"openai"}) + editable = editable_install_available() + print(f"surfaces={sorted(surfaces)} editable_install={editable}\n") + + tmp = Path(tempfile.mkdtemp(prefix="screen-")) + try: + for repo in repos: + dest = tmp / runner.slug_for(repo) + r = subprocess.run(["git", "clone", "-q", "--depth", "1", repo, str(dest)], + capture_output=True, text=True, timeout=300) + if r.returncode != 0: + print(f"{repo:<62} CLONE FAILED {r.stderr.strip()[-80:]}") + continue + s = do_screen(dest, surfaces=surfaces, editable_install=editable) + verdict = "IN SCOPE" if not s.reject else s.reject + print(f"{repo:<62} root_py={s.root_py_files:<3} py={s.py_files:<4} " + f"loc={s.loc:<6} {s.framework}/{s.llm_sdk:<9} {verdict}") + finally: + shutil.rmtree(tmp, ignore_errors=True) + return 0 + + def cmd_report(args: argparse.Namespace) -> int: conn = db.connect(args.db) rows = db.summary(conn) @@ -126,6 +163,11 @@ def main(argv: list[str] | None = None) -> int: run_p.add_argument("--disallowed-tools", default="") run_p.set_defaults(func=cmd_run) + scr_p = sub.add_parser("screen", help="clone and screen candidates, port nothing") + scr_p.add_argument("--repos", default=None, help="yaml list to screen") + scr_p.add_argument("repo", nargs="*", help="extra repo urls") + scr_p.set_defaults(func=cmd_screen) + rep_p = sub.add_parser("report", help="print the results table") rep_p.set_defaults(func=cmd_report) diff --git a/skill_harness/screen.py b/skill_harness/screen.py index 9c9ee73..d75ead2 100644 --- a/skill_harness/screen.py +++ b/skill_harness/screen.py @@ -52,6 +52,7 @@ @dataclass class Screen: py_files: int = 0 + root_py_files: int = 0 loc: int = 0 framework: str = "plain" llm_sdk: str = "none" @@ -77,15 +78,35 @@ def _matches(imports: set[str], markers: tuple[str, ...]) -> bool: return any(i == m or i.startswith(m + ".") for i in imports for m in markers) +def editable_install_available() -> bool: + """Whether the Ventis under test can install the source as a package. + + Asked of the code rather than assumed, the same way validate.py asks it. + Without it, M24 holds in its strict form: only modules that land flat at + /app import at all, and packaging metadata rescues nothing. + """ + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a missing install must not crash the screen + return False + return hasattr(stub_generator, "_install_step") + + def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, - surfaces: frozenset[str] = frozenset({"openai", "anthropic"})) -> Screen: + surfaces: frozenset[str] = frozenset({"openai", "anthropic"}), + editable_install: bool | None = None) -> Screen: """`surfaces` is which Bedrock wire formats the credential can actually reach. It is an account property, not a property of Bedrock: a Messages-API model can be listed by the control plane and still answer `permission_error`. A repo whose SDK needs a closed surface is rejected here rather than after an agent has spent its budget porting it. + + `editable_install` is the matching question for M24, asked of the Ventis under + test rather than assumed. """ + if editable_install is None: + editable_install = editable_install_available() out = Screen() imports: set[str] = set() models: set[str] = set() @@ -94,6 +115,8 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, if SKIP_DIRS & set(path.relative_to(root).parts): continue out.py_files += 1 + if path.parent == root: + out.root_py_files += 1 try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: @@ -122,8 +145,15 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, elif hits: out.llm_sdk = hits[0] - if (root / "src").is_dir(): + # `flat` means the port can import the source, which is a fact about where + # modules sit relative to the project root -- not about whether a `src/` + # directory happens to exist. + if out.root_py_files: + out.layout = "flat" + elif (root / "src").is_dir(): out.layout = "src" + else: + out.layout = "nested" for candidate in ("pyproject.toml", "setup.py", "setup.cfg"): if (root / candidate).is_file(): out.packaging = candidate @@ -148,10 +178,16 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, out.reject = f"{out.llm_sdk} surface unavailable on this credential" elif out.llm_sdk == "both" and not {"openai", "anthropic"} <= surfaces: out.reject = "needs both surfaces; only " + ",".join(sorted(surfaces)) - elif out.layout == "src" and out.packaging == "none": - # M24: without packaging metadata there is no editable install, and the - # Ventis change that would make a src/ layout importable has no PR. The - # port cannot succeed, and that is a finding about Ventis, not the skill. - out.reject = "src/ layout with no packaging metadata (M24, no PR)" + elif out.root_py_files == 0 and not editable_install: + # M24 in its strict form. With no editable install, an adapter can import + # only what lands flat at /app, so a tree whose modules all sit under + # sub-directories has no port this Ventis can load -- whatever its + # packaging says. Deciding it here costs nothing; letting it through + # costs an agent's whole budget to reach the same conclusion. + out.reject = (f"no module at the project root ({out.py_files} .py files, " + f"all nested) and no editable install (M24)") + elif out.root_py_files == 0 and out.packaging == "none": + # The editable install exists, but nothing tells it what the root is. + out.reject = "no module at the project root and no packaging metadata (M24)" return out From f9dd2d93f4739b7f1f96b8dc1857d9f6d205766e Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:30:56 -0700 Subject: [PATCH 19/43] ventis build: install the project itself when it declares packaging metadata Sweeping the tree into the image does not make it importable. The process starts at the context root, so sys.path[0] is /app and only modules sitting there resolve -- a src/ layout resolves to nothing, and an adapter importing one raises ModuleNotFoundError inside _load_agent, which surfaces only as "No agent loaded" on the first request. `-e .` hands the import root to the project's own packaging metadata so Ventis never guesses a directory name. A project declaring none gets the previous install, unchanged. Taken from jiajunh/can-228-create-a-skill-to-convert-a-langchain-project-to-ventis. Merging that branch whole was rejected: it is an older parallel line carrying its own pre-validate.py copy of the skill and an earlier env_file than the one PR #53 put on this branch, so the merge conflicted on twelve files and would have regressed the artifact under test. Measured motivation: of six LangChain sample repositories screened, five are src/ layouts with pyproject.toml that no port could load without this. --- ventis/stub_generator.py | 47 +++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index e132003..b551101 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -293,6 +293,39 @@ def _sweep_py_files(project_dir): return swept +# What a project must have at its root for `pip install -e .` to mean anything. +_PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") + + +def _install_step(project_dir): + """The Dockerfile lines that install requirements, plus the project itself. + + Sweeping the tree in is not enough to make it importable: the process starts + at the context root, so sys.path[0] is /app and only modules sitting there + resolve -- a src/ layout resolves to nothing. `-e .` hands the import root to + the project's own packaging metadata, so Ventis never has to guess a + directory name. A project that declares no metadata gets the plain install. + """ + installable = project_dir and any( + os.path.isfile(os.path.join(project_dir, name)) for name in _PACKAGING_FILES + ) + if not installable: + return ( + "COPY requirements.txt .\n" + "RUN --mount=type=cache,target=/root/.cache/uv " + "uv pip install --system -r requirements.txt\n" + "\n" + "COPY . .\n" + ) + # The project has to be in the context before it can be installed, so the + # copy moves ahead of the install and both resolve in one pass. + return ( + "COPY . .\n" + "RUN --mount=type=cache,target=/root/.cache/uv " + "uv pip install --system -r requirements.txt -e .\n" + ) + + def _stub_destinations(stub_file, stub_entrypoints): """Every path a stub is copied to, flat name first. @@ -435,17 +468,14 @@ def generate_docker( # ---- Dockerfile ------------------------------------------------------ agent_basename = os.path.basename(agent_file) + install_step = _install_step(project_dir) dockerfile = f"""# syntax=docker/dockerfile:1 FROM python:3.11-slim COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app -COPY requirements.txt . -RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt - -COPY . . - +{install_step} ENV VENTIS_AGENT_NAME={agent_name} ENV VENTIS_AGENT_FILE={agent_basename} @@ -570,17 +600,14 @@ def start_lc(): f.write(launcher) # ---- Dockerfile ------------------------------------------------------ + install_step = _install_step(project_dir) dockerfile = f"""# syntax=docker/dockerfile:1 FROM python:3.11-slim COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app -COPY requirements.txt . -RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt - -COPY . . - +{install_step} EXPOSE 50051 EXPOSE {api_port} From 80eb546d1abfe78243e2500faf869802af27ee40 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:32:55 -0700 Subject: [PATCH 20/43] validate.py: probe the function stub_generator actually defines The capability probe asked for _sweep_project_files; the function is named _sweep_py_files. sweeps_all_files therefore reported absent on every tree that has it, including this branch, which merged feature/all-the-files. A false negative here is worse than no probe: the skill tells an agent to trust the probe over its own assumptions, so the agent reasons from a capability it has been told it lacks. The other three probes were checked against the installed package and are correct. --- .claude/skills/porting-to-ventis/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/porting-to-ventis/validate.py b/.claude/skills/porting-to-ventis/validate.py index 86646d1..4650d5e 100755 --- a/.claude/skills/porting-to-ventis/validate.py +++ b/.claude/skills/porting-to-ventis/validate.py @@ -145,7 +145,7 @@ def probe_capabilities(): caps["ventis"] = True caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_py_files") caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") import importlib From e4c98b0e917b2368f7fce3e10750b260d441fa40 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:33:05 -0700 Subject: [PATCH 21/43] skill_harness: the four repos that screen in scope, and why the others do not --- skill_harness/repos.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml index 0e790d6..acad837 100644 --- a/skill_harness/repos.yaml +++ b/skill_harness/repos.yaml @@ -4,9 +4,19 @@ # selection are deliberately out of scope until the failure modes are known — # see DESIGN.md section 5. +# Screened in scope on 2026-08-28 with `python -m skill_harness screen`. +# All four are src/ layouts with pyproject.toml, which only became portable +# once ventis build learned to install the project itself. repos: - - https://github.com/langchain-ai/langchain-academy - - https://github.com/langchain-ai/rag-from-scratch + - https://github.com/langchain-ai/react-agent + - https://github.com/langchain-ai/memory-agent + - https://github.com/langchain-ai/retrieval-agent-template + - https://github.com/langchain-ai/data-enrichment + +# Screened and rejected, kept here so the reason is not rediscovered: +# langchain-academy no packaging metadata and no root module (M24) +# rag-from-scratch notebooks only, no .py at all +# social-media-agent anthropic SDK, and that surface is closed on this key # A repo's model id is hardcoded in its source, and the source is never edited, # so the shim rewrites the id on the way past. Anything not matched here falls to From c85bff66542a6866c4c0e2d00705fa8ce69f0cb9 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:34:09 -0700 Subject: [PATCH 22/43] docs: record what the corpus cost, and why can-228 was ported rather than merged --- skill_harness/DESIGN.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md index db6b9ae..6b70d9c 100644 --- a/skill_harness/DESIGN.md +++ b/skill_harness/DESIGN.md @@ -223,6 +223,33 @@ derivable from `artifacts/` by a script, at any time, without re-running anythin and running the pipeline is the expensive part. Write those scripts when there is a corpus worth aggregating and it is clear what to aggregate. +## 4a. What the corpus turned out to cost + +The first screening run answered a question this design had filed as a +by-product. Of six LangChain sample repositories, **none were in scope**: five +are `src/` layouts with `pyproject.toml`, and `ventis build` could not make such +a tree importable, so no port of them could load. `src/` plus packaging metadata +is not a quirk of those five — it is the shape LangChain's own templates ship. + +That made the corpus, not the harness, the binding constraint on CAN-238: a +hundred-repo run against a Ventis without an editable install would have produced +close to a hundred stage 2 rejections and tested almost nothing. + +`_install_step` — a Dockerfile step that runs `pip install -e .` when the project +declares packaging metadata, handing the import root to the project rather than +making Ventis guess a directory — was ported onto this branch from +`jiajunh/can-228-create-a-skill-to-convert-a-langchain-project-to-ventis`. Four of +the six came into scope immediately. + +**Ported rather than merged, deliberately.** That branch is an older parallel +line: it carries its own copy of the skill from before `validate.py` existed, its +own earlier `env_file.py`, and its own `joke_writer`. Merging it whole conflicted +on twelve files, three of them the skill — it would have regressed the artifact +under test in the act of enabling the test. + +The M24 rejection is still real for repos that declare no packaging metadata at +all, and `langchain-academy` remains rejected for exactly that reason. + ## 5. Scope of the first version Stages 1–8 straight through, concurrency fixed at 2, repo list supplied by hand — @@ -256,3 +283,8 @@ introspection of stage 4. **Restricting the run to repos already on Bedrock.** Rejected: too few exist to reach 100, and selecting for them would bias the sample toward projects that never exercise the credential wall the skill has the most to say about. + +**Merging `can-228` whole to obtain the editable install.** Rejected for the +reason in section 4a: the branch carries an older copy of the artifact under +test, so the merge would have changed what the run measures. The one capability +was ported instead, and its provenance recorded in the commit. From db97b60048232690873572bfe8797323316e9a63 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:39:06 -0700 Subject: [PATCH 23/43] skill_harness: read the provider a repo picks at runtime, not just its imports Every LangGraph template reaches its model through init_chat_model with a "/" string, so a repo can depend entirely on Anthropic while importing nothing named anthropic. The screen read only imports and classified three such repos as having no redirectable provider, then passed them to agents that spent budget porting projects whose provider surface is closed. It now reads the provider strings and classifies on what will actually be resolved. A closed surface also raised KeyError inside the shim's request handler, which killed the connection with no reply. It now answers 503 saying which surface is closed, and logs a warning -- a repo reaching a closed surface is evidence the screen let something through, and that is worth seeing rather than swallowing. .claude is now skipped when reading a tree: stage 4 copies the skill under test into the repo, so re-screening a tree that has been through a run would read validate.py's own imports as the repo's. --- skill_harness/screen.py | 31 ++++++++++++++++++++++++++++++- skill_harness/shim.py | 23 ++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/skill_harness/screen.py b/skill_harness/screen.py index d75ead2..f338a43 100644 --- a/skill_harness/screen.py +++ b/skill_harness/screen.py @@ -12,7 +12,11 @@ from dataclasses import dataclass, field from pathlib import Path -SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox", "build", "dist"} +# `.claude` is here because the harness copies the skill under test into the repo +# at stage 4. Screening a tree that has already been through a run would +# otherwise read validate.py's own imports as the repo's. +SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox", + "build", "dist", ".claude"} FRAMEWORK_MARKERS = [ ("langgraph", ("langgraph",)), @@ -46,6 +50,16 @@ r"\.[\w.\-]*(?:v?\d+(?::\d+)?|\d{4}))\b" ) +# LangChain's `init_chat_model` picks its provider at runtime from a +# "/" string, so a repo can depend entirely on Anthropic while +# importing nothing named anthropic. Every LangGraph template is built this way +# and most default to Claude -- reading only the imports classifies them as +# having no provider at all. +PROVIDER_STRING = re.compile( + r"[\"']((?:anthropic|openai|google_genai|google_vertexai|bedrock|bedrock_converse|" + r"cohere|mistralai|fireworks|groq|ollama|together|deepseek|xai)[:/][\w.\-:]+)[\"']" +) + MULTIAGENT_MARKERS = ("Send(", "StateGraph", "Crew(", "GroupChat", "add_edge", "Command(") @@ -58,6 +72,8 @@ class Screen: llm_sdk: str = "none" model_ids: list[str] = field(default_factory=list) is_multiagent: bool = False + # "/" literals -- what init_chat_model resolves at runtime. + provider_hints: list[str] = field(default_factory=list) layout: str = "flat" packaging: str = "none" description: str = "" @@ -110,6 +126,7 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, out = Screen() imports: set[str] = set() models: set[str] = set() + hints: set[str] = set() for path in root.rglob("*.py"): if SKIP_DIRS & set(path.relative_to(root).parts): @@ -123,6 +140,7 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, continue out.loc += text.count("\n") models.update(MODEL_LITERAL.findall(text)) + hints.update(PROVIDER_STRING.findall(text)) if any(m in text for m in MULTIAGENT_MARKERS): out.is_multiagent = True try: @@ -133,6 +151,7 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, continue out.model_ids = sorted(models) + out.provider_hints = sorted(hints) for name, markers in FRAMEWORK_MARKERS: if _matches(imports, markers): @@ -145,6 +164,16 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, elif hits: out.llm_sdk = hits[0] + # The imports did not name a provider we can redirect, but a runtime provider + # string may. Trust it: it is what init_chat_model will actually resolve. + if out.llm_sdk in ("none", "other"): + named = {h.split("/")[0].split(":")[0] for h in out.provider_hints} + named &= {"openai", "anthropic"} + if len(named) == 2: + out.llm_sdk = "both" + elif named: + out.llm_sdk = named.pop() + # `flat` means the port can import the source, which is a fact about where # modules sit relative to the project root -- not about whether a `src/` # directory happens to exist. diff --git a/skill_harness/shim.py b/skill_harness/shim.py index 9f83d4e..ea0e176 100644 --- a/skill_harness/shim.py +++ b/skill_harness/shim.py @@ -61,14 +61,21 @@ def __init__(self, exact: dict[str, str], prefixes: list[tuple[str, str]], self.defaults = defaults self.seen: dict[str, str] = {} - def resolve(self, model: str, surface: str) -> str: + def resolve(self, model: str, surface: str) -> str | None: + """None means the surface is closed -- there is no model to route to. + + A closed surface must answer legibly rather than raise: a repo that + reaches one has been mis-screened, and the reply is the evidence of it. + """ if model in self.exact: target = self.exact[model] else: target = next( (dst for pre, dst in self.prefixes if model.startswith(pre)), - self.defaults[surface], + self.defaults.get(surface), ) + if target is None: + return None if self.seen.get(model) != target: self.seen[model] = target log.info("model map: %s -> %s (%s)", model, target, surface) @@ -120,7 +127,17 @@ def _proxy(self, body: bytes): except json.JSONDecodeError: payload = None if isinstance(payload, dict) and "model" in payload: - payload["model"] = model_map.resolve(payload["model"], surface) + target = model_map.resolve(payload["model"], surface) + if target is None: + # Worth a warning, not just a reply: reaching a closed + # surface means stage 2 let a repo through that it should + # have rejected, and that is a screen defect to fix. + log.warning("%s reached the closed %s surface asking for %r", + slug, surface, payload["model"]) + return self._fail(503, f"the {surface} surface is closed on this " + f"credential; no model to route " + f"{payload['model']!r} to") + payload["model"] = target # Bedrock buffers; the repo may have asked to stream. payload.pop("stream", None) body = json.dumps(payload).encode() From d4e3d01d306db6cb34a5dec32c63a8bef02b19d8 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:43:23 -0700 Subject: [PATCH 24/43] skill_harness: do not outlive the harness Killing the harness orphaned its agents. Two of them survived a run being aborted, kept spending their budget for another eight minutes, and were still calling the shim of the run that started afterwards -- their calls showed up as a repo reaching a surface it had been screened out of, which is a confusing lie to leave in a log. Children now start in their own process group and are tracked, a timeout kills the group rather than the one process the harness holds, and SIGINT/SIGTERM/ SIGHUP plus atexit take every live child down first. --- skill_harness/__main__.py | 3 +- skill_harness/stages.py | 60 +++++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py index 5334379..1b009a6 100644 --- a/skill_harness/__main__.py +++ b/skill_harness/__main__.py @@ -16,7 +16,7 @@ import yaml from . import db, runner, shim -from .stages import Config +from .stages import Config, install_signal_handlers HARNESS_ROOT = Path(__file__).resolve().parent.parent DEFAULT_WORK = HARNESS_ROOT / ".harness" @@ -55,6 +55,7 @@ def cmd_run(args: argparse.Namespace) -> int: print("AWS_BEARER_TOKEN_BEDROCK is not set; stage 3 cannot run.", file=sys.stderr) return 2 + install_signal_handlers() model_map = _model_map(repos_file) surfaces = frozenset(model_map.defaults) if not surfaces: diff --git a/skill_harness/stages.py b/skill_harness/stages.py index cee3d9b..45e0334 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -11,6 +11,7 @@ from __future__ import annotations +import atexit import json import logging import os @@ -18,6 +19,7 @@ import signal import subprocess import sys +import threading import time import urllib.error import urllib.request @@ -115,22 +117,68 @@ def subprocess_env(extra: dict | None = None) -> dict: return env +# Every child the harness has started, so none of them outlives it. A killed +# harness used to orphan its agents: they kept running, kept spending their +# budget, and kept calling the shim of whatever run started next. +_LIVE: set[subprocess.Popen] = set() +_LIVE_LOCK = threading.Lock() + + +def kill_children(sig=signal.SIGTERM) -> int: + with _LIVE_LOCK: + procs = [p for p in _LIVE if p.poll() is None] + for proc in procs: + try: + os.killpg(os.getpgid(proc.pid), sig) + except Exception: # noqa: BLE001 - already gone, or not ours any more + pass + return len(procs) + + +def install_signal_handlers() -> None: + """Take the children down with us, however we are asked to stop.""" + def _handler(signum, _frame): + n = kill_children(signal.SIGTERM) + log.warning("signal %s: terminated %d child process(es)", signum, n) + raise SystemExit(128 + signum) + + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + try: + signal.signal(sig, _handler) + except ValueError: # not on the main thread + pass + atexit.register(kill_children) + + def run(ctx: Ctx, name: str, cmd: list[str], *, cwd: Path | None = None, timeout: int | None = None, env: dict | None = None) -> tuple[int, str]: """Run a subprocess, tee its output into the artifacts directory, return it.""" timeout = timeout or ctx.cfg.stage_timeout full_env = subprocess_env(env) log.debug("%s: %s", name, " ".join(cmd)) + proc = None try: - proc = subprocess.run( - cmd, cwd=cwd or ctx.root, env=full_env, timeout=timeout, + # Its own process group, so a timeout or a signal reaches the whole tree + # rather than just the process the harness happens to hold. + proc = subprocess.Popen( + cmd, cwd=cwd or ctx.root, env=full_env, start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) - rc, out = proc.returncode, proc.stdout - except subprocess.TimeoutExpired as e: - rc, out = 124, (e.output or "") + f"\n[harness] timed out after {timeout}s\n" + with _LIVE_LOCK: + _LIVE.add(proc) + out, _ = proc.communicate(timeout=timeout) + rc = proc.returncode + except subprocess.TimeoutExpired: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + out, _ = proc.communicate() + rc = 124 + out = (out or "") + f"\n[harness] timed out after {timeout}s\n" except FileNotFoundError as e: rc, out = 127, f"[harness] {e}\n" + finally: + if proc is not None: + with _LIVE_LOCK: + _LIVE.discard(proc) ctx.log_path(f"{name}.log").write_text(out or "", encoding="utf-8") return rc, out or "" @@ -362,6 +410,8 @@ def deploy(ctx: Ctx) -> Result: start_new_session=True, env=subprocess_env(), ) ctx._procs.append(proc) + with _LIVE_LOCK: + _LIVE.add(proc) port_no = _api_port(ctx.root) deadline = time.time() + ctx.cfg.stage_timeout From d78a5b96a47294a518373b6703339f7828b7f1e3 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:46:26 -0700 Subject: [PATCH 25/43] Revert "validate.py: probe the function stub_generator actually defines" This reverts commit 80eb546d1abfe78243e2500faf869802af27ee40. --- .claude/skills/porting-to-ventis/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/porting-to-ventis/validate.py b/.claude/skills/porting-to-ventis/validate.py index 4650d5e..86646d1 100755 --- a/.claude/skills/porting-to-ventis/validate.py +++ b/.claude/skills/porting-to-ventis/validate.py @@ -145,7 +145,7 @@ def probe_capabilities(): caps["ventis"] = True caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_py_files") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") import importlib From 3d3f1acdb4182db6f66ec5e813e7c371d8ab25e8 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:47:56 -0700 Subject: [PATCH 26/43] ventis build: sweep the whole project, not only its modules The editable install added in f9dd2d9 could never have worked. It emits `uv pip install -e .`, but the build context is docker_container// and the sweep put only .py files there, so /app held no pyproject.toml and every build died with "does not appear to be a Python project". _install_step and _sweep_project_files are one change on can-228 and only the first half was taken. The sweep now carries every project file, because packaging metadata routinely points at a README or a license and a .py-only sweep leaves nothing installable. Hidden files stay out: .env holds credentials and has no business in an image. This widens what PR #51's narrower _sweep_py_files copied. It also restores validate.py's original probe -- `_sweep_project_files` was that branch's name for the broader capability, not a typo, and d78a5b9 reverted the rename that made the probe report a capability this branch did not have. Verified on retrieval-agent-template: build exits 0, the context carries pyproject.toml and no .env, and both of SKILL.md Step 4's probes pass -- the runtime imports and the agent constructs. --- ventis/stub_generator.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index b551101..83e417b 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -272,11 +272,22 @@ def _format_source(source): # Directories ventis build itself generates inside a project -- never swept. -_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs", "__pycache__"} +# Written into the context by the generator itself; a project file of the same +# name at the root would land on top of it. +_GENERATED_ROOT_FILES = {"requirements.txt", "Dockerfile"} -def _sweep_py_files(project_dir): - """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + +def _sweep_project_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every project file, preserving its directory structure. + + Not only modules: the editable install below reads the project's packaging + metadata, and that metadata routinely points at a README or a license file, + so a sweep that took `.py` alone would leave nothing installable. Hidden + files are skipped -- `.env` holds credentials and has no business in an + image. + """ swept = [] for root, dirs, files in os.walk(project_dir): dirs[:] = [ @@ -284,12 +295,18 @@ def _sweep_py_files(project_dir): for d in dirs if not d.startswith(".") and not (root == project_dir and d in _GENERATED_DIRS) + and d != "__pycache__" ] + at_root = os.path.abspath(root) == os.path.abspath(project_dir) for fname in files: + if fname.startswith("."): + continue + if at_root and fname in _GENERATED_ROOT_FILES: + continue abs_src = os.path.join(root, fname) - if fname.endswith(".py") and not os.path.islink(abs_src): - rel_dst = os.path.relpath(abs_src, project_dir) - swept.append((abs_src, rel_dst)) + if os.path.islink(abs_src): + continue + swept.append((abs_src, os.path.relpath(abs_src, project_dir))) return swept @@ -420,7 +437,7 @@ def generate_docker( # Sweep the project for extra .py helper files not on the explicit list below. files_to_copy = [] if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy += _sweep_project_files(project_dir) # Copy general agent files files_to_copy += [ @@ -537,7 +554,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = _sweep_py_files(project_dir) if project_dir else [] + files_to_copy = _sweep_project_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), From 7dd92898ee6ff576c1f851f177905c2ec00f278e Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 17:51:56 -0700 Subject: [PATCH 27/43] skill_harness: talk to the providers directly, and stop rewriting anything Bedrock is gone from the path. Repos now keep their own provider and their own model ids and are given real keys, which satisfies M20 by construction rather than by a mapping table that had to be maintained and audited -- and it removes the model-id rewriting, the Bedrock id formats, and the surface entitlement that had closed Anthropic entirely. The shim stays, as a pass-through. It earns its place on two things a direct connection cannot give: per-repo token accounting for the results table, and one place that sees which models a repo actually calls. It swaps the key in so no real credential is written into a repo or baked into an image, and rewrites nothing else -- the rewrite rules are configurable and empty. A provider with no key is simply absent, so the screen rejects repos needing it at a shallow clone rather than after an agent has been paid to port them, and wire writes base URLs only for surfaces that can actually answer. Verified against the live OpenAI API: gpt-4o-mini passes through unchanged and is served as gpt-4o-mini, usage is attributed to the calling repo, and the unconfigured anthropic surface answers 503 rather than failing obscurely. --- skill_harness/DESIGN.md | 104 ++++++---------- skill_harness/__main__.py | 53 ++++---- skill_harness/repos.yaml | 49 +++----- skill_harness/shim.py | 251 ++++++++++++++++++++------------------ skill_harness/stages.py | 45 ++++--- 5 files changed, 242 insertions(+), 260 deletions(-) diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md index 6b70d9c..c7cf530 100644 --- a/skill_harness/DESIGN.md +++ b/skill_harness/DESIGN.md @@ -1,53 +1,19 @@ -# Testing `porting-to-ventis` across 100 repositories +# Testing and continually improve `porting-to-ventis` , its harness and core -CAN-238 design. Written 2026-08-27. - -## What this is for - -`.claude/skills/porting-to-ventis/` claims that an arbitrary agent project can be -moved onto Ventis by writing four files beside an untouched source tree. The claim -has been checked against one project (`examples/joke_writer`). This harness checks -it against a hundred, and produces per-repo evidence of where the claim broke. - -The output is not a pass rate on its own. It is a table of *how far each repo got* -and *what stopped it*, partitioned by whether the blame lies with the skill, with -Ventis, or with the repo. - -## The measurement problem, and what follows from it - -Two constraints shape everything below. Both come from the skill's own rules. - -**The source tree may not be edited.** M19 (`NEVER edit the source tree`) and M20 -(`NEVER swap the LLM provider the source uses`) are rules the skill is being -tested on. A harness that rewrites each repo's model calls onto Bedrock before -running the skill is not testing the skill — it is testing the rewrite, and every -downstream failure becomes unattributable. So the source tree is read-only for -the entire pipeline, and the LLM problem is solved outside it (§3). +## 1. Pipeline -**The skill may not change mid-run.** If the skill is edited between repo 1 and -repo 100, the two were not given the same test, the pass rate has no denominator, -and a fix that merely relocates a failure looks like a fix. So `tests.skill_sha` -is pinned on every row and the harness never writes to the skill. Fixes happen -between runs, as a new pinned version, with the affected repos re-run. Which -corner cases a new version closed is then a diff between two runs — which is what -CAN-237 wants anyway. -## 1. Pipeline +| # | Stage | What runs | Fails when | +| --- | ----------- | --------------------------------------------------------------------------- | ------------------------------------------------ | +| 1 | `fetched` | `git clone --depth 1`, record SHA | repo gone, too large, no license | +| 2 | `screened` | static scan: framework, LLM provider, hardcoded model ids, dependency shape | repo is out of scope for this run | +| 3 | `wired` | write `.env`, ensure model shim is up | no Bedrock credential, unmappable model | +| 4 | `ported` | `**claude -p**` running `porting-to-ventis` | agent gives up, budget exhausted, timeout | +| 5 | `validated` | `validate.py ` | contract violation the agent introduced | +| 6 | `built` | `ventis build` + both probes from SKILL.md Step 4 | image builds but container cannot import | +| 7 | `deployed` | `ventis deploy` | port/config/policy failure | +| 8 | `served` | `POST /main` → `GET /status/` | `"No agent loaded"`, provider error, wrong shape | -Eight stages. `tests.farthest_step` is the furthest one reached. Stage 5 is the -one exception: it does not gate what follows (see below), so its verdict is -recorded in `tests.validate_ok` rather than by halting the pipeline. - -| # | Stage | What runs | Fails when | -|---|-------|-----------|-----------| -| 1 | `fetched` | `git clone --depth 1`, record SHA | repo gone, too large, no license | -| 2 | `screened` | static scan: framework, LLM provider, hardcoded model ids, dependency shape | repo is out of scope for this run | -| 3 | `wired` | write `.env`, ensure model shim is up | no Bedrock credential, unmappable model | -| 4 | `ported` | **`claude -p`** running `porting-to-ventis` | agent gives up, budget exhausted, timeout | -| 5 | `validated` | `validate.py ` | contract violation the agent introduced | -| 6 | `built` | `ventis build` + both probes from SKILL.md Step 4 | image builds but container cannot import | -| 7 | `deployed` | `ventis deploy` | port/config/policy failure | -| 8 | `served` | `POST /main` → `GET /status/` | `"No agent loaded"`, provider error, wrong shape | **Only stage 4 uses an agent.** Everything else is a deterministic subprocess with a timeout. This is the property that makes failures attributable: a stage 6 failure @@ -84,21 +50,21 @@ claude -p "" \ Every flag above was checked against `claude --help` on the machine that will run it, not recalled. -- **`--bare` is not optional.** It suppresses hooks, auto-memory, plugin sync and - CLAUDE.md auto-discovery. Without it the operator's personal `~/.claude/CLAUDE.md` - and accumulated auto-memory enter all 100 runs, vary between them, and are - invisible in the results. Under `--bare` auth is strictly `ANTHROPIC_API_KEY`. -- **`--setting-sources ""`** keeps user/project/local settings out for the same - reason. -- **`--max-budget-usd`** is the containment mechanism; this CLI has no `--max-turns`. - A budget-exhausted run is recorded as its own failure mode, not as a crash. +- `**--bare` is not optional.** It suppresses hooks, auto-memory, plugin sync and +CLAUDE.md auto-discovery. Without it the operator's personal `~/.claude/CLAUDE.md` +and accumulated auto-memory enter all 100 runs, vary between them, and are +invisible in the results. Under `--bare` auth is strictly `ANTHROPIC_API_KEY`. +- `**--setting-sources ""**` keeps user/project/local settings out for the same +reason. +- `**--max-budget-usd**` is the containment mechanism; this CLI has no `--max-turns`. +A budget-exhausted run is recorded as its own failure mode, not as a crash. - **The skill is delivered explicitly**, by copying - `.claude/skills/porting-to-ventis/` into each repo working directory, so the - version under test is the version recorded — never whatever is globally installed. +`.claude/skills/porting-to-ventis/` into each repo working directory, so the +version under test is the version recorded — never whatever is globally installed. - **Tool restriction is unresolved and must be measured.** There are reports that - under `bypassPermissions`, `--allowedTools` is ignored and only `--disallowedTools` - constrains the tool set. This is verified on the first repo before the run scales; - it is not assumed in either direction. +under `bypassPermissions`, `--allowedTools` is ignored and only `--disallowedTools` +constrains the tool set. This is verified on the first repo before the run scales; +it is not assumed in either direction. `--output-format stream-json` is written into the run's `artifacts/` directory. The trace is the only record of *how* the agent reached its result, and it is what makes @@ -108,10 +74,12 @@ a skill defect diagnosable after the fact. Verified against AWS documentation on 2026-08-27: -| Source SDK | Base URL | Auth header | -|---|---|---| -| `openai` / `ChatOpenAI` | `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` | `Authorization: Bearer $AWS_BEARER_TOKEN_BEDROCK` | -| `anthropic` / `ChatAnthropic` | `https://bedrock-runtime.{region}.amazonaws.com/anthropic` | `x-api-key: $AWS_BEARER_TOKEN_BEDROCK` | + +| Source SDK | Base URL | Auth header | +| ----------------------------- | ---------------------------------------------------------- | ------------------------------------------------- | +| `openai` / `ChatOpenAI` | `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` | `Authorization: Bearer $AWS_BEARER_TOKEN_BEDROCK` | +| `anthropic` / `ChatAnthropic` | `https://bedrock-runtime.{region}.amazonaws.com/anthropic` | `x-api-key: $AWS_BEARER_TOKEN_BEDROCK` | + Both surfaces support client-side tool use. Both are reachable by environment variable alone, which is why the source tree never needs an edit. @@ -127,10 +95,12 @@ a gpt-oss / Qwen / Mistral class model, never on Claude. rather than a property of Bedrock.** Measured on 2026-08-28 against the key in use: -| Surface | Result | -|---|---| + +| Surface | Result | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | | OpenAI Chat Completions | works — `openai.gpt-oss-120b-1:0`, and gpt-oss-20b, qwen3-32b, mistral-large-3, deepseek-v3.2 all answer | -| Anthropic Messages | closed — every Messages-capable Claude answers `permission_error` | +| Anthropic Messages | closed — every Messages-capable Claude answers `permission_error` | + The control plane lists 121 models, which is what the platform offers and not what the account may call: a model can appear there and still be refused. Claude 3 @@ -287,4 +257,4 @@ exercise the credential wall the skill has the most to say about. **Merging `can-228` whole to obtain the editable install.** Rejected for the reason in section 4a: the branch carries an older copy of the artifact under test, so the merge would have changed what the run measures. The one capability -was ported instead, and its provenance recorded in the commit. +was ported instead, and its provenance recorded in the commit. \ No newline at end of file diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py index 1b009a6..e499c28 100644 --- a/skill_harness/__main__.py +++ b/skill_harness/__main__.py @@ -28,15 +28,28 @@ def _load_repos(path: Path) -> list[str]: return [r["repo"] if isinstance(r, dict) else r for r in doc.get("repos", [])] -def _model_map(path: Path) -> shim.ModelMap: - doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - m = doc.get("models", {}) - defaults = {k: v for k, v in (m.get("defaults") or {}).items() if v} - return shim.ModelMap( - exact=m.get("exact", {}), - prefixes=[(p["prefix"], p["to"]) for p in m.get("prefixes", [])], - defaults=defaults, - ) +def _dotenv(path: Path) -> dict[str, str]: + """Keys may live in the harness repo's own .env rather than the environment.""" + out: dict[str, str] = {} + if not path.is_file(): + return out + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, _, v = line.partition("=") + out[k.strip()] = v.strip().strip("\"'") + return out + + +def _providers(repos_file: Path) -> dict[str, shim.Provider]: + doc = yaml.safe_load(repos_file.read_text(encoding="utf-8")) or {} + config = doc.get("providers") or {} + ambient = {**_dotenv(HARNESS_ROOT / ".env"), **os.environ} + keys = { + name: ambient.get((entry or {}).get("key_env", ""), "") + for name, entry in config.items() + } + return shim.build_providers(config, keys) def cmd_run(args: argparse.Namespace) -> int: @@ -48,26 +61,19 @@ def cmd_run(args: argparse.Namespace) -> int: print(f"no repos listed in {repos_file}", file=sys.stderr) return 2 - key = os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "") - if not key: - # Stage 3 would report this per repo, but failing here says it once and - # avoids cloning a hundred repos to learn it. - print("AWS_BEARER_TOKEN_BEDROCK is not set; stage 3 cannot run.", file=sys.stderr) - return 2 - install_signal_handlers() - model_map = _model_map(repos_file) - surfaces = frozenset(model_map.defaults) + providers = _providers(repos_file) + surfaces = frozenset(providers) if not surfaces: - print("no usable surface: every models.defaults entry is empty", file=sys.stderr) + print("no provider has a key; nothing can be tested. See providers: in " + f"{repos_file}", file=sys.stderr) return 2 - logging.info("reachable surfaces: %s", ", ".join(sorted(surfaces))) - shim.start(region=args.region, key=key, model_map=model_map, port=args.shim_port) + logging.info("open surfaces: %s", ", ".join(sorted(surfaces))) + shim.start(providers, port=args.shim_port) cfg = Config( harness_root=HARNESS_ROOT, work_root=work, - region=args.region, shim_base=f"{args.shim_host}:{args.shim_port}", model=args.model, effort=args.effort, @@ -104,7 +110,7 @@ def cmd_screen(args: argparse.Namespace) -> int: repos = _load_repos(Path(args.repos).resolve()) if args.repos else [] repos += args.repo - surfaces = frozenset(_model_map(Path(args.repos).resolve()).defaults) if args.repos \ + surfaces = frozenset(_providers(Path(args.repos).resolve())) if args.repos \ else frozenset({"openai"}) editable = editable_install_available() print(f"surfaces={sorted(surfaces)} editable_install={editable}\n") @@ -152,7 +158,6 @@ def main(argv: list[str] | None = None) -> int: run_p.add_argument("--repos", default=str(HARNESS_ROOT / "skill_harness" / "repos.yaml")) run_p.add_argument("--work", default=str(DEFAULT_WORK)) run_p.add_argument("--concurrency", type=int, default=2) - run_p.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) run_p.add_argument("--shim-port", type=int, default=8300) # Containers reach the host by a different name than the harness does. run_p.add_argument("--shim-host", default="http://host.docker.internal") diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml index acad837..8fcedc1 100644 --- a/skill_harness/repos.yaml +++ b/skill_harness/repos.yaml @@ -1,4 +1,4 @@ -# The repos under test, and the model mapping the shim applies to them. +# The repos under test, and which provider surfaces are open for them. # # Hand-supplied while the pipeline is being proven. GitHub search and automated # selection are deliberately out of scope until the failure modes are known — @@ -16,37 +16,22 @@ repos: # Screened and rejected, kept here so the reason is not rediscovered: # langchain-academy no packaging metadata and no root module (M24) # rag-from-scratch notebooks only, no .py at all -# social-media-agent anthropic SDK, and that surface is closed on this key +# social-media-agent anthropic, which is closed until a repo key is supplied -# A repo's model id is hardcoded in its source, and the source is never edited, -# so the shim rewrites the id on the way past. Anything not matched here falls to -# the per-surface default. +# Which provider surfaces are open. A provider whose key is missing is left out, +# and the screen then rejects any repo that needs it -- at a shallow clone, +# rather than after an agent has been paid to port it. # -# An empty default closes that surface: the screen then rejects any repo whose -# SDK would need it, instead of letting an agent spend its budget porting a repo -# that cannot reach a model. -models: - defaults: - # Measured against this credential on 2026-08-28, not assumed. - openai: openai.gpt-oss-120b-1:0 +# Repos keep their own provider and their own model ids: the shim swaps the key +# in and forwards everything else untouched, so nothing here swaps a provider +# (M20). `rewrite` exists only for a model id that cannot be served as written, +# and is empty on purpose. +providers: + openai: + key_env: OPENAI_KEY - # Closed. Every Messages-API-capable Claude on this account answers - # `permission_error`, and Claude 3 Haiku says why: "Model use case details - # have not been submitted for this account." Claude *is* reachable here, but - # only through Converse — a different wire format, so routing an Anthropic - # SDK call to it would need the protocol translation this design avoids. - # Submitting the Anthropic use case form is what reopens this; then put - # a model id back here and anthropic-SDK repos come back into scope. - anthropic: "" - - exact: {} - - prefixes: - - prefix: gpt- - to: openai.gpt-oss-120b-1:0 - - prefix: o1 - to: openai.gpt-oss-120b-1:0 - - prefix: o3 - to: openai.gpt-oss-120b-1:0 - - prefix: o4 - to: openai.gpt-oss-120b-1:0 + # Deliberately not ANTHROPIC_API_KEY: that one runs the porting agent, and + # sharing it would blur agent spend with repo spend and let a runaway repo + # exhaust the porting budget. Set REPO_ANTHROPIC_API_KEY to open this surface. + anthropic: + key_env: REPO_ANTHROPIC_API_KEY diff --git a/skill_harness/shim.py b/skill_harness/shim.py index ea0e176..4d59b90 100644 --- a/skill_harness/shim.py +++ b/skill_harness/shim.py @@ -1,17 +1,19 @@ -"""A model-id rewriting proxy in front of Bedrock. +"""A pass-through proxy in front of the model providers. -Bedrock serves both the OpenAI Chat Completions wire format and the Anthropic -Messages wire format natively, so a ported repo needs no protocol translation — -only its base URL changed, which an environment variable can do. The one thing an -environment variable cannot reach is the model id, which travels in the request -body: a repo asks for `gpt-4o-mini` and Bedrock rejects it. +Each repo is pointed at `/r///...` by its own `.env`, and the shim +forwards the request upstream unchanged except for the API key, which it swaps in +so the real credential never reaches the repo or its image. -So this rewrites the `model` field and forwards everything else unchanged. That -is the whole job. See DESIGN.md section 3. +It rewrites nothing else. Repos keep their own provider and their own model ids, +which is what M20 asks for; the shim exists for the two things a direct +connection cannot give — **per-repo token accounting**, which feeds the results +table, and one place that sees which models a repo actually calls. + +Optional `rewrite` rules cover the case where a model id cannot be served as +written. Leave them out and the request passes through untouched. Requests are buffered, not streamed, matching the scope llm_proxy already set. -When PR #54 lands this should fold into `llm_proxy/providers/` as another -provider rather than continuing to live here. +When PR #54 lands this should fold into `llm_proxy/providers/`. """ from __future__ import annotations @@ -23,22 +25,34 @@ import urllib.error import urllib.request from collections import defaultdict +from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer log = logging.getLogger("shim") -# Per-repo token accounting, keyed by the slug in the request path. This is the -# seam llm_proxy's hooks.py exists for; here it is four lines. +# Per-repo token accounting, keyed by the slug in the request path. USAGE: dict[str, dict[str, int]] = defaultdict(lambda: {"input": 0, "output": 0, "calls": 0}) _USAGE_LOCK = threading.Lock() -# Surface -> (upstream path prefix, auth header name, auth value template). -SURFACES = { - "openai": ("/openai/v1", "Authorization", "Bearer {key}"), - "anthropic": ("/anthropic", "x-api-key", "{key}"), +# How each provider's wire protocol carries its credential. The upstream default +# is the provider's own API; pointing it elsewhere — a gateway, or Bedrock's +# compatible surfaces — is a config change rather than a code change. +PROTOCOLS = { + "openai": { + "upstream": "https://api.openai.com/v1", + "auth_header": "Authorization", + "auth_template": "Bearer {key}", + "extra_headers": {}, + }, + "anthropic": { + "upstream": "https://api.anthropic.com", + "auth_header": "x-api-key", + "auth_template": "{key}", + "extra_headers": {"anthropic-version": "2023-06-01"}, + }, } -_PATH = re.compile(r"^/r/(?P[\w.\-]+)/(?Popenai/v1|anthropic)(?P/.*)$") +_PATH = re.compile(r"^/r/(?P[\w.\-]+)/(?P[\w\-]+)(?P/.*)$") # Headers that describe the hop, not the request. Forwarding them corrupts the # upstream call. @@ -46,67 +60,100 @@ "accept-encoding", "transfer-encoding"} -class ModelMap: - """Resolves a source model id to a Bedrock one. +@dataclass +class Provider: + """One open provider surface. A provider absent from the registry is closed.""" + + name: str + key: str + upstream: str + auth_header: str + auth_template: str + extra_headers: dict = field(default_factory=dict) + # Optional model-id substitutions. Empty means pass through unchanged. + rewrite_exact: dict = field(default_factory=dict) + rewrite_prefixes: list = field(default_factory=list) + _seen: dict = field(default_factory=dict) + + def resolve(self, model: str) -> str: + target = self.rewrite_exact.get(model) or next( + (dst for pre, dst in self.rewrite_prefixes if model.startswith(pre)), model + ) + if self._seen.get(model) != target: + self._seen[model] = target + log.info("%s: %s%s", self.name, model, + "" if target == model else f" -> {target}") + return target - Exact matches first, then prefix rules, then a per-surface default. The - resolved mapping is logged for every distinct source id so a result can - always be read against the model that actually produced it. - """ - def __init__(self, exact: dict[str, str], prefixes: list[tuple[str, str]], - defaults: dict[str, str]): - self.exact = exact - self.prefixes = prefixes - self.defaults = defaults - self.seen: dict[str, str] = {} - - def resolve(self, model: str, surface: str) -> str | None: - """None means the surface is closed -- there is no model to route to. - - A closed surface must answer legibly rather than raise: a repo that - reaches one has been mis-screened, and the reply is the evidence of it. - """ - if model in self.exact: - target = self.exact[model] - else: - target = next( - (dst for pre, dst in self.prefixes if model.startswith(pre)), - self.defaults.get(surface), - ) - if target is None: - return None - if self.seen.get(model) != target: - self.seen[model] = target - log.info("model map: %s -> %s (%s)", model, target, surface) - return target +def build_providers(config: dict, keys: dict[str, str]) -> dict[str, Provider]: + """Assemble the open providers from config plus the keys actually present. + + A provider with no key is left out rather than half-configured: the screen + reads the same registry, so a missing key becomes a stage 2 rejection instead + of a failure at the first request, after an agent has been paid for. + """ + out: dict[str, Provider] = {} + for name, entry in (config or {}).items(): + proto = PROTOCOLS.get(name) + if proto is None: + log.warning("unknown provider %r in config; ignored", name) + continue + key = keys.get(name, "") + if not key: + log.info("provider %s has no key; that surface stays closed", name) + continue + rewrite = (entry or {}).get("rewrite") or {} + out[name] = Provider( + name=name, + key=key, + upstream=(entry or {}).get("upstream") or proto["upstream"], + auth_header=proto["auth_header"], + auth_template=proto["auth_template"], + extra_headers=dict(proto["extra_headers"]), + rewrite_exact=rewrite.get("exact") or {}, + rewrite_prefixes=[(r["prefix"], r["to"]) for r in rewrite.get("prefixes") or []], + ) + return out + + +def _account(slug: str, raw: bytes) -> None: + try: + usage = json.loads(raw).get("usage") or {} + except (json.JSONDecodeError, AttributeError): + return + # OpenAI names them prompt/completion; Anthropic input/output. + inp = usage.get("input_tokens") or usage.get("prompt_tokens") or 0 + out = usage.get("output_tokens") or usage.get("completion_tokens") or 0 + with _USAGE_LOCK: + bucket = USAGE[slug] + bucket["input"] += inp + bucket["output"] += out + bucket["calls"] += 1 -def _handler_class(upstream_host: str, key: str, model_map: ModelMap, timeout: float): +def _handler_class(providers: dict[str, Provider], timeout: float): class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" - def log_message(self, fmt, *args): # quieter than the stdlib default + def log_message(self, fmt, *args): log.debug(fmt, *args) - def _fail(self, code: int, detail: str): - body = json.dumps({"error": "shim_error", "detail": detail}).encode() + def _send(self, code: int, body: bytes): self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) + def _fail(self, code: int, detail: str): + self._send(code, json.dumps({"error": "shim_error", "detail": detail}).encode()) + def do_GET(self): if self.path == "/healthz": - body = b'{"status":"ok"}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - return + return self._send(200, json.dumps( + {"status": "ok", "providers": sorted(providers)}).encode()) self._proxy(b"") def do_POST(self): @@ -117,9 +164,16 @@ def _proxy(self, body: bytes): match = _PATH.match(self.path) if not match: return self._fail(404, f"unroutable path {self.path!r}") - slug, surface_path, rest = match["slug"], match["surface"], match["rest"] - surface = "openai" if surface_path.startswith("openai") else "anthropic" - prefix, auth_header, auth_template = SURFACES[surface] + slug, name, rest = match["slug"], match["provider"], match["rest"] + + provider = providers.get(name) + if provider is None: + # Worth a warning, not just a reply: a repo reaching a closed + # provider means stage 2 let something through that it should + # have rejected, and that is a screen defect to go and fix. + log.warning("%s reached the closed %s surface", slug, name) + return self._fail(503, f"the {name} surface is closed on this harness; " + f"no key is configured for it") if body: try: @@ -127,33 +181,18 @@ def _proxy(self, body: bytes): except json.JSONDecodeError: payload = None if isinstance(payload, dict) and "model" in payload: - target = model_map.resolve(payload["model"], surface) - if target is None: - # Worth a warning, not just a reply: reaching a closed - # surface means stage 2 let a repo through that it should - # have rejected, and that is a screen defect to fix. - log.warning("%s reached the closed %s surface asking for %r", - slug, surface, payload["model"]) - return self._fail(503, f"the {surface} surface is closed on this " - f"credential; no model to route " - f"{payload['model']!r} to") - payload["model"] = target - # Bedrock buffers; the repo may have asked to stream. - payload.pop("stream", None) + payload["model"] = provider.resolve(payload["model"]) body = json.dumps(payload).encode() - headers = { - k: v for k, v in self.headers.items() if k.lower() not in _HOP_BY_HOP - } - headers[auth_header] = auth_template.format(key=key) - if surface == "anthropic": - headers.setdefault("anthropic-version", "2023-06-01") + headers = {k: v for k, v in self.headers.items() + if k.lower() not in _HOP_BY_HOP} + headers[provider.auth_header] = provider.auth_template.format(key=provider.key) + for k, v in provider.extra_headers.items(): + headers.setdefault(k, v) req = urllib.request.Request( - f"{upstream_host}{prefix}{rest}", - data=body or None, - headers=headers, - method=self.command, + f"{provider.upstream}{rest}", data=body or None, + headers=headers, method=self.command, ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: @@ -163,47 +202,23 @@ def _proxy(self, body: bytes): except Exception as e: # network, DNS, timeout return self._fail(502, f"{type(e).__name__}: {e}") - self._account(slug, raw) - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(raw))) - self.end_headers() - self.wfile.write(raw) - - @staticmethod - def _account(slug: str, raw: bytes): - try: - usage = json.loads(raw).get("usage") or {} - except (json.JSONDecodeError, AttributeError): - return - # OpenAI names them prompt/completion; Anthropic input/output. - inp = usage.get("input_tokens") or usage.get("prompt_tokens") or 0 - out = usage.get("output_tokens") or usage.get("completion_tokens") or 0 - with _USAGE_LOCK: - bucket = USAGE[slug] - bucket["input"] += inp - bucket["output"] += out - bucket["calls"] += 1 + _account(slug, raw) + self._send(status, raw) return Handler -def start(region: str, key: str, model_map: ModelMap, host: str = "0.0.0.0", - port: int = 8300, timeout: float = 600.0, - upstream: str | None = None) -> ThreadingHTTPServer: +def start(providers: dict[str, Provider], host: str = "0.0.0.0", port: int = 8300, + timeout: float = 600.0) -> ThreadingHTTPServer: """Start the shim on a daemon thread and return the server. It binds 0.0.0.0 because the callers are agent containers, which reach the - host by a different address than the harness does. `upstream` is injectable - so the shim can be tested without Bedrock, and pointed at `bedrock-mantle` - without a code change. + host by a different address than the harness does. """ - upstream = upstream or f"https://bedrock-runtime.{region}.amazonaws.com" - server = ThreadingHTTPServer( - (host, port), _handler_class(upstream, key, model_map, timeout) - ) + server = ThreadingHTTPServer((host, port), _handler_class(providers, timeout)) threading.Thread(target=server.serve_forever, daemon=True, name="shim").start() - log.info("shim listening on %s:%s -> %s", host, port, upstream) + log.info("shim on %s:%s -> %s", host, port, + ", ".join(f"{n}={p.upstream}" for n, p in sorted(providers.items())) or "(nothing open)") return server diff --git a/skill_harness/stages.py b/skill_harness/stages.py index 45e0334..24fac9d 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -78,7 +78,6 @@ def log_path(self, name: str) -> Path: class Config: harness_root: Path work_root: Path - region: str shim_base: str # what a *container* uses to reach the shim model: str effort: str @@ -240,33 +239,41 @@ def screen(ctx: Ctx) -> Result: # 3. wired # --------------------------------------------------------------------------- # +# A repo's SDK refuses to send without *a* key, so it gets a placeholder; the +# shim replaces it with the real one on the way out. No real credential is +# written into the repo, and none enters the image. +_PLACEHOLDER = "supplied-by-the-shim" + + def wire(ctx: Ctx) -> Result: """Write the .env the port will point `env_file:` at. The source tree is never edited; this adds a file beside it, which is what - the skill's own credential path expects (M18, M23). + the skill's own credential path expects (M18, M23). Only the surfaces this + harness can actually serve are written -- pointing a repo at a base URL that + answers 503 would be worse than leaving it unset, because the failure would + read as the repo's rather than the harness's. """ - key = os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "") - if not key: - return Result(False, "AWS_BEARER_TOKEN_BEDROCK is not set") + if not ctx.cfg.surfaces: + return Result(False, "no provider surface is open") base = f"{ctx.cfg.shim_base}/r/{ctx.slug}" - env = "\n".join([ + lines = [ "# Written by skill_harness. The source tree is untouched; this file is", - "# what `env_file:` in the port's config points at.", - f"OPENAI_BASE_URL={base}/openai/v1", - f"OPENAI_API_KEY=shim-not-used", - f"ANTHROPIC_BASE_URL={base}/anthropic", - f"ANTHROPIC_API_KEY=shim-not-used", - f"AWS_BEARER_TOKEN_BEDROCK={key}", - f"AWS_REGION={ctx.cfg.region}", - "", - ]) + "# what `env_file:` in the port's config points at. The keys are", + "# placeholders: the shim swaps the real ones in as requests pass.", + ] + if "openai" in ctx.cfg.surfaces: + lines += [f"OPENAI_BASE_URL={base}/openai", f"OPENAI_API_KEY={_PLACEHOLDER}"] + if "anthropic" in ctx.cfg.surfaces: + lines += [f"ANTHROPIC_BASE_URL={base}/anthropic", + f"ANTHROPIC_API_KEY={_PLACEHOLDER}"] + lines.append("") + + env = "\n".join(lines) (ctx.root / ".env").write_text(env, encoding="utf-8") - ctx.log_path("3-wire.log").write_text( - env.replace(key, "***"), encoding="utf-8" - ) - return Result(True, f"shim base {base}") + ctx.log_path("3-wire.log").write_text(env, encoding="utf-8") + return Result(True, f"{'+'.join(sorted(ctx.cfg.surfaces))} via {base}") # --------------------------------------------------------------------------- # From 8f660ffd76b7e082ed29c275ae67cda614f01038 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 18:03:13 -0700 Subject: [PATCH 28/43] skill_harness: read both provider signals, and screen out backing services The run reached a real request and failed on ELASTICSEARCH_API_KEY. Two things the screen should have caught first, each costing an agent budget to rediscover. An import hit was short-circuiting the runtime provider strings. A repo can import langchain_openai for its embeddings while its chat model comes from init_chat_model("anthropic/..."), and letting the import decide reported that repo as openai-only. Both signals now count and neither wins. A repo that reads ELASTICSEARCH_URL or PINECONE_API_KEY gets all the way to a served request before failing on a credential nobody supplied. That is a fact about the repo's dependencies, not a defect in the port, so it is now a stage 2 rejection -- and when one slips through anyway, a bare missing env var at stage 8 is recorded as blocked rather than failed. Verified on retrieval-agent-template: what took ten minutes and one agent budget to discover is now visible from a shallow clone. --- skill_harness/runner.py | 5 +++++ skill_harness/screen.py | 40 +++++++++++++++++++++++++++++----------- skill_harness/stages.py | 11 +++++++++++ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/skill_harness/runner.py b/skill_harness/runner.py index 99346f1..56e89ce 100644 --- a/skill_harness/runner.py +++ b/skill_harness/runner.py @@ -62,6 +62,11 @@ def _classify(stage: str, result: Result, ctx: Ctx) -> str: return "blocked" # out of scope for this run, not a skill failure if stage == "wired": return "blocked" # missing credential, nothing was tested + if stage == "served" and ctx.missing_credential: + # The port served a real request far enough to run the source, which + # then asked for a credential nobody gave it. Nothing about the skill or + # about Ventis failed here. + return "blocked" if stage == "ported": if ctx.reported_and_stopped: # The skill told the agent to report rather than fix, and it did. diff --git a/skill_harness/screen.py b/skill_harness/screen.py index f338a43..d056869 100644 --- a/skill_harness/screen.py +++ b/skill_harness/screen.py @@ -62,6 +62,17 @@ MULTIAGENT_MARKERS = ("Send(", "StateGraph", "Crew(", "GroupChat", "add_edge", "Command(") +# Backing services the harness does not stand up. A repo that reads one of these +# gets as far as a real request and then fails on a credential -- which is a fact +# about the repo's dependencies, not about the port, and costs a whole agent +# budget to discover. Redis is absent from the list: ventis deploy provides it. +EXTERNAL_SERVICE_VARS = re.compile( + r"\b(ELASTICSEARCH_(?:URL|API_KEY|USER|PASSWORD)|PINECONE_[A-Z_]+|MONGODB_[A-Z_]+|" + r"WEAVIATE_[A-Z_]+|QDRANT_[A-Z_]+|CHROMA_[A-Z_]+|SUPABASE_[A-Z_]+|" + r"TAVILY_[A-Z_]+|SERPAPI_[A-Z_]+|EXA_API_KEY|FIRECRAWL_[A-Z_]+|" + r"LANGSMITH_[A-Z_]+|POSTGRES_[A-Z_]+|DATABASE_URL)\b" +) + @dataclass class Screen: @@ -72,6 +83,7 @@ class Screen: llm_sdk: str = "none" model_ids: list[str] = field(default_factory=list) is_multiagent: bool = False + external_services: list[str] = field(default_factory=list) # "/" literals -- what init_chat_model resolves at runtime. provider_hints: list[str] = field(default_factory=list) layout: str = "flat" @@ -127,6 +139,7 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, imports: set[str] = set() models: set[str] = set() hints: set[str] = set() + services: set[str] = set() for path in root.rglob("*.py"): if SKIP_DIRS & set(path.relative_to(root).parts): @@ -141,6 +154,7 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, out.loc += text.count("\n") models.update(MODEL_LITERAL.findall(text)) hints.update(PROVIDER_STRING.findall(text)) + services.update(EXTERNAL_SERVICE_VARS.findall(text)) if any(m in text for m in MULTIAGENT_MARKERS): out.is_multiagent = True try: @@ -152,6 +166,8 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, out.model_ids = sorted(models) out.provider_hints = sorted(hints) + # LangSmith is observability, not a dependency the agent needs to answer. + out.external_services = sorted(s for s in services if not s.startswith("LANGSMITH_")) for name, markers in FRAMEWORK_MARKERS: if _matches(imports, markers): @@ -159,21 +175,20 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, break hits = [name for name, markers in SDK_MARKERS if _matches(imports, markers)] - if {"openai", "anthropic"} <= set(hits): + + # Both signals count, and neither may short-circuit the other. A repo can + # import langchain_openai for its embeddings while its chat model comes from + # init_chat_model("anthropic/..."), and letting the import win would report + # such a repo as openai-only and send it to an agent it cannot finish. + named = {h.split("/")[0].split(":")[0] for h in out.provider_hints} + redirectable = (set(hits) | named) & {"openai", "anthropic"} + if len(redirectable) == 2: out.llm_sdk = "both" + elif redirectable: + out.llm_sdk = redirectable.pop() elif hits: out.llm_sdk = hits[0] - # The imports did not name a provider we can redirect, but a runtime provider - # string may. Trust it: it is what init_chat_model will actually resolve. - if out.llm_sdk in ("none", "other"): - named = {h.split("/")[0].split(":")[0] for h in out.provider_hints} - named &= {"openai", "anthropic"} - if len(named) == 2: - out.llm_sdk = "both" - elif named: - out.llm_sdk = named.pop() - # `flat` means the port can import the source, which is a fact about where # modules sit relative to the project root -- not about whether a `src/` # directory happens to exist. @@ -215,6 +230,9 @@ def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, # costs an agent's whole budget to reach the same conclusion. out.reject = (f"no module at the project root ({out.py_files} .py files, " f"all nested) and no editable install (M24)") + elif out.external_services: + out.reject = ("needs backing services this harness does not provide: " + + ", ".join(out.external_services[:4])) elif out.root_py_files == 0 and out.packaging == "none": # The editable install exists, but nothing tells it what the root is. out.reject = "no module at the project root and no packaging metadata (M24)" diff --git a/skill_harness/stages.py b/skill_harness/stages.py index 24fac9d..7284063 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -15,6 +15,7 @@ import json import logging import os +import re import shutil import signal import subprocess @@ -68,6 +69,8 @@ class Ctx: # that takes one has followed the skill, so this is not a port failure and # must not be scored as one. reported_and_stopped: bool = False + # An env var the repo needed and the harness never supplied. + missing_credential: str | None = None _procs: list = field(default_factory=list) def log_path(self, name: str) -> Path: @@ -471,6 +474,14 @@ def serve(ctx: Ctx, query: str = "animals") -> Result: if status == "done": return Result(True, "served") if status == "error": + detail = str(last.get("error", "")) + # A bare env var name is what a repo raises when a backing service it + # needs was never configured. That is a fact about the repo, not a + # defect in the port, so it must not be scored as one. + missing = re.fullmatch(r"'([A-Z][A-Z0-9_]{3,})'", detail.strip()) + if missing: + ctx.missing_credential = missing.group(1) + return Result(False, f"needs {missing.group(1)}, which was never configured") ctx.skill_issue.append({"kind": "request_error", "detail": last}) return Result(False, f"request errored: {str(last)[:300]}") time.sleep(3) From fa51554c019873740a2032abd6e1e36ef4061b7b Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 18:04:10 -0700 Subject: [PATCH 29/43] skill_harness: an openai-only repo that needs no backing service --- skill_harness/repos.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml index 8fcedc1..93e7178 100644 --- a/skill_harness/repos.yaml +++ b/skill_harness/repos.yaml @@ -4,19 +4,23 @@ # selection are deliberately out of scope until the failure modes are known — # see DESIGN.md section 5. -# Screened in scope on 2026-08-28 with `python -m skill_harness screen`. -# All four are src/ layouts with pyproject.toml, which only became portable -# once ventis build learned to install the project itself. +# Screened in scope with `python -m skill_harness screen`. repos: - - https://github.com/langchain-ai/react-agent - - https://github.com/langchain-ai/memory-agent - - https://github.com/langchain-ai/retrieval-agent-template - - https://github.com/langchain-ai/data-enrichment + - https://github.com/yeesimonwong/langGraph-agent-openai -# Screened and rejected, kept here so the reason is not rediscovered: -# langchain-academy no packaging metadata and no root module (M24) -# rag-from-scratch notebooks only, no .py at all -# social-media-agent anthropic, which is closed until a repo key is supplied +# Screened and rejected, kept so the reasons are not rediscovered. Note how few +# survive an OpenAI-only credential: langchain-ai's own templates reach their +# chat model through init_chat_model("anthropic/...") and default to Claude. +# +# react-agent, memory-agent, data-enrichment, langgraph-example, +# rag-research-agent-template anthropic, or both -- closed without a repo key +# retrieval-agent-template both, and needs an Elasticsearch/Pinecone store +# langchain-academy no packaging metadata, no root module (M24) +# rag-from-scratch notebooks only, no .py at all +# new-langgraph-project a skeleton; makes no LLM call +# pharmaceutical-supply-chain-... needs MongoDB +# ai-support-agent needs a database +# Content-Generation-AI-Agent needs MongoDB and Supabase # Which provider surfaces are open. A provider whose key is missing is left out, # and the screen then rejects any repo that needs it -- at a shallow clone, From f112eca90623f5493157cf79a241663ced8116ae Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 18:06:29 -0700 Subject: [PATCH 30/43] skill_harness: record the tokens, and keep blocked rows out of the confusion matrix cost_usd sat empty while the shim counted tokens nobody stored. The counts now land in the row as tokens_in/tokens_out/llm_calls, which are measured; cost_usd stays for when there is a price table to multiply by. A repo stopped by its own missing backing service never put the port to the test, so counting it as a validation miss blamed validate.py for an Elasticsearch instance nobody configured. Blocked rows are excluded, and the report says so where the number is printed. Columns added after a database exists are now added in place, so a schema change costs an ALTER rather than another agent budget. --- skill_harness/__main__.py | 7 +++++-- skill_harness/db.py | 26 ++++++++++++++++++++++++-- skill_harness/runner.py | 6 +++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py index e499c28..65a7652 100644 --- a/skill_harness/__main__.py +++ b/skill_harness/__main__.py @@ -142,8 +142,11 @@ def cmd_report(args: argparse.Namespace) -> int: width = max(len(r["repo"]) for r in rows) for r in rows: v = {None: "-", 1: "pass", 0: "FAIL"}[r["validate_ok"]] - print(f"{r['repo']:<{width}} {r['farthest_step']:<10} {r['status']:<18} validate={v}") - print("\nvalidate.py against the eventual outcome:") + tok = f"{r['tokens_in'] or 0}/{r['tokens_out'] or 0} in {r['llm_calls'] or 0} calls" + print(f"{r['repo']:<{width}} {r['farthest_step']:<10} {r['status']:<10} " + f"validate={v:<4} {tok}") + print("\nvalidate.py against the eventual outcome (blocked rows excluded — they") + print("never put the port to the test):") print(json.dumps(db.confusion(conn), indent=2)) return 0 diff --git a/skill_harness/db.py b/skill_harness/db.py index 20924f0..d52ce60 100644 --- a/skill_harness/db.py +++ b/skill_harness/db.py @@ -39,6 +39,9 @@ core_issue TEXT, skill_issue TEXT, analysis TEXT, + tokens_in INTEGER, + tokens_out INTEGER, + llm_calls INTEGER, cost_usd REAL, artifacts TEXT NOT NULL, started_at TEXT NOT NULL, @@ -47,12 +50,25 @@ """ +# Columns added after the first databases were written. Cheap to add in place, +# and cheaper than re-running a repo to change a schema. +_ADDED_COLUMNS = { + "tests": {"tokens_in": "INTEGER", "tokens_out": "INTEGER", + "llm_calls": "INTEGER", "cost_usd": "REAL"}, +} + + def connect(path: str | Path) -> sqlite3.Connection: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path, check_same_thread=False) conn.row_factory = sqlite3.Row conn.executescript(SCHEMA) + for table, columns in _ADDED_COLUMNS.items(): + have = {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")} + for name, decl in columns.items(): + if name not in have: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}") conn.commit() return conn @@ -91,7 +107,8 @@ def record_test(conn: sqlite3.Connection, **fields) -> int: def summary(conn: sqlite3.Connection) -> list[sqlite3.Row]: return conn.execute( """ - SELECT repo, farthest_step, status, validate_ok, cost_usd, artifacts + SELECT repo, farthest_step, status, validate_ok, + tokens_in, tokens_out, llm_calls, artifacts FROM tests ORDER BY id """ ).fetchall() @@ -103,9 +120,14 @@ def confusion(conn: sqlite3.Connection) -> dict[str, int]: A false negative is a check validate.py is missing. A false positive is a check that was wrong to block, and is only observable because the build ran anyway. + + `blocked` rows are excluded. A repo stopped by its own missing backing + service never put the port to the test, so counting it as a validation miss + would blame validate.py for a vector store nobody configured. """ rows = conn.execute( - "SELECT validate_ok, farthest_step FROM tests WHERE validate_ok IS NOT NULL" + "SELECT validate_ok, farthest_step FROM tests " + "WHERE validate_ok IS NOT NULL AND status != 'blocked'" ).fetchall() served = lambda r: r["farthest_step"] == "served" # noqa: E731 return { diff --git a/skill_harness/runner.py b/skill_harness/runner.py index 56e89ce..65bd2ff 100644 --- a/skill_harness/runner.py +++ b/skill_harness/runner.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import logging import re import subprocess @@ -119,7 +120,7 @@ def run_repo(repo: str, cfg: Config, conn, docker_lock: threading.Lock) -> dict: stages.teardown(ctx) usage = shim.usage_for(slug) - (artifacts / "usage.json").write_text(str(usage), encoding="utf-8") + (artifacts / "usage.json").write_text(json.dumps(usage, indent=2), encoding="utf-8") if ctx.screen: db.upsert_repo(conn, repo, framework=ctx.screen.framework, @@ -139,6 +140,9 @@ def run_repo(repo: str, cfg: Config, conn, docker_lock: threading.Lock) -> dict: core_issue=ctx.core_issue or None, skill_issue=ctx.skill_issue or None, analysis=None, + tokens_in=usage["input"], + tokens_out=usage["output"], + llm_calls=usage["calls"], cost_usd=None, artifacts=str(artifacts), started_at=started, From 24160f63d7b463b999d993e88b91d60255bd4276 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 18:14:58 -0700 Subject: [PATCH 31/43] skill_harness: separate the port serving from the project working The first repo to reach stage 8 answered with status done at the Ventis layer and status failed inside its own result: it is an SSH operations agent, and the harness had asked it about animals. The port did exactly what the skill promises -- carried a request to the source and returned the source's own result -- but recording that as an unqualified pass would let a hundred-repo pass rate mean much less than it appears to. served still means the port worked, because that is what is under test. An application-level error inside the payload is now recorded alongside it, so the two can be told apart when the corpus is large enough to summarise. --- skill_harness/runner.py | 3 ++- skill_harness/stages.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/skill_harness/runner.py b/skill_harness/runner.py index 65bd2ff..01f192d 100644 --- a/skill_harness/runner.py +++ b/skill_harness/runner.py @@ -139,7 +139,8 @@ def run_repo(repo: str, cfg: Config, conn, docker_lock: threading.Lock) -> dict: validate_ok=None if ctx.validate_ok is None else int(ctx.validate_ok), core_issue=ctx.core_issue or None, skill_issue=ctx.skill_issue or None, - analysis=None, + analysis=(f"served, but the project's own logic errored: {ctx.app_error}" + if ctx.app_error else None), tokens_in=usage["input"], tokens_out=usage["output"], llm_calls=usage["calls"], diff --git a/skill_harness/stages.py b/skill_harness/stages.py index 7284063..d10caa7 100644 --- a/skill_harness/stages.py +++ b/skill_harness/stages.py @@ -71,6 +71,9 @@ class Ctx: reported_and_stopped: bool = False # An env var the repo needed and the harness never supplied. missing_credential: str | None = None + # The port served a result and the source's own logic failed inside it. + # Not a port defect, but it means the run proved less than "served" suggests. + app_error: str | None = None _procs: list = field(default_factory=list) def log_path(self, name: str) -> Path: @@ -472,6 +475,20 @@ def serve(ctx: Ctx, query: str = "animals") -> Result: ctx.log_path("8-serve.json").write_text(json.dumps(last, indent=2), encoding="utf-8") status = last.get("status") if status == "done": + # Ventis served the request. Whether the *project* then did anything + # useful is a separate question, and conflating the two would let a + # hundred-repo pass rate mean much less than it appears to: a port + # can be perfect while the source fails on a query that means + # nothing to it, or on a host it was never given. + inner = last.get("result") + if isinstance(inner, dict): + app = inner.get("status") or inner.get("error") or inner.get("error_message") + if inner.get("status") in ("failed", "error") or inner.get("error_message"): + ctx.app_error = str( + inner.get("error_message") or inner.get("error") or app + )[:300] + return Result(True, f"served; the project itself errored: " + f"{ctx.app_error[:80]}") return Result(True, "served") if status == "error": detail = str(last.get("error", "")) From 55beca94e80ffc26516674dd654578992bec6333 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 18:57:52 -0700 Subject: [PATCH 32/43] Replace the Python harness with a skill The pipeline now runs inside Claude Code as testing-porting-to-ventis, rather than as an orchestrator that shells out to claude -p. Three files: the procedure, the schema, and a helper that writes a row without hand-quoting JSON into SQL. The database loses the columns nothing needed. Token counts and cost came from a proxy that existed mostly to produce them; stars, framework, is_multiagent, description, core_issue, skill_issue and analysis are written by the agent that ran the port, which is the only thing in the loop that can judge them. Static screening is gone with it. It was wrong twice in ways that each cost a full agent budget -- it read imports and missed that every LangGraph template picks its provider from an init_chat_model string, and it did not look for the backing services a repo needs -- so the skill asks the agent to read the repo and tells it, from those failures, exactly what to look for. What the harness enforced mechanically the skill now has to state, so the rules that made results attributable are written down: the source tree is never edited, the skill under test is pinned and never edited mid-corpus, validate.py does not gate the build, both probes run, and a run that proved nothing is blocked rather than failed. The agent judges scope and writes the analysis; every other stage is a command whose exit code it records rather than interprets. The harness is recoverable at f112eca if any of that turns out to be worth having back as code. --- .../skills/testing-porting-to-ventis/SKILL.md | 192 +++++++ .../testing-porting-to-ventis/record.py | 75 +++ .../testing-porting-to-ventis/schema.sql | 38 ++ .gitignore | 3 +- skill_harness/DESIGN.md | 260 --------- skill_harness/README.md | 52 -- skill_harness/__init__.py | 0 skill_harness/__main__.py | 193 ------- skill_harness/db.py | 138 ----- skill_harness/repos.yaml | 41 -- skill_harness/runner.py | 161 ------ skill_harness/screen.py | 240 -------- skill_harness/shim.py | 227 -------- skill_harness/stages.py | 526 ------------------ 14 files changed, 307 insertions(+), 1839 deletions(-) create mode 100644 .claude/skills/testing-porting-to-ventis/SKILL.md create mode 100644 .claude/skills/testing-porting-to-ventis/record.py create mode 100644 .claude/skills/testing-porting-to-ventis/schema.sql delete mode 100644 skill_harness/DESIGN.md delete mode 100644 skill_harness/README.md delete mode 100644 skill_harness/__init__.py delete mode 100644 skill_harness/__main__.py delete mode 100644 skill_harness/db.py delete mode 100644 skill_harness/repos.yaml delete mode 100644 skill_harness/runner.py delete mode 100644 skill_harness/screen.py delete mode 100644 skill_harness/shim.py delete mode 100644 skill_harness/stages.py diff --git a/.claude/skills/testing-porting-to-ventis/SKILL.md b/.claude/skills/testing-porting-to-ventis/SKILL.md new file mode 100644 index 0000000..9a06f8e --- /dev/null +++ b/.claude/skills/testing-porting-to-ventis/SKILL.md @@ -0,0 +1,192 @@ +--- +name: testing-porting-to-ventis +description: Use when running a repository through the porting-to-ventis skill to find out where the port stops, or when building a corpus of such results across many repositories +--- + +# Testing `porting-to-ventis` against a repository + +One repository per run. The output is a row in `.ventis-tests/results.sqlite` +saying how far the port got and what stopped it, plus a directory of every +command's raw output. + +**A pass rate is not the deliverable.** The deliverable is attribution: for each +repository, whether the blame lies with the skill, with Ventis, or with the +repository itself. A run that ends `blocked` because the repo needs a vector +store nobody configured is not evidence about the skill, and recording it as a +failure makes the whole corpus mean less than it appears to. + +## Two rules the run is built on + +**Never edit the source tree.** M19 and M20 are rules the skill is being tested +on. Rewriting a repo's model calls, or setting its model config to a different +provider, tests the rewrite instead — and every later failure becomes +unattributable. Add files beside the source; leave `git status` on the source +clean. + +**Never edit `porting-to-ventis` during a run.** Its git tree hash is pinned into +every row. A skill edited between repo 1 and repo 100 means the two were not +given the same test. Fix it between runs, as a new pinned version, and re-run +what it affects. + +## What you judge, and what you only record + +You judge two things: **whether the repo is in scope** (step 2) and **what the +result means** (the write-up). Everything else is a command whose exit code and +output you record verbatim. + +**Never decide that a build "basically worked".** `ventis build` prints +`Build complete.` and exits 0 for a project whose container dies on startup, so +its exit code is not evidence on its own — that is what the two probes in step 6 +are for. If a command failed, the stage failed, whatever you think of the reason. + +## The run + +Work in `.ventis-tests//`, with the clone at `src/` and every command's +output written into `artifacts/`. + +```bash +mkdir -p .ventis-tests//artifacts +git clone --depth 1 .ventis-tests//src +git -C .ventis-tests//src rev-parse HEAD # repo_sha +git rev-parse HEAD:.claude/skills/porting-to-ventis # skill_sha +git rev-parse HEAD:ventis # ventis_sha +``` + +| # | `farthest_step` | What runs | +| --- | --- | --- | +| 1 | `fetched` | the clone above | +| 2 | `screened` | your read of the repo — see below | +| 3 | `wired` | write `.env` beside the source with the keys the repo needs | +| 4 | `ported` | **the `porting-to-ventis` skill**, on this repo | +| 5 | `validated` | `python .claude/skills/porting-to-ventis/validate.py .` | +| 6 | `built` | `ventis build -c config/global_controller.yaml`, then both probes | +| 7 | `deployed` | `ventis deploy -c config/global_controller.yaml`, backgrounded | +| 8 | `served` | `POST /main`, then poll `GET /status/` | + +`farthest_step` is the furthest stage reached. Step 5 is the exception: it does +**not** gate what follows. + +### Step 2 — read the repo before spending anything on it + +Answer these from the source. Each rejection below was learned by paying an +agent's full budget to rediscover it. + +| Question | Reject when | +| --- | --- | +| Is there a module the adapter can import from the project root? | No root-level `.py` **and** no `pyproject.toml`/`setup.py`/`setup.cfg`. Without packaging metadata nothing is importable at `/app`. This is M24, and it rejects most tutorial repos. | +| Which provider will it actually call? | It needs one whose key you do not have. | +| Does it need a backing service? | It reads `ELASTICSEARCH_*`, `PINECONE_*`, `MONGODB_*`, `QDRANT_*`, `WEAVIATE_*`, `SUPABASE_*`, `TAVILY_*`, `DATABASE_URL`… Ventis provides Redis and nothing else. | +| Is there Python at all? | Notebooks only, or no LLM call anywhere. | +| Is it small to medium? | Hundreds of modules, or a framework rather than a project. | + +**Reading the imports is not enough to answer the provider question.** Every +LangGraph template reaches its model through `init_chat_model("anthropic/…")` or +a config default string, so a repo can depend entirely on Anthropic while +importing nothing named `anthropic` — and can import `langchain_openai` for its +embeddings while its chat model is Claude. Grep the string literals as well as +the imports, and take the union: + +```bash +grep -rnoE '"(openai|anthropic|google_genai|bedrock|cohere|mistralai)[:/][^"]+"' +``` + +A repo needing a provider you cannot serve is `blocked`, not `failed`. Record +which provider and stop — that count is the argument for obtaining the key. + +### Step 3 — the credential goes beside the source, never inside it + +Write `.env` at the project root with the real keys, and let the port's +`config/global_controller.yaml` point `env_file:` at it. Never bake a key into +the build context: `ventis build` sweeps the project into every image, and +`_sweep_project_files` skips dotfiles precisely so `.env` cannot ride along. + +### Step 4 — run the skill + +Use the `porting-to-ventis` skill on the clone. Follow it as written; it is the +artifact under test. When it tells you to report something rather than fix it, +write `PORT_REPORT.md` in the repo and stop — **that is the skill working, and +the run is `blocked`, not `failed`.** Those paths fire on things Ventis cannot +do, so the finding belongs in `core_issue`. + +### Step 6 — build, then probe twice + +```bash +ventis build -c config/global_controller.yaml + +# 1. the runtime, which fails before your agent is reached +docker run --rm ventis- python -c "import local_controller" + +# 2. the agent, loaded the way _load_agent loads it +docker run --rm ventis- python -c " +import importlib.util, sys +spec = importlib.util.spec_from_file_location('m', '.py') +m = importlib.util.module_from_spec(spec); sys.modules['m'] = m +spec.loader.exec_module(m); m.(); print('ok')" +``` + +Probe 1 catches the protobuf/gRPC wall — a `core_issue`, since the fix belongs in +`generate_docker`. Probe 2 catches everything `_load_agent` swallows, which +otherwise surfaces only as `"No agent loaded"` at step 8. + +### Step 8 — served means the port answered + +`POST /main {"query": ...}` and poll `/status/`. **Send a query the +repo can actually act on** — read its README first. Asking an SSH operations +agent about "animals" tests nothing. + +An outer `"status": "done"` with an inner `"status": "failed"` means the port +worked and the project's own logic did not. That is `passed`, because the port +did what the skill promises — carry a request to the source and return the +source's own result — but say so in `analysis`. A bare missing env var +(`'ELASTICSEARCH_API_KEY'`) is `blocked`: nobody configured it. + +## Deciding the status + +| `status` | When | +| --- | --- | +| `passed` | Step 8 returned the source's own result. | +| `blocked` | Nothing was tested: out of scope at step 2, a missing key or backing service, or the skill correctly reported-and-stopped. | +| `failed` | The port was attempted and something about it did not work. | + +`blocked` is not a soft `failed`. It means this repository produced no evidence +about the skill, and rows that produced no evidence must not be counted as if +they had. + +## Recording + +Always record, including for `blocked` runs — a rejection is the datum. + +```bash +python .claude/skills/testing-porting-to-ventis/record.py \ + --db .ventis-tests/results.sqlite <<'JSON' +{ + "repo": "https://github.com/owner/name", + "repo_sha": "…", "skill_sha": "…", "ventis_sha": "…", + "stars": 128, "framework": "langgraph", "is_multiagent": 1, + "description": "what it does, technically", + "farthest_step": "built", "status": "failed", "validate_ok": 1, + "core_issue": [{"kind": "runtime_import", "detail": "…"}], + "skill_issue": [{"kind": "no_fanout", "detail": "…"}], + "analysis": "what happened and why, in a few sentences", + "artifacts": ".ventis-tests//artifacts", + "started_at": "2026-08-28T00:00:00Z", "ended_at": "2026-08-28T00:10:00Z" +} +JSON +``` + +`core_issue` is what a Ventis owner must fix; `skill_issue` is what the skill +file must say better. Keep them apart — collapsing them loses the distinction the +whole exercise exists to produce. Leave both empty when the run had no findings. + +## Common mistakes + +| Mistake | What it costs | +| --- | --- | +| Screening on imports alone | Anthropic-only repos reach step 4 and burn a budget before failing | +| Skipping the `provider/model` grep | Same, and it is the default shape of every LangGraph template | +| Letting `validate.py` gate the build | The one case where a validation was wrong to block becomes unobservable | +| Treating `Build complete.` as evidence | A green build and a healthy replica are both compatible with a container that serves nothing | +| Running one probe instead of two | Probe 1's failure is a Ventis bug; probe 2's is the port's; neither covers the other | +| Scoring report-and-stop as `failed` | Counts the skill working as the skill failing, and buries the Ventis gap that caused it | +| Sending `{"query": "animals"}` to everything | `served` stops meaning anything | +| Editing the skill mid-corpus | The pass rate loses its denominator | diff --git a/.claude/skills/testing-porting-to-ventis/record.py b/.claude/skills/testing-porting-to-ventis/record.py new file mode 100644 index 0000000..c3d25aa --- /dev/null +++ b/.claude/skills/testing-porting-to-ventis/record.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Write one test result into the results database. + +Reads a JSON object on stdin so that findings and analysis -- which contain +quotes, newlines and error text -- reach SQLite as data. Hand-quoting them into +a `sqlite3` heredoc is how a run's own error message ends up truncating the row +that was supposed to record it. + + python record.py --db .ventis-tests/results.sqlite <<'JSON' + {"repo": "...", "repo_sha": "...", ..., "core_issue": [...]} + JSON +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +REPO_FIELDS = ("stars", "framework", "is_multiagent", "description") +TEST_FIELDS = ("repo", "repo_sha", "skill_sha", "ventis_sha", "farthest_step", + "status", "validate_ok", "core_issue", "skill_issue", "analysis", + "artifacts", "started_at", "ended_at") +REQUIRED = ("repo", "repo_sha", "skill_sha", "ventis_sha", "farthest_step", + "status", "artifacts", "started_at") +STATUSES = {"passed", "failed", "blocked"} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--db", required=True) + ap.add_argument("--schema", default=str(Path(__file__).with_name("schema.sql"))) + args = ap.parse_args() + + row = json.load(sys.stdin) + + missing = [f for f in REQUIRED if not row.get(f)] + if missing: + print(f"missing required field(s): {', '.join(missing)}", file=sys.stderr) + return 2 + if row["status"] not in STATUSES: + print(f"status must be one of {sorted(STATUSES)}", file=sys.stderr) + return 2 + + db = Path(args.db) + db.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db) + conn.executescript(Path(args.schema).read_text(encoding="utf-8")) + + with conn: + conn.execute("INSERT OR IGNORE INTO repos (repo) VALUES (?)", (row["repo"],)) + cols = {f: row[f] for f in REPO_FIELDS if row.get(f) is not None} + if cols: + assigns = ", ".join(f"{k} = ?" for k in cols) + conn.execute(f"UPDATE repos SET {assigns} WHERE repo = ?", + (*cols.values(), row["repo"])) + + test = {} + for f in TEST_FIELDS: + v = row.get(f) + test[f] = json.dumps(v) if isinstance(v, (list, dict)) else v + names = ", ".join(test) + marks = ", ".join("?" for _ in test) + cur = conn.execute(f"INSERT INTO tests ({names}) VALUES ({marks})", + tuple(test.values())) + + print(f"recorded test #{cur.lastrowid}: {row['repo']} -> " + f"{row['status']} at {row['farthest_step']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/testing-porting-to-ventis/schema.sql b/.claude/skills/testing-porting-to-ventis/schema.sql new file mode 100644 index 0000000..41abbc9 --- /dev/null +++ b/.claude/skills/testing-porting-to-ventis/schema.sql @@ -0,0 +1,38 @@ +-- Results of running `porting-to-ventis` against a repository. +-- +-- Everything a machine can produce is a column; everything that needs judgement +-- is written by the agent that ran the port. No analysis is stored that could be +-- recomputed later from the artifacts directory -- running the pipeline is the +-- expensive part, and reading its output afterwards is not. + +CREATE TABLE IF NOT EXISTS repos ( + id INTEGER PRIMARY KEY, + repo TEXT UNIQUE NOT NULL, -- github url + stars INTEGER, -- gh api + framework TEXT, -- langchain|langgraph|crewai|autogen|adk|plain + is_multiagent INTEGER, -- does one request fan out to independent work? + description TEXT -- technical, written by the agent +); + +CREATE TABLE IF NOT EXISTS tests ( + id INTEGER PRIMARY KEY, + repo TEXT NOT NULL REFERENCES repos(repo), + + -- The three pins. Nothing else here can be reconstructed once a run is over: + -- which source, which skill, and which Ventis produced this result. + repo_sha TEXT NOT NULL, + skill_sha TEXT NOT NULL, + ventis_sha TEXT NOT NULL, + + farthest_step TEXT NOT NULL, -- the furthest stage reached + status TEXT NOT NULL, -- passed|failed|blocked + validate_ok INTEGER, -- stage 5's verdict, kept apart from the outcome + + core_issue TEXT, -- json: findings a Ventis owner must fix + skill_issue TEXT, -- json: findings the skill file must fix + analysis TEXT, -- the agent's recap: what happened and why + + artifacts TEXT NOT NULL, -- directory holding every command's output + started_at TEXT NOT NULL, + ended_at TEXT +); diff --git a/.gitignore b/.gitignore index 1b1c177..085b4a3 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,6 @@ uv.lock Agent Artifacts docs/ -# skill_harness working tree: clones, artifacts, results db +# testing-porting-to-ventis working tree: clones, artifacts, results db +.ventis-tests/ .harness/ diff --git a/skill_harness/DESIGN.md b/skill_harness/DESIGN.md deleted file mode 100644 index c7cf530..0000000 --- a/skill_harness/DESIGN.md +++ /dev/null @@ -1,260 +0,0 @@ -# Testing and continually improve `porting-to-ventis` , its harness and core - -## 1. Pipeline - - -| # | Stage | What runs | Fails when | -| --- | ----------- | --------------------------------------------------------------------------- | ------------------------------------------------ | -| 1 | `fetched` | `git clone --depth 1`, record SHA | repo gone, too large, no license | -| 2 | `screened` | static scan: framework, LLM provider, hardcoded model ids, dependency shape | repo is out of scope for this run | -| 3 | `wired` | write `.env`, ensure model shim is up | no Bedrock credential, unmappable model | -| 4 | `ported` | `**claude -p**` running `porting-to-ventis` | agent gives up, budget exhausted, timeout | -| 5 | `validated` | `validate.py ` | contract violation the agent introduced | -| 6 | `built` | `ventis build` + both probes from SKILL.md Step 4 | image builds but container cannot import | -| 7 | `deployed` | `ventis deploy` | port/config/policy failure | -| 8 | `served` | `POST /main` → `GET /status/` | `"No agent loaded"`, provider error, wrong shape | - - -**Only stage 4 uses an agent.** Everything else is a deterministic subprocess with -a timeout. This is the property that makes failures attributable: a stage 6 failure -is a fact about the port, not about how the agent happened to behave that day. - -Stage 6 runs *both* probes from SKILL.md Step 4, in order, because neither covers -the other — probe 1 (`import local_controller`) catches the protobuf/gRPC wall -before the agent is ever reached; probe 2 (`_load_agent`-shaped import) catches -the failures that otherwise surface only as `"No agent loaded"` at stage 8. - -**Stage 5 does not gate stages 6–8.** A failed `validate.py` is recorded and the -pipeline continues. This is the only way to observe a validation that was wrong to -block — validate says no, the port would have served anyway — and that observation -cannot be recovered later, because the build never ran. Together with the opposite -case (validate passes, a later stage fails, which is visible by default) it gives -`validate.py` a confusion matrix, which is the only quantitative basis on which the -script can be improved. The cost is a few wasted builds. - -## 2. Driving Claude Code - -A `claude -p` subprocess per repo, concurrency 2 until the pipeline is proven. - -``` -claude -p "" \ - --bare \ - --setting-sources "" \ - --permission-mode bypassPermissions \ - --output-format stream-json --verbose \ - --model --effort \ - --max-budget-usd \ - --no-session-persistence -``` - -Every flag above was checked against `claude --help` on the machine that will run -it, not recalled. - -- `**--bare` is not optional.** It suppresses hooks, auto-memory, plugin sync and -CLAUDE.md auto-discovery. Without it the operator's personal `~/.claude/CLAUDE.md` -and accumulated auto-memory enter all 100 runs, vary between them, and are -invisible in the results. Under `--bare` auth is strictly `ANTHROPIC_API_KEY`. -- `**--setting-sources ""**` keeps user/project/local settings out for the same -reason. -- `**--max-budget-usd**` is the containment mechanism; this CLI has no `--max-turns`. -A budget-exhausted run is recorded as its own failure mode, not as a crash. -- **The skill is delivered explicitly**, by copying -`.claude/skills/porting-to-ventis/` into each repo working directory, so the -version under test is the version recorded — never whatever is globally installed. -- **Tool restriction is unresolved and must be measured.** There are reports that -under `bypassPermissions`, `--allowedTools` is ignored and only `--disallowedTools` -constrains the tool set. This is verified on the first repo before the run scales; -it is not assumed in either direction. - -`--output-format stream-json` is written into the run's `artifacts/` directory. The -trace is the only record of *how* the agent reached its result, and it is what makes -a skill defect diagnosable after the fact. - -## 3. Reaching Bedrock without touching the source - -Verified against AWS documentation on 2026-08-27: - - -| Source SDK | Base URL | Auth header | -| ----------------------------- | ---------------------------------------------------------- | ------------------------------------------------- | -| `openai` / `ChatOpenAI` | `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` | `Authorization: Bearer $AWS_BEARER_TOKEN_BEDROCK` | -| `anthropic` / `ChatAnthropic` | `https://bedrock-runtime.{region}.amazonaws.com/anthropic` | `x-api-key: $AWS_BEARER_TOKEN_BEDROCK` | - - -Both surfaces support client-side tool use. Both are reachable by environment -variable alone, which is why the source tree never needs an edit. - -**Model coverage is asymmetric and constrains repo selection.** Counted from -Bedrock's API-compatibility tables: 40 models serve Chat Completions (OpenAI, Qwen, -Mistral, Google, Z.AI, NVIDIA, DeepSeek, MiniMax, xAI, Moonshot); 7 serve the -Messages API, all Anthropic Claude. No Claude model serves Chat Completions, and -Meta / Amazon / Cohere / AI21 serve neither. A `ChatOpenAI` repo therefore lands on -a gpt-oss / Qwen / Mistral class model, never on Claude. - -**What the credential can reach is narrower still, and is an account property -rather than a property of Bedrock.** Measured on 2026-08-28 against the key in -use: - - -| Surface | Result | -| ----------------------- | -------------------------------------------------------------------------------------------------------- | -| OpenAI Chat Completions | works — `openai.gpt-oss-120b-1:0`, and gpt-oss-20b, qwen3-32b, mistral-large-3, deepseek-v3.2 all answer | -| Anthropic Messages | closed — every Messages-capable Claude answers `permission_error` | - - -The control plane lists 121 models, which is what the platform offers and not -what the account may call: a model can appear there and still be refused. Claude 3 -Haiku gives the reason — *"Model use case details have not been submitted for this -account"* — so this is an entitlement, reopened by submitting the Anthropic use -case form rather than by any change here. - -Claude is reachable on this account through **Converse**, which was confirmed. It -is not a way around the closed surface: Converse is a third wire format, so -routing an Anthropic SDK call to it means the protocol translation this design -exists to avoid. - -The consequence is a scope limit that must be stated with any result from this -run: **repos using the Anthropic SDK are rejected at stage 2, not tested.** The -harness expresses this as data rather than in code — a surface whose entry in -`repos.yaml` is empty is a surface the screen refuses to route to — so the day -the entitlement lands, one line of configuration brings those repos back. - -**The model id is the one thing an env var cannot reach.** A repo writes -`ChatOpenAI(model="gpt-4o-mini")`; the id travels in the request body, and Bedrock -rejects it. The fix is a shim in front of Bedrock that **rewrites the `model` field -and forwards everything else unchanged**. No protocol translation is involved — -Bedrock speaks both wire formats natively — so this is a small addition to the -`llm_proxy` skeleton on PR #54 (`core.proxy_request`, `providers/base.HttpProvider`), -not a new subsystem. Its `hooks.py` seam yields per-repo token accounting for free. - -The mapping from observed model id to Bedrock model id is shim configuration, -recorded per run, so a result can always be read against the model that produced it. - -Credentials reach the containers through `env_file:` (PR #53, merged into this -branch), which is the only sanctioned path — M18 forbids baking a key into the -build context. - -**Credential shape.** The key in use is a short-term Bedrock bearer token: an -`ASIA...` STS credential scoped to one region, valid 12 hours. That is ample for -proving the pipeline on two repos and too short for a hundred, so a run at full -size needs either a long-term key or a refresh step. The harness reads the token -from `AWS_BEARER_TOKEN_BEDROCK` on each `wire`, so a refreshed token is picked up -by repos that have not started yet, but not by containers already running. - -## 4. Storage - -SQLite, holding the two tables from the ticket. The database stores **artifacts and -versions, not analysis.** - -```sql -CREATE TABLE repos ( - id INTEGER PRIMARY KEY, - repo TEXT UNIQUE NOT NULL, -- github url - stars INTEGER, - framework TEXT, -- langchain|langgraph|crewai|autogen|plain|adk - is_multiagent INTEGER, - description TEXT -); - -CREATE TABLE tests ( - id INTEGER PRIMARY KEY, - repo TEXT NOT NULL REFERENCES repos(repo), - repo_sha TEXT NOT NULL, -- which source - skill_sha TEXT NOT NULL, -- which skill - ventis_sha TEXT NOT NULL, -- which core - farthest_step TEXT NOT NULL, -- the stage enum of §1 - status TEXT NOT NULL, -- passed|failed|blocked|budget_exhausted|timeout - validate_ok INTEGER, -- stage 5's verdict, kept apart from the outcome - core_issue TEXT, -- json: Ventis defects - skill_issue TEXT, -- json: skill defects - analysis TEXT, -- AI recap - cost_usd REAL, - artifacts TEXT NOT NULL -- directory: trace, the four written files, - -- validate output, per-stage stderr -); -``` - -The three SHAs are the only things that cannot be reconstructed afterwards — once -the run is over, which skill and which core produced a result is unrecoverable. -Everything else about *why* a repo failed is computed later by reading `artifacts/`. - -`validate_ok` is a separate column from `farthest_step` so the two can be joined: -that join is the confusion matrix of §1. - -`core_issue` and `skill_issue` stay separate columns: "Ventis cannot do this" and -"the skill fails to say this" are findings with different owners, and collapsing -them loses the distinction the run exists to produce. - -Deliberately **not** in the schema: per-check validation statistics, agent-behaviour -counts (which skill files were read, how many edits were retried), and the -deterministic audits of the MUST rules a machine can decide (M19's clean -`git status` on the source, M20's unchanged provider imports). All of these are -derivable from `artifacts/` by a script, at any time, without re-running anything — -and running the pipeline is the expensive part. Write those scripts when there is a -corpus worth aggregating and it is clear what to aggregate. - -## 4a. What the corpus turned out to cost - -The first screening run answered a question this design had filed as a -by-product. Of six LangChain sample repositories, **none were in scope**: five -are `src/` layouts with `pyproject.toml`, and `ventis build` could not make such -a tree importable, so no port of them could load. `src/` plus packaging metadata -is not a quirk of those five — it is the shape LangChain's own templates ship. - -That made the corpus, not the harness, the binding constraint on CAN-238: a -hundred-repo run against a Ventis without an editable install would have produced -close to a hundred stage 2 rejections and tested almost nothing. - -`_install_step` — a Dockerfile step that runs `pip install -e .` when the project -declares packaging metadata, handing the import root to the project rather than -making Ventis guess a directory — was ported onto this branch from -`jiajunh/can-228-create-a-skill-to-convert-a-langchain-project-to-ventis`. Four of -the six came into scope immediately. - -**Ported rather than merged, deliberately.** That branch is an older parallel -line: it carries its own copy of the skill from before `validate.py` existed, its -own earlier `env_file.py`, and its own `joke_writer`. Merging it whole conflicted -on twelve files, three of them the skill — it would have regressed the artifact -under test in the act of enabling the test. - -The M24 rejection is still real for repos that declare no packaging metadata at -all, and `langchain-academy` remains rejected for exactly that reason. - -## 5. Scope of the first version - -Stages 1–8 straight through, concurrency fixed at 2, repo list supplied by hand — -two repos from `langchain-samples`. - -Deliberately excluded until the pipeline is proven: GitHub search and automated -repo selection, retry policy, parallelism above 2, and any cross-repo aggregation -beyond the raw table. These are worth writing once the failure modes are known and -not before. - -## 6. Rejected alternatives - -**Rewriting each repo's LLM calls onto Bedrock before the port** — the literal -reading of the ticket plan. Rejected: it violates M19 and M20, which are rules -under test, and it contaminates every downstream stage. `examples/joke_writer`'s -README already records this conclusion for the one project where the rewrite was -done deliberately: it "is not something the `porting-to-ventis` skill should do on -a user's project — it is the credential wall, and the skill's instruction is to -report it." - -**A protocol-translating proxy** (OpenAI/Anthropic wire format → Bedrock Converse). -Rejected as unnecessary: Bedrock serves both wire formats natively, so only the -model id needs rewriting. - -**The Claude Agent SDK as the driver.** Considered for its structured event stream. -Rejected for the first version: a `claude -p` subprocess gives the same trace via -`--output-format stream-json` with one less dependency, and the pipeline's -attribution comes from stages 5–8 being deterministic rather than from finer -introspection of stage 4. - -**Restricting the run to repos already on Bedrock.** Rejected: too few exist to -reach 100, and selecting for them would bias the sample toward projects that never -exercise the credential wall the skill has the most to say about. - -**Merging `can-228` whole to obtain the editable install.** Rejected for the -reason in section 4a: the branch carries an older copy of the artifact under -test, so the merge would have changed what the run measures. The one capability -was ported instead, and its provenance recorded in the commit. \ No newline at end of file diff --git a/skill_harness/README.md b/skill_harness/README.md deleted file mode 100644 index b658dff..0000000 --- a/skill_harness/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# skill_harness - -Runs `porting-to-ventis` against a list of repositories and records how far each -one got. `DESIGN.md` is why it is shaped this way; this file is how to run it. - -## Setup - -```shell -uv venv --python 3.12 .venv -uv pip install -e . -export AWS_BEARER_TOKEN_BEDROCK=... # required; stage 3 cannot run without it -export ANTHROPIC_API_KEY=... # required; `claude --bare` reads only this -``` - -## Run - -```shell -.venv/bin/python -m skill_harness run --repos skill_harness/repos.yaml -.venv/bin/python -m skill_harness report -``` - -Results land in `.harness/results.sqlite`; each repo's artifacts — the agent -trace, the four files it wrote, every stage's log — in `.harness//artifacts/`. -Nothing is analysed at write time, so the aggregations come later, from those -directories. - -## What each module does - -| File | Stage | Job | -|---|---|---| -| `runner.py` | — | sequences the pipeline; concurrency 2, with 6–8 serialised | -| `screen.py` | 2 | reads the repo without running it; finds the hardcoded model ids | -| `shim.py` | 3 | rewrites the `model` field on the way to Bedrock; nothing else | -| `stages.py` | 1–8 | one function per stage, each writing its own log | -| `db.py` | — | schema, and `confusion()` — validate.py's accuracy | - -## Two things to know before reading a result - -**Stages 6–8 hold a global lock.** They build images tagged `ventis-`, -bind the workflow's `api_port`, and `ventis deploy` starts its own Redis -container. Two repos cannot be in those stages at once no matter how wide -`--concurrency` is, so raising it past 2 buys less than it looks like it should. - -**Stage 5 does not gate.** `validate.py` failing does not stop the build. That is -deliberate — it is the only way to find out that a validation was wrong to block — -so `farthest_step` can read `served` on a repo whose `validate_ok` is 0. Those -rows are the interesting ones. `report` prints the resulting confusion matrix. - -## Not done yet - -GitHub search and repo selection (the list is hand-written), retry policy, and -any aggregation over `artifacts/`. Deliberately — see `DESIGN.md` section 5. diff --git a/skill_harness/__init__.py b/skill_harness/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/skill_harness/__main__.py b/skill_harness/__main__.py deleted file mode 100644 index 65a7652..0000000 --- a/skill_harness/__main__.py +++ /dev/null @@ -1,193 +0,0 @@ -"""CLI. - - python -m skill_harness run --repos skill_harness/repos.yaml - python -m skill_harness report -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import sys -from pathlib import Path - -import yaml - -from . import db, runner, shim -from .stages import Config, install_signal_handlers - -HARNESS_ROOT = Path(__file__).resolve().parent.parent -DEFAULT_WORK = HARNESS_ROOT / ".harness" -DEFAULT_DB = DEFAULT_WORK / "results.sqlite" - - -def _load_repos(path: Path) -> list[str]: - doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - return [r["repo"] if isinstance(r, dict) else r for r in doc.get("repos", [])] - - -def _dotenv(path: Path) -> dict[str, str]: - """Keys may live in the harness repo's own .env rather than the environment.""" - out: dict[str, str] = {} - if not path.is_file(): - return out - for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if line and not line.startswith("#") and "=" in line: - k, _, v = line.partition("=") - out[k.strip()] = v.strip().strip("\"'") - return out - - -def _providers(repos_file: Path) -> dict[str, shim.Provider]: - doc = yaml.safe_load(repos_file.read_text(encoding="utf-8")) or {} - config = doc.get("providers") or {} - ambient = {**_dotenv(HARNESS_ROOT / ".env"), **os.environ} - keys = { - name: ambient.get((entry or {}).get("key_env", ""), "") - for name, entry in config.items() - } - return shim.build_providers(config, keys) - - -def cmd_run(args: argparse.Namespace) -> int: - work = Path(args.work).resolve() - work.mkdir(parents=True, exist_ok=True) - repos_file = Path(args.repos).resolve() - repos = _load_repos(repos_file) - if not repos: - print(f"no repos listed in {repos_file}", file=sys.stderr) - return 2 - - install_signal_handlers() - providers = _providers(repos_file) - surfaces = frozenset(providers) - if not surfaces: - print("no provider has a key; nothing can be tested. See providers: in " - f"{repos_file}", file=sys.stderr) - return 2 - logging.info("open surfaces: %s", ", ".join(sorted(surfaces))) - shim.start(providers, port=args.shim_port) - - cfg = Config( - harness_root=HARNESS_ROOT, - work_root=work, - shim_base=f"{args.shim_host}:{args.shim_port}", - model=args.model, - effort=args.effort, - budget_usd=args.budget, - port_timeout=args.port_timeout, - stage_timeout=args.stage_timeout, - skill_sha=runner._tree_sha(HARNESS_ROOT, ".claude/skills/porting-to-ventis"), - ventis_sha=runner._tree_sha(HARNESS_ROOT, "ventis"), - disallowed_tools=args.disallowed_tools, - surfaces=surfaces, - ) - logging.info("skill %s | core %s | model %s/%s", - cfg.skill_sha[:12], cfg.ventis_sha[:12], cfg.model, cfg.effort) - - conn = db.connect(args.db) - records = runner.run_all(repos, cfg, conn, concurrency=args.concurrency) - - failed = sum(1 for r in records if r["status"] != "passed") - print(f"\n{len(records)} repos, {len(records) - failed} served, {failed} did not") - return 0 - - -def cmd_screen(args: argparse.Namespace) -> int: - """Clone and screen candidates without porting anything. - - Stage 2 is a static read, so answering "is this repo in scope" costs a - shallow clone and no agent budget. This is how the repo list gets built. - """ - import shutil - import subprocess - import tempfile - - from .screen import editable_install_available, screen as do_screen - - repos = _load_repos(Path(args.repos).resolve()) if args.repos else [] - repos += args.repo - surfaces = frozenset(_providers(Path(args.repos).resolve())) if args.repos \ - else frozenset({"openai"}) - editable = editable_install_available() - print(f"surfaces={sorted(surfaces)} editable_install={editable}\n") - - tmp = Path(tempfile.mkdtemp(prefix="screen-")) - try: - for repo in repos: - dest = tmp / runner.slug_for(repo) - r = subprocess.run(["git", "clone", "-q", "--depth", "1", repo, str(dest)], - capture_output=True, text=True, timeout=300) - if r.returncode != 0: - print(f"{repo:<62} CLONE FAILED {r.stderr.strip()[-80:]}") - continue - s = do_screen(dest, surfaces=surfaces, editable_install=editable) - verdict = "IN SCOPE" if not s.reject else s.reject - print(f"{repo:<62} root_py={s.root_py_files:<3} py={s.py_files:<4} " - f"loc={s.loc:<6} {s.framework}/{s.llm_sdk:<9} {verdict}") - finally: - shutil.rmtree(tmp, ignore_errors=True) - return 0 - - -def cmd_report(args: argparse.Namespace) -> int: - conn = db.connect(args.db) - rows = db.summary(conn) - if not rows: - print("no results yet") - return 0 - width = max(len(r["repo"]) for r in rows) - for r in rows: - v = {None: "-", 1: "pass", 0: "FAIL"}[r["validate_ok"]] - tok = f"{r['tokens_in'] or 0}/{r['tokens_out'] or 0} in {r['llm_calls'] or 0} calls" - print(f"{r['repo']:<{width}} {r['farthest_step']:<10} {r['status']:<10} " - f"validate={v:<4} {tok}") - print("\nvalidate.py against the eventual outcome (blocked rows excluded — they") - print("never put the port to the test):") - print(json.dumps(db.confusion(conn), indent=2)) - return 0 - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="skill_harness") - parser.add_argument("--db", default=str(DEFAULT_DB)) - parser.add_argument("-v", "--verbose", action="store_true") - sub = parser.add_subparsers(dest="command", required=True) - - run_p = sub.add_parser("run", help="run the pipeline over a repo list") - run_p.add_argument("--repos", default=str(HARNESS_ROOT / "skill_harness" / "repos.yaml")) - run_p.add_argument("--work", default=str(DEFAULT_WORK)) - run_p.add_argument("--concurrency", type=int, default=2) - run_p.add_argument("--shim-port", type=int, default=8300) - # Containers reach the host by a different name than the harness does. - run_p.add_argument("--shim-host", default="http://host.docker.internal") - run_p.add_argument("--model", default="opus") - run_p.add_argument("--effort", default="high") - run_p.add_argument("--budget", type=float, default=8.0) - run_p.add_argument("--port-timeout", type=int, default=3600) - run_p.add_argument("--stage-timeout", type=int, default=900) - run_p.add_argument("--disallowed-tools", default="") - run_p.set_defaults(func=cmd_run) - - scr_p = sub.add_parser("screen", help="clone and screen candidates, port nothing") - scr_p.add_argument("--repos", default=None, help="yaml list to screen") - scr_p.add_argument("repo", nargs="*", help="extra repo urls") - scr_p.set_defaults(func=cmd_screen) - - rep_p = sub.add_parser("report", help="print the results table") - rep_p.set_defaults(func=cmd_report) - - args = parser.parse_args(argv) - logging.basicConfig( - level=logging.DEBUG if args.verbose else logging.INFO, - format="%(asctime)s %(levelname)-5s %(name)-8s %(message)s", - datefmt="%H:%M:%S", - ) - return args.func(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skill_harness/db.py b/skill_harness/db.py deleted file mode 100644 index d52ce60..0000000 --- a/skill_harness/db.py +++ /dev/null @@ -1,138 +0,0 @@ -"""SQLite storage for harness runs. - -The database holds artifacts and versions, not analysis. The three SHAs are the -only things that cannot be reconstructed once a run is over; everything about -*why* a repo failed is recomputed later by reading its artifacts directory. -See DESIGN.md section 4. -""" - -from __future__ import annotations - -import json -import sqlite3 -from pathlib import Path - -# The gating stages, in order. Stage 5 (`validated`) is deliberately absent: it -# does not halt the pipeline, so it cannot be the "furthest step reached" — its -# verdict lives in tests.validate_ok instead. See DESIGN.md section 1. -STAGES = ["fetched", "screened", "wired", "ported", "built", "deployed", "served"] - -SCHEMA = """ -CREATE TABLE IF NOT EXISTS repos ( - id INTEGER PRIMARY KEY, - repo TEXT UNIQUE NOT NULL, - stars INTEGER, - framework TEXT, - is_multiagent INTEGER, - description TEXT -); - -CREATE TABLE IF NOT EXISTS tests ( - id INTEGER PRIMARY KEY, - repo TEXT NOT NULL REFERENCES repos(repo), - repo_sha TEXT NOT NULL, - skill_sha TEXT NOT NULL, - ventis_sha TEXT NOT NULL, - farthest_step TEXT NOT NULL, - status TEXT NOT NULL, - validate_ok INTEGER, - core_issue TEXT, - skill_issue TEXT, - analysis TEXT, - tokens_in INTEGER, - tokens_out INTEGER, - llm_calls INTEGER, - cost_usd REAL, - artifacts TEXT NOT NULL, - started_at TEXT NOT NULL, - ended_at TEXT -); -""" - - -# Columns added after the first databases were written. Cheap to add in place, -# and cheaper than re-running a repo to change a schema. -_ADDED_COLUMNS = { - "tests": {"tokens_in": "INTEGER", "tokens_out": "INTEGER", - "llm_calls": "INTEGER", "cost_usd": "REAL"}, -} - - -def connect(path: str | Path) -> sqlite3.Connection: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path, check_same_thread=False) - conn.row_factory = sqlite3.Row - conn.executescript(SCHEMA) - for table, columns in _ADDED_COLUMNS.items(): - have = {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")} - for name, decl in columns.items(): - if name not in have: - conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}") - conn.commit() - return conn - - -def upsert_repo(conn: sqlite3.Connection, repo: str, **fields) -> None: - """Insert the repo if new, then update whichever columns were supplied. - - Stage 2 learns most of these, so a repo row is written twice: once empty at - fetch time, once populated after the screen. - """ - with conn: - conn.execute("INSERT OR IGNORE INTO repos (repo) VALUES (?)", (repo,)) - known = {"stars", "framework", "is_multiagent", "description"} - cols = {k: v for k, v in fields.items() if k in known and v is not None} - if cols: - assigns = ", ".join(f"{k} = ?" for k in cols) - conn.execute( - f"UPDATE repos SET {assigns} WHERE repo = ?", - (*cols.values(), repo), - ) - - -def record_test(conn: sqlite3.Connection, **fields) -> int: - for key in ("core_issue", "skill_issue"): - if isinstance(fields.get(key), (list, dict)): - fields[key] = json.dumps(fields[key]) - cols = ", ".join(fields) - marks = ", ".join("?" for _ in fields) - with conn: - cur = conn.execute( - f"INSERT INTO tests ({cols}) VALUES ({marks})", tuple(fields.values()) - ) - return cur.lastrowid - - -def summary(conn: sqlite3.Connection) -> list[sqlite3.Row]: - return conn.execute( - """ - SELECT repo, farthest_step, status, validate_ok, - tokens_in, tokens_out, llm_calls, artifacts - FROM tests ORDER BY id - """ - ).fetchall() - - -def confusion(conn: sqlite3.Connection) -> dict[str, int]: - """validate.py's confusion matrix — the point of not letting stage 5 gate. - - A false negative is a check validate.py is missing. A false positive is a - check that was wrong to block, and is only observable because the build ran - anyway. - - `blocked` rows are excluded. A repo stopped by its own missing backing - service never put the port to the test, so counting it as a validation miss - would blame validate.py for a vector store nobody configured. - """ - rows = conn.execute( - "SELECT validate_ok, farthest_step FROM tests " - "WHERE validate_ok IS NOT NULL AND status != 'blocked'" - ).fetchall() - served = lambda r: r["farthest_step"] == "served" # noqa: E731 - return { - "true_positive": sum(1 for r in rows if not r["validate_ok"] and not served(r)), - "false_positive": sum(1 for r in rows if not r["validate_ok"] and served(r)), - "false_negative": sum(1 for r in rows if r["validate_ok"] and not served(r)), - "true_negative": sum(1 for r in rows if r["validate_ok"] and served(r)), - } diff --git a/skill_harness/repos.yaml b/skill_harness/repos.yaml deleted file mode 100644 index 93e7178..0000000 --- a/skill_harness/repos.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# The repos under test, and which provider surfaces are open for them. -# -# Hand-supplied while the pipeline is being proven. GitHub search and automated -# selection are deliberately out of scope until the failure modes are known — -# see DESIGN.md section 5. - -# Screened in scope with `python -m skill_harness screen`. -repos: - - https://github.com/yeesimonwong/langGraph-agent-openai - -# Screened and rejected, kept so the reasons are not rediscovered. Note how few -# survive an OpenAI-only credential: langchain-ai's own templates reach their -# chat model through init_chat_model("anthropic/...") and default to Claude. -# -# react-agent, memory-agent, data-enrichment, langgraph-example, -# rag-research-agent-template anthropic, or both -- closed without a repo key -# retrieval-agent-template both, and needs an Elasticsearch/Pinecone store -# langchain-academy no packaging metadata, no root module (M24) -# rag-from-scratch notebooks only, no .py at all -# new-langgraph-project a skeleton; makes no LLM call -# pharmaceutical-supply-chain-... needs MongoDB -# ai-support-agent needs a database -# Content-Generation-AI-Agent needs MongoDB and Supabase - -# Which provider surfaces are open. A provider whose key is missing is left out, -# and the screen then rejects any repo that needs it -- at a shallow clone, -# rather than after an agent has been paid to port it. -# -# Repos keep their own provider and their own model ids: the shim swaps the key -# in and forwards everything else untouched, so nothing here swaps a provider -# (M20). `rewrite` exists only for a model id that cannot be served as written, -# and is empty on purpose. -providers: - openai: - key_env: OPENAI_KEY - - # Deliberately not ANTHROPIC_API_KEY: that one runs the porting agent, and - # sharing it would blur agent spend with repo spend and let a runaway repo - # exhaust the porting budget. Set REPO_ANTHROPIC_API_KEY to open this surface. - anthropic: - key_env: REPO_ANTHROPIC_API_KEY diff --git a/skill_harness/runner.py b/skill_harness/runner.py deleted file mode 100644 index 01f192d..0000000 --- a/skill_harness/runner.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Orchestration. - -Stages 1-5 run concurrently across repos. Stages 6-8 take a global lock: they -build images tagged `ventis-`, bind the workflow's api_port, and -`ventis deploy` starts its own Redis container, so two repos cannot be in them at -once regardless of how wide the pool is. -""" - -from __future__ import annotations - -import json -import logging -import re -import subprocess -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timezone -from pathlib import Path - -from . import db, shim, stages -from .stages import Config, Ctx, Result - -log = logging.getLogger("runner") - -# (stage name, function, gating). Stage 5 is the one non-gating stage: a failed -# validate.py is recorded and the pipeline continues, which is the only way to -# observe a validation that was wrong to block. See DESIGN.md section 1. -PIPELINE = [ - ("fetched", stages.fetch, True), - ("screened", stages.screen, True), - ("wired", stages.wire, True), - ("ported", stages.port, True), - ("validated", stages.validate, False), - ("built", stages.build, True), - ("deployed", stages.deploy, True), - ("served", stages.serve, True), -] - -DOCKER_STAGES = {"built", "deployed", "served"} - -_SLUG = re.compile(r"[^a-z0-9]+") - - -def slug_for(repo: str) -> str: - return _SLUG.sub("-", repo.rstrip("/").split("/")[-1].removesuffix(".git").lower()).strip("-") - - -def _tree_sha(root: Path, path: str) -> str: - """The git tree hash of a subdirectory — it changes when that subtree changes - and not when anything else in the repo does, which is exactly what pinning - the skill and the core each require.""" - try: - out = subprocess.run(["git", "rev-parse", f"HEAD:{path}"], cwd=root, - capture_output=True, text=True, timeout=30) - return out.stdout.strip() or "unknown" - except Exception: - return "unknown" - - -def _classify(stage: str, result: Result, ctx: Ctx) -> str: - if stage == "screened": - return "blocked" # out of scope for this run, not a skill failure - if stage == "wired": - return "blocked" # missing credential, nothing was tested - if stage == "served" and ctx.missing_credential: - # The port served a real request far enough to run the source, which - # then asked for a credential nobody gave it. Nothing about the skill or - # about Ventis failed here. - return "blocked" - if stage == "ported": - if ctx.reported_and_stopped: - # The skill told the agent to report rather than fix, and it did. - # Scoring this as a failure would count the skill working as the - # skill failing, and would bury the Ventis gap that caused it. - return "blocked" - trace = ctx.log_path("4-port.log") - text = trace.read_text(encoding="utf-8", errors="replace") if trace.is_file() else "" - if "budget" in text.lower() and "exceed" in text.lower(): - return "budget_exhausted" - if "timed out" in result.detail or "timed out" in text: - return "timeout" - return "failed" - - -def run_repo(repo: str, cfg: Config, conn, docker_lock: threading.Lock) -> dict: - slug = slug_for(repo) - artifacts = cfg.work_root / slug / "artifacts" - artifacts.mkdir(parents=True, exist_ok=True) - ctx = Ctx(repo=repo, slug=slug, root=cfg.work_root / slug / "src", - artifacts=artifacts, cfg=cfg) - - started = datetime.now(timezone.utc).isoformat(timespec="seconds") - farthest, status = "none", "passed" - began = time.time() - - try: - for stage, fn, gating in PIPELINE: - if stage in DOCKER_STAGES: - docker_lock.acquire() - try: - result = fn(ctx) - except Exception as e: # a harness bug, not a port failure - log.exception("%s: %s crashed", slug, stage) - result = Result(False, f"harness error: {type(e).__name__}: {e}") - finally: - if stage in DOCKER_STAGES: - docker_lock.release() - - mark = "ok " if result.ok else "FAIL" - log.info("%-24s %-10s %s %s", slug, stage, mark, result.detail) - - if result.ok: - if gating: - farthest = stage - elif gating: - status = _classify(stage, result, ctx) - break - finally: - stages.teardown(ctx) - - usage = shim.usage_for(slug) - (artifacts / "usage.json").write_text(json.dumps(usage, indent=2), encoding="utf-8") - - if ctx.screen: - db.upsert_repo(conn, repo, framework=ctx.screen.framework, - is_multiagent=int(ctx.screen.is_multiagent), - description=ctx.screen.description) - else: - db.upsert_repo(conn, repo) - - record = dict( - repo=repo, - repo_sha=ctx.repo_sha or "unknown", - skill_sha=cfg.skill_sha, - ventis_sha=cfg.ventis_sha, - farthest_step=farthest, - status=status, - validate_ok=None if ctx.validate_ok is None else int(ctx.validate_ok), - core_issue=ctx.core_issue or None, - skill_issue=ctx.skill_issue or None, - analysis=(f"served, but the project's own logic errored: {ctx.app_error}" - if ctx.app_error else None), - tokens_in=usage["input"], - tokens_out=usage["output"], - llm_calls=usage["calls"], - cost_usd=None, - artifacts=str(artifacts), - started_at=started, - ended_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), - ) - db.record_test(conn, **record) - log.info("%-24s => %s at %s (%.0fs)", slug, status, farthest, time.time() - began) - return record - - -def run_all(repos: list[str], cfg: Config, conn, concurrency: int = 2) -> list[dict]: - docker_lock = threading.Lock() - with ThreadPoolExecutor(max_workers=concurrency) as pool: - futures = [pool.submit(run_repo, r, cfg, conn, docker_lock) for r in repos] - return [f.result() for f in futures] diff --git a/skill_harness/screen.py b/skill_harness/screen.py deleted file mode 100644 index d056869..0000000 --- a/skill_harness/screen.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Stage 2 — read the repo without running it. - -Two jobs. It decides whether the repo is in scope for this run, and it finds the -hardcoded model ids that stage 3 has to teach the shim about, because that is the -one thing an environment variable cannot reach (DESIGN.md section 3). -""" - -from __future__ import annotations - -import ast -import re -from dataclasses import dataclass, field -from pathlib import Path - -# `.claude` is here because the harness copies the skill under test into the repo -# at stage 4. Screening a tree that has already been through a run would -# otherwise read validate.py's own imports as the repo's. -SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox", - "build", "dist", ".claude"} - -FRAMEWORK_MARKERS = [ - ("langgraph", ("langgraph",)), - ("langchain", ("langchain", "langchain_core", "langchain_community")), - ("crewai", ("crewai",)), - ("autogen", ("autogen", "autogen_agentchat")), - ("adk", ("google.adk", "google_adk")), -] - -# Ordered: the first two decide which base URL stage 3 writes, so they must be -# recognised by more than their own package name — a repo commonly reaches a -# provider through a wrapper, and matching only `openai`/`anthropic` misses it. -SDK_MARKERS = [ - ("openai", ("openai", "langchain_openai", "llama_index.llms.openai")), - ("anthropic", ("anthropic", "langchain_anthropic", "llama_index.llms.anthropic")), - ("bedrock", ("boto3", "botocore", "langchain_aws", "ventis.llm", "ventis")), - # Reaches a model, but not through a provider SDK we can redirect by env var. - ("other", ("litellm", "instructor", "google.generativeai", "google.genai", - "cohere", "mistralai", "ollama", "langchain.chat_models")), -] - -# Model ids as they appear in source. A literal this matches is a candidate for -# the shim's mapping table, which a human reads before the run — so a miss costs -# a stage 8 provider error, and a false match costs that human's attention. -MODEL_LITERAL = re.compile( - r"\b(gpt-[\w.\-]+|o[134](?:-[\w.\-]+)?|claude-[\w.\-]+|" - # A vendor-prefixed Bedrock id ends in a version marker (`-v1:0`, `-1:0`, - # `-2507`). Requiring one keeps `meta.com` and other hostnames out; without - # it the list a human reads to build the model map fills with domains. - r"(?:meta|mistral|amazon|cohere|anthropic|openai|qwen|deepseek)" - r"\.[\w.\-]*(?:v?\d+(?::\d+)?|\d{4}))\b" -) - -# LangChain's `init_chat_model` picks its provider at runtime from a -# "/" string, so a repo can depend entirely on Anthropic while -# importing nothing named anthropic. Every LangGraph template is built this way -# and most default to Claude -- reading only the imports classifies them as -# having no provider at all. -PROVIDER_STRING = re.compile( - r"[\"']((?:anthropic|openai|google_genai|google_vertexai|bedrock|bedrock_converse|" - r"cohere|mistralai|fireworks|groq|ollama|together|deepseek|xai)[:/][\w.\-:]+)[\"']" -) - -MULTIAGENT_MARKERS = ("Send(", "StateGraph", "Crew(", "GroupChat", "add_edge", "Command(") - -# Backing services the harness does not stand up. A repo that reads one of these -# gets as far as a real request and then fails on a credential -- which is a fact -# about the repo's dependencies, not about the port, and costs a whole agent -# budget to discover. Redis is absent from the list: ventis deploy provides it. -EXTERNAL_SERVICE_VARS = re.compile( - r"\b(ELASTICSEARCH_(?:URL|API_KEY|USER|PASSWORD)|PINECONE_[A-Z_]+|MONGODB_[A-Z_]+|" - r"WEAVIATE_[A-Z_]+|QDRANT_[A-Z_]+|CHROMA_[A-Z_]+|SUPABASE_[A-Z_]+|" - r"TAVILY_[A-Z_]+|SERPAPI_[A-Z_]+|EXA_API_KEY|FIRECRAWL_[A-Z_]+|" - r"LANGSMITH_[A-Z_]+|POSTGRES_[A-Z_]+|DATABASE_URL)\b" -) - - -@dataclass -class Screen: - py_files: int = 0 - root_py_files: int = 0 - loc: int = 0 - framework: str = "plain" - llm_sdk: str = "none" - model_ids: list[str] = field(default_factory=list) - is_multiagent: bool = False - external_services: list[str] = field(default_factory=list) - # "/" literals -- what init_chat_model resolves at runtime. - provider_hints: list[str] = field(default_factory=list) - layout: str = "flat" - packaging: str = "none" - description: str = "" - reject: str | None = None - - -def _imports(tree: ast.AST) -> set[str]: - names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - names.update(a.name for a in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - names.add(node.module) - return names - - -def _matches(imports: set[str], markers: tuple[str, ...]) -> bool: - return any(i == m or i.startswith(m + ".") for i in imports for m in markers) - - -def editable_install_available() -> bool: - """Whether the Ventis under test can install the source as a package. - - Asked of the code rather than assumed, the same way validate.py asks it. - Without it, M24 holds in its strict form: only modules that land flat at - /app import at all, and packaging metadata rescues nothing. - """ - try: - from ventis import stub_generator - except Exception: # noqa: BLE001 - a missing install must not crash the screen - return False - return hasattr(stub_generator, "_install_step") - - -def screen(root: Path, max_py_files: int = 200, max_loc: int = 40_000, - surfaces: frozenset[str] = frozenset({"openai", "anthropic"}), - editable_install: bool | None = None) -> Screen: - """`surfaces` is which Bedrock wire formats the credential can actually reach. - - It is an account property, not a property of Bedrock: a Messages-API model - can be listed by the control plane and still answer `permission_error`. A - repo whose SDK needs a closed surface is rejected here rather than after an - agent has spent its budget porting it. - - `editable_install` is the matching question for M24, asked of the Ventis under - test rather than assumed. - """ - if editable_install is None: - editable_install = editable_install_available() - out = Screen() - imports: set[str] = set() - models: set[str] = set() - hints: set[str] = set() - services: set[str] = set() - - for path in root.rglob("*.py"): - if SKIP_DIRS & set(path.relative_to(root).parts): - continue - out.py_files += 1 - if path.parent == root: - out.root_py_files += 1 - try: - text = path.read_text(encoding="utf-8", errors="replace") - except OSError: - continue - out.loc += text.count("\n") - models.update(MODEL_LITERAL.findall(text)) - hints.update(PROVIDER_STRING.findall(text)) - services.update(EXTERNAL_SERVICE_VARS.findall(text)) - if any(m in text for m in MULTIAGENT_MARKERS): - out.is_multiagent = True - try: - imports |= _imports(ast.parse(text)) - except SyntaxError: - # Not fatal to the screen. validate.py's V001/V002 is where a file - # that does not parse becomes a finding about the port. - continue - - out.model_ids = sorted(models) - out.provider_hints = sorted(hints) - # LangSmith is observability, not a dependency the agent needs to answer. - out.external_services = sorted(s for s in services if not s.startswith("LANGSMITH_")) - - for name, markers in FRAMEWORK_MARKERS: - if _matches(imports, markers): - out.framework = name - break - - hits = [name for name, markers in SDK_MARKERS if _matches(imports, markers)] - - # Both signals count, and neither may short-circuit the other. A repo can - # import langchain_openai for its embeddings while its chat model comes from - # init_chat_model("anthropic/..."), and letting the import win would report - # such a repo as openai-only and send it to an agent it cannot finish. - named = {h.split("/")[0].split(":")[0] for h in out.provider_hints} - redirectable = (set(hits) | named) & {"openai", "anthropic"} - if len(redirectable) == 2: - out.llm_sdk = "both" - elif redirectable: - out.llm_sdk = redirectable.pop() - elif hits: - out.llm_sdk = hits[0] - - # `flat` means the port can import the source, which is a fact about where - # modules sit relative to the project root -- not about whether a `src/` - # directory happens to exist. - if out.root_py_files: - out.layout = "flat" - elif (root / "src").is_dir(): - out.layout = "src" - else: - out.layout = "nested" - for candidate in ("pyproject.toml", "setup.py", "setup.cfg"): - if (root / candidate).is_file(): - out.packaging = candidate - break - - readme = next((p for p in root.glob("README*") if p.is_file()), None) - if readme: - body = readme.read_text(encoding="utf-8", errors="replace").strip().splitlines() - out.description = " ".join(line for line in body[:12] if line.strip())[:500] - - # Rejections. Each is a fact about scope, not a failure of the skill, so the - # test row records `screened` as the furthest step and stops there. - if out.py_files == 0: - out.reject = "no python files" - elif out.py_files > max_py_files or out.loc > max_loc: - out.reject = f"too large: {out.py_files} files, {out.loc} loc" - elif out.llm_sdk == "none" and not out.model_ids: - # Both signals absent. One alone is not enough to reject on: a wrapper - # hides the SDK, and a model id read from config leaves no literal. - out.reject = "no LLM call found" - elif out.llm_sdk in ("openai", "anthropic") and out.llm_sdk not in surfaces: - out.reject = f"{out.llm_sdk} surface unavailable on this credential" - elif out.llm_sdk == "both" and not {"openai", "anthropic"} <= surfaces: - out.reject = "needs both surfaces; only " + ",".join(sorted(surfaces)) - elif out.root_py_files == 0 and not editable_install: - # M24 in its strict form. With no editable install, an adapter can import - # only what lands flat at /app, so a tree whose modules all sit under - # sub-directories has no port this Ventis can load -- whatever its - # packaging says. Deciding it here costs nothing; letting it through - # costs an agent's whole budget to reach the same conclusion. - out.reject = (f"no module at the project root ({out.py_files} .py files, " - f"all nested) and no editable install (M24)") - elif out.external_services: - out.reject = ("needs backing services this harness does not provide: " - + ", ".join(out.external_services[:4])) - elif out.root_py_files == 0 and out.packaging == "none": - # The editable install exists, but nothing tells it what the root is. - out.reject = "no module at the project root and no packaging metadata (M24)" - - return out diff --git a/skill_harness/shim.py b/skill_harness/shim.py deleted file mode 100644 index 4d59b90..0000000 --- a/skill_harness/shim.py +++ /dev/null @@ -1,227 +0,0 @@ -"""A pass-through proxy in front of the model providers. - -Each repo is pointed at `/r///...` by its own `.env`, and the shim -forwards the request upstream unchanged except for the API key, which it swaps in -so the real credential never reaches the repo or its image. - -It rewrites nothing else. Repos keep their own provider and their own model ids, -which is what M20 asks for; the shim exists for the two things a direct -connection cannot give — **per-repo token accounting**, which feeds the results -table, and one place that sees which models a repo actually calls. - -Optional `rewrite` rules cover the case where a model id cannot be served as -written. Leave them out and the request passes through untouched. - -Requests are buffered, not streamed, matching the scope llm_proxy already set. -When PR #54 lands this should fold into `llm_proxy/providers/`. -""" - -from __future__ import annotations - -import json -import logging -import re -import threading -import urllib.error -import urllib.request -from collections import defaultdict -from dataclasses import dataclass, field -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -log = logging.getLogger("shim") - -# Per-repo token accounting, keyed by the slug in the request path. -USAGE: dict[str, dict[str, int]] = defaultdict(lambda: {"input": 0, "output": 0, "calls": 0}) -_USAGE_LOCK = threading.Lock() - -# How each provider's wire protocol carries its credential. The upstream default -# is the provider's own API; pointing it elsewhere — a gateway, or Bedrock's -# compatible surfaces — is a config change rather than a code change. -PROTOCOLS = { - "openai": { - "upstream": "https://api.openai.com/v1", - "auth_header": "Authorization", - "auth_template": "Bearer {key}", - "extra_headers": {}, - }, - "anthropic": { - "upstream": "https://api.anthropic.com", - "auth_header": "x-api-key", - "auth_template": "{key}", - "extra_headers": {"anthropic-version": "2023-06-01"}, - }, -} - -_PATH = re.compile(r"^/r/(?P[\w.\-]+)/(?P[\w\-]+)(?P/.*)$") - -# Headers that describe the hop, not the request. Forwarding them corrupts the -# upstream call. -_HOP_BY_HOP = {"host", "content-length", "connection", "authorization", "x-api-key", - "accept-encoding", "transfer-encoding"} - - -@dataclass -class Provider: - """One open provider surface. A provider absent from the registry is closed.""" - - name: str - key: str - upstream: str - auth_header: str - auth_template: str - extra_headers: dict = field(default_factory=dict) - # Optional model-id substitutions. Empty means pass through unchanged. - rewrite_exact: dict = field(default_factory=dict) - rewrite_prefixes: list = field(default_factory=list) - _seen: dict = field(default_factory=dict) - - def resolve(self, model: str) -> str: - target = self.rewrite_exact.get(model) or next( - (dst for pre, dst in self.rewrite_prefixes if model.startswith(pre)), model - ) - if self._seen.get(model) != target: - self._seen[model] = target - log.info("%s: %s%s", self.name, model, - "" if target == model else f" -> {target}") - return target - - -def build_providers(config: dict, keys: dict[str, str]) -> dict[str, Provider]: - """Assemble the open providers from config plus the keys actually present. - - A provider with no key is left out rather than half-configured: the screen - reads the same registry, so a missing key becomes a stage 2 rejection instead - of a failure at the first request, after an agent has been paid for. - """ - out: dict[str, Provider] = {} - for name, entry in (config or {}).items(): - proto = PROTOCOLS.get(name) - if proto is None: - log.warning("unknown provider %r in config; ignored", name) - continue - key = keys.get(name, "") - if not key: - log.info("provider %s has no key; that surface stays closed", name) - continue - rewrite = (entry or {}).get("rewrite") or {} - out[name] = Provider( - name=name, - key=key, - upstream=(entry or {}).get("upstream") or proto["upstream"], - auth_header=proto["auth_header"], - auth_template=proto["auth_template"], - extra_headers=dict(proto["extra_headers"]), - rewrite_exact=rewrite.get("exact") or {}, - rewrite_prefixes=[(r["prefix"], r["to"]) for r in rewrite.get("prefixes") or []], - ) - return out - - -def _account(slug: str, raw: bytes) -> None: - try: - usage = json.loads(raw).get("usage") or {} - except (json.JSONDecodeError, AttributeError): - return - # OpenAI names them prompt/completion; Anthropic input/output. - inp = usage.get("input_tokens") or usage.get("prompt_tokens") or 0 - out = usage.get("output_tokens") or usage.get("completion_tokens") or 0 - with _USAGE_LOCK: - bucket = USAGE[slug] - bucket["input"] += inp - bucket["output"] += out - bucket["calls"] += 1 - - -def _handler_class(providers: dict[str, Provider], timeout: float): - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def log_message(self, fmt, *args): - log.debug(fmt, *args) - - def _send(self, code: int, body: bytes): - self.send_response(code) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def _fail(self, code: int, detail: str): - self._send(code, json.dumps({"error": "shim_error", "detail": detail}).encode()) - - def do_GET(self): - if self.path == "/healthz": - return self._send(200, json.dumps( - {"status": "ok", "providers": sorted(providers)}).encode()) - self._proxy(b"") - - def do_POST(self): - length = int(self.headers.get("Content-Length") or 0) - self._proxy(self.rfile.read(length) if length else b"") - - def _proxy(self, body: bytes): - match = _PATH.match(self.path) - if not match: - return self._fail(404, f"unroutable path {self.path!r}") - slug, name, rest = match["slug"], match["provider"], match["rest"] - - provider = providers.get(name) - if provider is None: - # Worth a warning, not just a reply: a repo reaching a closed - # provider means stage 2 let something through that it should - # have rejected, and that is a screen defect to go and fix. - log.warning("%s reached the closed %s surface", slug, name) - return self._fail(503, f"the {name} surface is closed on this harness; " - f"no key is configured for it") - - if body: - try: - payload = json.loads(body) - except json.JSONDecodeError: - payload = None - if isinstance(payload, dict) and "model" in payload: - payload["model"] = provider.resolve(payload["model"]) - body = json.dumps(payload).encode() - - headers = {k: v for k, v in self.headers.items() - if k.lower() not in _HOP_BY_HOP} - headers[provider.auth_header] = provider.auth_template.format(key=provider.key) - for k, v in provider.extra_headers.items(): - headers.setdefault(k, v) - - req = urllib.request.Request( - f"{provider.upstream}{rest}", data=body or None, - headers=headers, method=self.command, - ) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - status, raw = resp.status, resp.read() - except urllib.error.HTTPError as e: - status, raw = e.code, e.read() - except Exception as e: # network, DNS, timeout - return self._fail(502, f"{type(e).__name__}: {e}") - - _account(slug, raw) - self._send(status, raw) - - return Handler - - -def start(providers: dict[str, Provider], host: str = "0.0.0.0", port: int = 8300, - timeout: float = 600.0) -> ThreadingHTTPServer: - """Start the shim on a daemon thread and return the server. - - It binds 0.0.0.0 because the callers are agent containers, which reach the - host by a different address than the harness does. - """ - server = ThreadingHTTPServer((host, port), _handler_class(providers, timeout)) - threading.Thread(target=server.serve_forever, daemon=True, name="shim").start() - log.info("shim on %s:%s -> %s", host, port, - ", ".join(f"{n}={p.upstream}" for n, p in sorted(providers.items())) or "(nothing open)") - return server - - -def usage_for(slug: str) -> dict[str, int]: - with _USAGE_LOCK: - return dict(USAGE[slug]) diff --git a/skill_harness/stages.py b/skill_harness/stages.py deleted file mode 100644 index d10caa7..0000000 --- a/skill_harness/stages.py +++ /dev/null @@ -1,526 +0,0 @@ -"""The eight stages. - -Only stage 4 runs an agent. Everything else is a deterministic subprocess with a -timeout, which is what makes a stage 6 failure a fact about the port rather than -about how the agent happened to behave that day. See DESIGN.md section 1. - -Every stage writes its own log into the test's artifacts directory. Those logs, -plus the agent trace and the four written files, are what every later analysis -reads — the database deliberately stores none of it. -""" - -from __future__ import annotations - -import atexit -import json -import logging -import os -import re -import shutil -import signal -import subprocess -import sys -import threading -import time -import urllib.error -import urllib.request -from dataclasses import dataclass, field -from pathlib import Path - -import yaml - -from . import screen as screen_mod - -log = logging.getLogger("stages") - -SKILL_REL = ".claude/skills/porting-to-ventis" - -PORT_PROMPT = """\ -Port the project in this directory onto Ventis. - -Use the porting-to-ventis skill. Follow it; it is the thing under test. - -Do not ask for confirmation — there is nobody to answer. When the skill tells you -to report something and stop rather than fix it, write the report to -PORT_REPORT.md in this directory and stop, which counts as following it. -""" - - -@dataclass -class Result: - ok: bool - detail: str = "" - - -@dataclass -class Ctx: - repo: str - slug: str - root: Path - artifacts: Path - cfg: "Config" - repo_sha: str = "" - screen: screen_mod.Screen | None = None - validate_ok: bool | None = None - core_issue: list = field(default_factory=list) - skill_issue: list = field(default_factory=list) - # The skill's "report rather than fix" paths -- the credential wall, the - # import root, a dependency mismatch -- are all Ventis limitations. An agent - # that takes one has followed the skill, so this is not a port failure and - # must not be scored as one. - reported_and_stopped: bool = False - # An env var the repo needed and the harness never supplied. - missing_credential: str | None = None - # The port served a result and the source's own logic failed inside it. - # Not a port defect, but it means the run proved less than "served" suggests. - app_error: str | None = None - _procs: list = field(default_factory=list) - - def log_path(self, name: str) -> Path: - return self.artifacts / name - - -@dataclass -class Config: - harness_root: Path - work_root: Path - shim_base: str # what a *container* uses to reach the shim - model: str - effort: str - budget_usd: float - port_timeout: int - stage_timeout: int - # The two pins that cannot be reconstructed after a run: which skill and - # which core produced the result. Git tree hashes, so each moves only when - # its own subtree does. - skill_sha: str = "unknown" - ventis_sha: str = "unknown" - disallowed_tools: str = "" - # Which Bedrock wire formats this credential can actually reach. An account - # property, measured rather than assumed — see README. - surfaces: frozenset = frozenset({"openai"}) - - -# --------------------------------------------------------------------------- # -# helpers -# --------------------------------------------------------------------------- # - -# The harness runs under its own virtualenv, but launching it does not put that -# venv's bin on PATH. Without this the `ventis` CLI is simply not found -- stage 6 -# exits 127 and the run records a harness setup fault as a defect in the port -- -# and the agent in stage 4 has no interpreter that can import ventis, so it goes -# looking for one outside the tree under test. -_BIN = str(Path(sys.executable).parent) - - -def subprocess_env(extra: dict | None = None) -> dict: - env = {**os.environ, **(extra or {})} - path = env.get("PATH", "") - if _BIN not in path.split(os.pathsep): - env["PATH"] = _BIN + os.pathsep + path - env.setdefault("VIRTUAL_ENV", str(Path(_BIN).parent)) - return env - - -# Every child the harness has started, so none of them outlives it. A killed -# harness used to orphan its agents: they kept running, kept spending their -# budget, and kept calling the shim of whatever run started next. -_LIVE: set[subprocess.Popen] = set() -_LIVE_LOCK = threading.Lock() - - -def kill_children(sig=signal.SIGTERM) -> int: - with _LIVE_LOCK: - procs = [p for p in _LIVE if p.poll() is None] - for proc in procs: - try: - os.killpg(os.getpgid(proc.pid), sig) - except Exception: # noqa: BLE001 - already gone, or not ours any more - pass - return len(procs) - - -def install_signal_handlers() -> None: - """Take the children down with us, however we are asked to stop.""" - def _handler(signum, _frame): - n = kill_children(signal.SIGTERM) - log.warning("signal %s: terminated %d child process(es)", signum, n) - raise SystemExit(128 + signum) - - for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): - try: - signal.signal(sig, _handler) - except ValueError: # not on the main thread - pass - atexit.register(kill_children) - - -def run(ctx: Ctx, name: str, cmd: list[str], *, cwd: Path | None = None, - timeout: int | None = None, env: dict | None = None) -> tuple[int, str]: - """Run a subprocess, tee its output into the artifacts directory, return it.""" - timeout = timeout or ctx.cfg.stage_timeout - full_env = subprocess_env(env) - log.debug("%s: %s", name, " ".join(cmd)) - proc = None - try: - # Its own process group, so a timeout or a signal reaches the whole tree - # rather than just the process the harness happens to hold. - proc = subprocess.Popen( - cmd, cwd=cwd or ctx.root, env=full_env, start_new_session=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - ) - with _LIVE_LOCK: - _LIVE.add(proc) - out, _ = proc.communicate(timeout=timeout) - rc = proc.returncode - except subprocess.TimeoutExpired: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - out, _ = proc.communicate() - rc = 124 - out = (out or "") + f"\n[harness] timed out after {timeout}s\n" - except FileNotFoundError as e: - rc, out = 127, f"[harness] {e}\n" - finally: - if proc is not None: - with _LIVE_LOCK: - _LIVE.discard(proc) - ctx.log_path(f"{name}.log").write_text(out or "", encoding="utf-8") - return rc, out or "" - - -def _config_path(root: Path) -> Path: - return root / "config" / "global_controller.yaml" - - -def _load_yaml(path: Path) -> dict: - try: - return yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except (OSError, yaml.YAMLError): - return {} - - -def _agent_entries(root: Path) -> list[dict]: - return _load_yaml(_config_path(root)).get("agents") or [] - - -def _api_port(root: Path, default: int = 8080) -> int: - for entry in _agent_entries(root): - if entry.get("type") == "workflow": - return int(entry.get("api_port", default)) - return default - - -# --------------------------------------------------------------------------- # -# 1. fetched -# --------------------------------------------------------------------------- # - -def fetch(ctx: Ctx) -> Result: - if ctx.root.exists(): - shutil.rmtree(ctx.root) - ctx.root.parent.mkdir(parents=True, exist_ok=True) - rc, out = run(ctx, "1-fetch", ["git", "clone", "--depth", "1", ctx.repo, str(ctx.root)], - cwd=ctx.root.parent) - if rc != 0: - return Result(False, f"clone failed: {out.strip()[-300:]}") - rc, sha = run(ctx, "1-sha", ["git", "rev-parse", "HEAD"]) - ctx.repo_sha = sha.strip() if rc == 0 else "unknown" - return Result(True, ctx.repo_sha[:12]) - - -# --------------------------------------------------------------------------- # -# 2. screened -# --------------------------------------------------------------------------- # - -def screen(ctx: Ctx) -> Result: - ctx.screen = screen_mod.screen(ctx.root, surfaces=ctx.cfg.surfaces) - ctx.log_path("2-screen.json").write_text( - json.dumps(ctx.screen.__dict__, indent=2, default=str), encoding="utf-8" - ) - if ctx.screen.reject: - return Result(False, ctx.screen.reject) - return Result(True, f"{ctx.screen.framework}/{ctx.screen.llm_sdk}, {ctx.screen.loc} loc") - - -# --------------------------------------------------------------------------- # -# 3. wired -# --------------------------------------------------------------------------- # - -# A repo's SDK refuses to send without *a* key, so it gets a placeholder; the -# shim replaces it with the real one on the way out. No real credential is -# written into the repo, and none enters the image. -_PLACEHOLDER = "supplied-by-the-shim" - - -def wire(ctx: Ctx) -> Result: - """Write the .env the port will point `env_file:` at. - - The source tree is never edited; this adds a file beside it, which is what - the skill's own credential path expects (M18, M23). Only the surfaces this - harness can actually serve are written -- pointing a repo at a base URL that - answers 503 would be worse than leaving it unset, because the failure would - read as the repo's rather than the harness's. - """ - if not ctx.cfg.surfaces: - return Result(False, "no provider surface is open") - - base = f"{ctx.cfg.shim_base}/r/{ctx.slug}" - lines = [ - "# Written by skill_harness. The source tree is untouched; this file is", - "# what `env_file:` in the port's config points at. The keys are", - "# placeholders: the shim swaps the real ones in as requests pass.", - ] - if "openai" in ctx.cfg.surfaces: - lines += [f"OPENAI_BASE_URL={base}/openai", f"OPENAI_API_KEY={_PLACEHOLDER}"] - if "anthropic" in ctx.cfg.surfaces: - lines += [f"ANTHROPIC_BASE_URL={base}/anthropic", - f"ANTHROPIC_API_KEY={_PLACEHOLDER}"] - lines.append("") - - env = "\n".join(lines) - (ctx.root / ".env").write_text(env, encoding="utf-8") - ctx.log_path("3-wire.log").write_text(env, encoding="utf-8") - return Result(True, f"{'+'.join(sorted(ctx.cfg.surfaces))} via {base}") - - -# --------------------------------------------------------------------------- # -# 4. ported — the only stage that runs an agent -# --------------------------------------------------------------------------- # - -def port(ctx: Ctx) -> Result: - """Copy the skill in, then run `claude -p` against the repo. - - The skill is delivered explicitly rather than relied on globally, so the - version under test is the version recorded in tests.skill_sha. - """ - src = ctx.cfg.harness_root / SKILL_REL - dst = ctx.root / SKILL_REL - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(src, dst, dirs_exist_ok=True) - - cmd = [ - "claude", "-p", PORT_PROMPT, - "--bare", - "--setting-sources", "", - "--permission-mode", "bypassPermissions", - "--output-format", "stream-json", "--verbose", - "--model", ctx.cfg.model, - "--effort", ctx.cfg.effort, - "--max-budget-usd", str(ctx.cfg.budget_usd), - "--no-session-persistence", - ] - if ctx.cfg.disallowed_tools: - cmd += ["--disallowedTools", ctx.cfg.disallowed_tools] - - rc, out = run(ctx, "4-port", cmd, timeout=ctx.cfg.port_timeout) - # The stream-json trace is the only record of *how* the agent got there. - ctx.log_path("4-port.trace.jsonl").write_text(out, encoding="utf-8") - - report = ctx.root / "PORT_REPORT.md" - if report.is_file(): - # Filed as a core issue: every "report and stop" the skill defines is - # triggered by something Ventis cannot do, so it is a finding with a - # Ventis owner. The full text stays in artifacts; this is the grouping key. - ctx.reported_and_stopped = True - ctx.core_issue.append({"kind": "reported_and_stopped", - "text": report.read_text(encoding="utf-8")[:4000]}) - - if rc != 0: - return Result(False, f"claude exited {rc}") - - if not _config_path(ctx.root).is_file(): - if ctx.reported_and_stopped: - return Result(False, "reported and stopped: " + _report_headline(report)) - return Result(False, "no port written, and no report explaining why") - return Result(True, "port written") - - -def _report_headline(report: Path) -> str: - """The report's first non-empty, non-heading line, for the run log.""" - for line in report.read_text(encoding="utf-8", errors="replace").splitlines(): - line = line.strip().lstrip("#").strip() - if line and not line.startswith(("**Date", "**Skill", "---")): - return line[:160] - return "(no summary line)" - - -# --------------------------------------------------------------------------- # -# 5. validated — records a verdict, does not gate -# --------------------------------------------------------------------------- # - -def validate(ctx: Ctx) -> Result: - script = ctx.cfg.harness_root / SKILL_REL / "validate.py" - rc, out = run(ctx, "5-validate", [sys.executable, str(script), ".", "--json"]) - ctx.validate_ok = rc == 0 - if rc == 127: - ctx.validate_ok = None - return Result(False, "validate.py could not run") - try: - findings = json.loads(out) - ctx.log_path("5-validate.json").write_text(json.dumps(findings, indent=2), - encoding="utf-8") - except json.JSONDecodeError: - pass - return Result(True, "pass" if ctx.validate_ok else "fail (not gating)") - - -# --------------------------------------------------------------------------- # -# 6. built — build, then both probes from SKILL.md Step 4 -# --------------------------------------------------------------------------- # - -def build(ctx: Ctx) -> Result: - rc, out = run(ctx, "6-build", ["ventis", "build", "-c", "config/global_controller.yaml"]) - if rc != 0: - return Result(False, f"ventis build exited {rc}") - - # `ventis build` prints "Build complete." and exits 0 for a project whose - # container dies on startup, so the build result is not evidence on its own. - for entry in _agent_entries(ctx.root): - if entry.get("type") == "workflow": - continue - name = entry.get("name", "") - image = f"ventis-{name.lower()}" - - rc, out = run(ctx, f"6-probe1-{name}", - ["docker", "run", "--rm", image, "python", "-c", - "import local_controller"]) - if rc != 0: - # The gRPC/protobuf stack is unpinned and resolved on the host; this - # failure belongs to Ventis, not to the port. - ctx.core_issue.append({"kind": "runtime_import", "agent": name, - "detail": out.strip()[-500:]}) - return Result(False, f"{image}: import local_controller failed") - - entrypoint = Path(entry.get("entrypoint", "")).name - probe = ( - "import importlib.util, sys\n" - f"spec = importlib.util.spec_from_file_location('m', '{entrypoint}')\n" - "m = importlib.util.module_from_spec(spec); sys.modules['m'] = m\n" - f"spec.loader.exec_module(m); m.{name}(); print('ok')" - ) - rc, out = run(ctx, f"6-probe2-{name}", - ["docker", "run", "--rm", image, "python", "-c", probe]) - if rc != 0: - # _load_agent swallows every exception, so without this probe the - # symptom would only appear as "No agent loaded" at stage 8. - ctx.skill_issue.append({"kind": "agent_unloadable", "agent": name, - "detail": out.strip()[-500:]}) - return Result(False, f"{image}: agent would not load") - - return Result(True, "built, both probes pass") - - -# --------------------------------------------------------------------------- # -# 7. deployed -# --------------------------------------------------------------------------- # - -def deploy(ctx: Ctx) -> Result: - """`ventis deploy` blocks in a health-monitoring loop, so it runs detached - and the harness waits on the workflow's port instead.""" - logfile = ctx.log_path("7-deploy.log").open("w", encoding="utf-8") - proc = subprocess.Popen( - ["ventis", "deploy", "-c", "config/global_controller.yaml"], - cwd=ctx.root, stdout=logfile, stderr=subprocess.STDOUT, text=True, - start_new_session=True, env=subprocess_env(), - ) - ctx._procs.append(proc) - with _LIVE_LOCK: - _LIVE.add(proc) - - port_no = _api_port(ctx.root) - deadline = time.time() + ctx.cfg.stage_timeout - while time.time() < deadline: - if proc.poll() is not None: - return Result(False, f"ventis deploy exited early ({proc.returncode})") - try: - urllib.request.urlopen(f"http://localhost:{port_no}/", timeout=2) - return Result(True, f"workflow answering on :{port_no}") - except urllib.error.HTTPError: - return Result(True, f"workflow answering on :{port_no}") # 404 is an answer - except Exception: - time.sleep(2) - return Result(False, f"workflow never answered on :{port_no}") - - -# --------------------------------------------------------------------------- # -# 8. served -# --------------------------------------------------------------------------- # - -def serve(ctx: Ctx, query: str = "animals") -> Result: - port_no = _api_port(ctx.root) - body = json.dumps({"query": query}).encode() - req = urllib.request.Request( - f"http://localhost:{port_no}/main", data=body, - headers={"Content-Type": "application/json"}, method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=60) as resp: - accepted = json.loads(resp.read()) - except Exception as e: - return Result(False, f"POST /main failed: {type(e).__name__}: {e}") - - request_id = accepted.get("request_id") - if not request_id: - return Result(False, f"POST /main returned no request_id: {accepted}") - - deadline = time.time() + ctx.cfg.stage_timeout - last = {} - while time.time() < deadline: - try: - with urllib.request.urlopen( - f"http://localhost:{port_no}/status/{request_id}", timeout=30 - ) as resp: - last = json.loads(resp.read()) - except Exception as e: - last = {"status": "poll_failed", "detail": str(e)} - ctx.log_path("8-serve.json").write_text(json.dumps(last, indent=2), encoding="utf-8") - status = last.get("status") - if status == "done": - # Ventis served the request. Whether the *project* then did anything - # useful is a separate question, and conflating the two would let a - # hundred-repo pass rate mean much less than it appears to: a port - # can be perfect while the source fails on a query that means - # nothing to it, or on a host it was never given. - inner = last.get("result") - if isinstance(inner, dict): - app = inner.get("status") or inner.get("error") or inner.get("error_message") - if inner.get("status") in ("failed", "error") or inner.get("error_message"): - ctx.app_error = str( - inner.get("error_message") or inner.get("error") or app - )[:300] - return Result(True, f"served; the project itself errored: " - f"{ctx.app_error[:80]}") - return Result(True, "served") - if status == "error": - detail = str(last.get("error", "")) - # A bare env var name is what a repo raises when a backing service it - # needs was never configured. That is a fact about the repo, not a - # defect in the port, so it must not be scored as one. - missing = re.fullmatch(r"'([A-Z][A-Z0-9_]{3,})'", detail.strip()) - if missing: - ctx.missing_credential = missing.group(1) - return Result(False, f"needs {missing.group(1)}, which was never configured") - ctx.skill_issue.append({"kind": "request_error", "detail": last}) - return Result(False, f"request errored: {str(last)[:300]}") - time.sleep(3) - return Result(False, f"request never completed: {str(last)[:300]}") - - -# --------------------------------------------------------------------------- # -# teardown -# --------------------------------------------------------------------------- # - -def teardown(ctx: Ctx) -> None: - """Containers and the controller outlive the pipeline unless killed.""" - for proc in ctx._procs: - if proc.poll() is None: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGTERM) - proc.wait(timeout=30) - except Exception: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - except Exception: - pass - ctx._procs.clear() - if _config_path(ctx.root).is_file(): - run(ctx, "9-clean", ["ventis", "clean"], timeout=120) From 78645e101c0ec71b94d4460d9670f62b5499e3a7 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 19:24:18 -0700 Subject: [PATCH 33/43] testing-porting-to-ventis: ask what must be running, not what matches a list Running the skill found the hole in its own step 2. The backing-service check named prefixes -- ELASTICSEARCH_*, PINECONE_*, MONGODB_* -- and a list reads as a checklist: it passed a repo whose every node runs commands over SSH, because SSH_HOST was not on it. That repo had already been through the whole pipeline in an earlier run and served a response; the response was the NoneType error from int(os.getenv("SSH_PORT")), and the run proved nothing. The check is now the question the list was standing in for -- read the env vars the source reads, and for each ask what would have to be running for this to work -- with the grep to find them and the SSH case as the worked example. Also states that a repo whose work is reaching such a service stays out of scope even when a port of it builds and serves, since that is exactly the case that looked like a pass. --- .../skills/testing-porting-to-ventis/SKILL.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.claude/skills/testing-porting-to-ventis/SKILL.md b/.claude/skills/testing-porting-to-ventis/SKILL.md index 9a06f8e..b688d88 100644 --- a/.claude/skills/testing-porting-to-ventis/SKILL.md +++ b/.claude/skills/testing-porting-to-ventis/SKILL.md @@ -75,7 +75,7 @@ agent's full budget to rediscover it. | --- | --- | | Is there a module the adapter can import from the project root? | No root-level `.py` **and** no `pyproject.toml`/`setup.py`/`setup.cfg`. Without packaging metadata nothing is importable at `/app`. This is M24, and it rejects most tutorial repos. | | Which provider will it actually call? | It needs one whose key you do not have. | -| Does it need a backing service? | It reads `ELASTICSEARCH_*`, `PINECONE_*`, `MONGODB_*`, `QDRANT_*`, `WEAVIATE_*`, `SUPABASE_*`, `TAVILY_*`, `DATABASE_URL`… Ventis provides Redis and nothing else. | +| Does it need something to reach? | It reads the address or credentials of a service you are not standing up. Ventis provides Redis; everything else is on you. | | Is there Python at all? | Notebooks only, or no LLM call anywhere. | | Is it small to medium? | Hundreds of modules, or a framework rather than a project. | @@ -93,6 +93,24 @@ grep -rnoE '"(openai|anthropic|google_genai|bedrock|cohere|mistralai)[:/][^"]+"' A repo needing a provider you cannot serve is `blocked`, not `failed`. Record which provider and stop — that count is the argument for obtaining the key. +**Ask the backing-service question as a principle, not as a list.** Enumerating +prefixes reads as a checklist and lets everything unlisted through: a list +naming `ELASTICSEARCH_*` and `PINECONE_*` passed a repo whose first node calls +`int(os.getenv("SSH_PORT"))` against a remote host that does not exist. Read the +env vars the source actually reads, and for each one ask **what would have to be +running for this to work**: + +```bash +grep -rhoE "getenv\(\s*[\"'][A-Z_]{3,}|environ\[[\"'][A-Z_]{3,}" \ + | grep -oE "[A-Z_]{3,}" | sort -u +``` + +An LLM key you hold is fine. A host to SSH into, a vector store, a database, a +search API, an object store — anything the source must connect to and you are +not providing — is out of scope. A repo whose work is *reaching* such a service +stays out of scope even when a port of it builds and serves: the request returns +the source's own failure, and the run proves nothing about the skill. + ### Step 3 — the credential goes beside the source, never inside it Write `.env` at the project root with the real keys, and let the port's @@ -189,4 +207,5 @@ whole exercise exists to produce. Leave both empty when the run had no findings. | Running one probe instead of two | Probe 1's failure is a Ventis bug; probe 2's is the port's; neither covers the other | | Scoring report-and-stop as `failed` | Counts the skill working as the skill failing, and buries the Ventis gap that caused it | | Sending `{"query": "animals"}` to everything | `served` stops meaning anything | +| Screening backing services against a list of prefixes | Whatever is not on the list gets in — ask what must be running, not what matches | | Editing the skill mid-corpus | The pass rate loses its denominator | From 579eae0ef1b3dc297de08068de78a6f5e4a15da0 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 21:35:51 -0700 Subject: [PATCH 34/43] testing-porting-to-ventis: tear down, and check the ports before deploying The skill had no cleanup step at all, and the omission cost a run. ventis deploy holds a container per replica, one for the workflow and a Redis it starts itself; none of it stops when a request finishes. A previous repo's containers were still holding :8080 when the next deploy ran, and it failed with 'Failed to launch ventis-local-workflow-0' and nothing more, because _runtime.py drops docker's stderr. Diagnosing that needed docker inspect. So: a preflight in step 7 that costs a second, a step 9 that runs on the failure paths too, and a line in the mistakes table saying what a leak actually breaks -- not the repo that leaked, the one after it. Both blocks were run as written. xargs -r rather than a command substitution: the substitution form calls docker rm with no arguments when nothing is left, and the 2>/dev/null needed to hide that would hide a real failure too. --- .../skills/testing-porting-to-ventis/SKILL.md | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.claude/skills/testing-porting-to-ventis/SKILL.md b/.claude/skills/testing-porting-to-ventis/SKILL.md index b688d88..99d87bd 100644 --- a/.claude/skills/testing-porting-to-ventis/SKILL.md +++ b/.claude/skills/testing-porting-to-ventis/SKILL.md @@ -62,6 +62,7 @@ git rev-parse HEAD:ventis # ventis_sha | 6 | `built` | `ventis build -c config/global_controller.yaml`, then both probes | | 7 | `deployed` | `ventis deploy -c config/global_controller.yaml`, backgrounded | | 8 | `served` | `POST /main`, then poll `GET /status/` | +| 9 | — | tear down; not a stage, but the run is not over without it | `farthest_step` is the furthest stage reached. Step 5 is the exception: it does **not** gate what follows. @@ -146,6 +147,19 @@ Probe 1 catches the protobuf/gRPC wall — a `core_issue`, since the fix belongs `generate_docker`. Probe 2 catches everything `_load_agent` swallows, which otherwise surfaces only as `"No agent loaded"` at step 8. +### Step 7 — check the ports are free before deploying + +```bash +docker ps -a --format '{{.Names}}' | grep -i '^ventis-' || echo "clean" +lsof -nP -iTCP:8080 -sTCP:LISTEN +``` + +Both must come back empty. A container the previous repo left behind still holds +`:8080`, and `ventis deploy` fails on it with `Failed to launch +ventis-local-workflow-0` and nothing else — it drops docker's stderr, so the +message names the symptom and not the cause. Checking first costs a second; +diagnosing it afterwards costs `docker inspect` and a confused half hour. + ### Step 8 — served means the port answered `POST /main {"query": ...}` and poll `/status/`. **Send a query the @@ -158,7 +172,35 @@ did what the skill promises — carry a request to the source and return the source's own result — but say so in `analysis`. A bare missing env var (`'ELASTICSEARCH_API_KEY'`) is `blocked`: nobody configured it. -## Deciding the status +### Step 9 — tear down, especially when the run failed + +`ventis deploy` blocks and holds a fleet: one container per replica, one for the +workflow, and a Redis it started itself. None of it stops when the request +finishes, and a failed launch leaves a container behind in `Created` state that +the next run's own stale-container sweep does not clear. + +Run this at the end of every repo, on the failure paths too — a leak does not +break the repo that leaked, it breaks the next one: + +```bash +pkill -f "ventis deploy" +docker ps -aq --filter 'name=^ventis-local-' | xargs -r docker rm -f +docker ps -aq --filter 'name=^ventis-redis-' | xargs -r docker rm -f +ventis clean # stubs/, grpc_stubs/, docker_container/ +``` + +`xargs -r` rather than `docker rm -f $(...) 2>/dev/null`: the substitution form +runs `docker rm` with no arguments when nothing is left, which is an error, and +the `2>/dev/null` that hides it would hide a real removal failure just as well. + +Then confirm it worked, because believing it did is how the next repo fails: + +```bash +docker ps -a --format '{{.Names}}' | grep -i '^ventis-' && echo "STILL THERE" +``` + +Leave the clone and `artifacts/` in place. They are the row's evidence, and the +database only stores a path to them. | `status` | When | | --- | --- | @@ -209,3 +251,4 @@ whole exercise exists to produce. Leave both empty when the run had no findings. | Sending `{"query": "animals"}` to everything | `served` stops meaning anything | | Screening backing services against a list of prefixes | Whatever is not on the list gets in — ask what must be running, not what matches | | Editing the skill mid-corpus | The pass rate loses its denominator | +| Skipping teardown after a failed run | The next repo fails on a port this one still holds, and its error names the wrong thing | From 9b7bf52faf078a0bfbe129c0cc3abde961efa1ba Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 21:37:45 -0700 Subject: [PATCH 35/43] testing-porting-to-ventis: the port goes to a subagent, and its report is a claim The last run was a compromised measurement. I had read stub_generator.py before writing the port, so I stepped over the ImportError the build log walks you into; I knew probe 2 wanted --env-file and that -e . installs the project's own dependencies. None of that came from the skill. Every trap I already knew was a trap the skill got credit for warning about, and I graded a port whose every decision I had made. Step 4 now dispatches to a general-purpose subagent and says why never a fork: a fork inherits this conversation, which reproduces exactly the contamination the split exists to remove. The prompt is deliberately thin -- a trap you spare the porter is a trap the skill is credited for. The subagent still runs the skill's own validate and probe steps, because those are instructions in the artifact under test and stopping it would measure something else. But the observer re-runs them from scratch: there is no transcript to read, so what gets recorded is what this thread's own commands said, and a subagent claiming a green build where ours fails is a finding rather than a discrepancy to reconcile. Deploy and serve stay with the observer so step 9 has an owner. Recorded as a known gap: a subagent has no --max-budget-usd, which the Python harness had. --- .../skills/testing-porting-to-ventis/SKILL.md | 80 +++++++++++++++++-- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/.claude/skills/testing-porting-to-ventis/SKILL.md b/.claude/skills/testing-porting-to-ventis/SKILL.md index 99d87bd..d4a627b 100644 --- a/.claude/skills/testing-porting-to-ventis/SKILL.md +++ b/.claude/skills/testing-porting-to-ventis/SKILL.md @@ -30,6 +30,9 @@ what it affects. ## What you judge, and what you only record +You do not do the port. A subagent does (step 4), and you stay the observer — +otherwise the run is graded by the person who wrote it. + You judge two things: **whether the repo is in scope** (step 2) and **what the result means** (the write-up). Everything else is a command whose exit code and output you record verbatim. @@ -57,7 +60,7 @@ git rev-parse HEAD:ventis # ventis_sha | 1 | `fetched` | the clone above | | 2 | `screened` | your read of the repo — see below | | 3 | `wired` | write `.env` beside the source with the keys the repo needs | -| 4 | `ported` | **the `porting-to-ventis` skill**, on this repo | +| 4 | `ported` | a **`general-purpose` subagent** running the `porting-to-ventis` skill | | 5 | `validated` | `python .claude/skills/porting-to-ventis/validate.py .` | | 6 | `built` | `ventis build -c config/global_controller.yaml`, then both probes | | 7 | `deployed` | `ventis deploy -c config/global_controller.yaml`, backgrounded | @@ -119,13 +122,68 @@ Write `.env` at the project root with the real keys, and let the port's the build context: `ventis build` sweeps the project into every image, and `_sweep_project_files` skips dotfiles precisely so `.env` cannot ride along. -### Step 4 — run the skill +### Step 4 — dispatch the port to a subagent. Do not port it yourself. + +**Send it to a `general-purpose` subagent — never a `fork`.** A fork inherits +this conversation, which is the one thing that must not reach the porter: by the +time you are running repo twenty you know that the build log misnames the stub +class, that probe 2 needs `--env-file`, that `-e .` installs the project's own +dependencies. None of that came from the skill. A porter carrying it will get +past traps the skill never warned it about, and the corpus will report a skill +that is better than the one a new reader actually gets. + +Porting it yourself is the same mistake wearing a second hat: you would be +grading a port whose every decision you made, knowing what you meant rather than +what the skill said. + +Give it the repo and the skill, and nothing else: + +``` +Port the project at onto Ventis. + +Use the porting-to-ventis skill and follow it as written, including the steps +that tell you to validate and to probe the built images. + +Do not ask for confirmation; there is nobody to answer. Where the skill tells +you to report something rather than fix it, write PORT_REPORT.md in the project +root and stop -- that counts as following it. + +Report back: what you wrote, what you ran and what it said, anything the skill +left you guessing about, and anything you had to work out that the skill could +have told you. +``` -Use the `porting-to-ventis` skill on the clone. Follow it as written; it is the -artifact under test. When it tells you to report something rather than fix it, -write `PORT_REPORT.md` in the repo and stop — **that is the skill working, and -the run is `blocked`, not `failed`.** Those paths fire on things Ventis cannot -do, so the finding belongs in `core_issue`. +No hints, no warnings, no "watch out for". A trap you spare it is a trap the +skill gets credit for warning about. + +**Let it run the skill's own Step 3 and Step 4** — validate, build, probe. Those +are instructions in the artifact under test; a porter that skips them is not +following the skill, and stopping it would be measuring something else. + +**Then verify from scratch. Its report is a claim, not a result.** You have no +transcript of what it did, so what you record must come from what you can see +yourself: + +| Check | How | +| --- | --- | +| Did it edit the source? (M19) | `git status --porcelain` in `src/` — only new directories should appear | +| What did it actually write? | read the four files; the report is not evidence for them | +| Does validate pass? | run step 5 yourself; do not take the report's word | +| Does it build and load? | run step 6 yourself, both probes | + +A subagent reporting a green build where yours fails is a finding, not a +discrepancy to reconcile. Record what your own commands said. + +**Deploy and serve stay here.** A subagent holding a fleet of containers has no +clear owner for step 9, and the leak lands on the next repo. + +When the subagent reports and stops rather than porting — **that is the skill +working, and the run is `blocked`, not `failed`.** Those paths fire on things +Ventis cannot do, so the finding belongs in `core_issue`. + +Its answer to *"anything the skill left you guessing about"* is the most +valuable thing it returns: unlike a defect you find by tripping over it, that is +the skill's silence reported by the only reader who did not already know. ### Step 6 — build, then probe twice @@ -202,6 +260,11 @@ docker ps -a --format '{{.Names}}' | grep -i '^ventis-' && echo "STILL THERE" Leave the clone and `artifacts/` in place. They are the row's evidence, and the database only stores a path to them. +**Known gap:** a subagent has no spend cap. The Python harness this replaced +passed `--max-budget-usd` per repo and recorded exhaustion as its own outcome; +there is no equivalent here, so a repo that sends a porter in circles costs +whatever it costs. Watch the first runs of any new repo shape. + | `status` | When | | --- | --- | | `passed` | Step 8 returned the source's own result. | @@ -252,3 +315,6 @@ whole exercise exists to produce. Leave both empty when the run had no findings. | Screening backing services against a list of prefixes | Whatever is not on the list gets in — ask what must be running, not what matches | | Editing the skill mid-corpus | The pass rate loses its denominator | | Skipping teardown after a failed run | The next repo fails on a port this one still holds, and its error names the wrong thing | +| Dispatching the port to a `fork` | It inherits everything you have learned, and the skill gets credit for warning about traps it never mentions | +| Doing the port yourself | You grade a port whose decisions you made, knowing what you meant rather than what the skill said | +| Recording the subagent's claims | Its report is a claim. Run validate and build yourself and record what *your* commands said | From ced63fda08c5d54da938706d3a89330a3508f900 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 22:16:38 -0700 Subject: [PATCH 36/43] ventis build: take PR #51's sweep and stub placement, and let -e . carry its own metadata stub_generator.py was a hybrid -- PR #51's base with can-228's _sweep_project_files and _stub_destinations swapped in. It is now #51's file, verbatim, plus the editable install that #51 does not have. _sweep_py_files and _stub_destination are #51's, untouched, which means the flat stub copy that 01a70f2 added is gone again: #51 places a stub at one path, not two. That regression is #51's to decide on, and this branch no longer hides it. _install_step stays, because without it every src/ layout with a pyproject.toml -- the shape every langchain-ai template ships -- is unportable, and the corpus returns to empty. It no longer leans on the sweep to deliver pyproject.toml: _packaging_files copies the metadata and the README/LICENSE that metadata names, so the divergence from #51 is purely additive and confined to a feature #51 has no opinion about. Verified by rebuilding memory-agent: build exits 0, the context carries pyproject.toml, README.md and LICENSE, the install line keeps -e ., and both probes pass with the src/ package importable inside the image. test_stub_generator.py passes; the two test_cli failures predate this and come from PR #53's global os.path.isfile patch. --- ventis/stub_generator.py | 118 ++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 56 deletions(-) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 83e417b..472fe73 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -178,7 +178,7 @@ def __init__(self): ...stub methods... """ # class_name = agent_config["name"] + "Stub" - class_name = agent_config["name"] + class_name = agent_config["name"] functions = agent_config.get("functions", []) # __init__ method: simple pass, no gRPC setup needed. @@ -272,22 +272,11 @@ def _format_source(source): # Directories ventis build itself generates inside a project -- never swept. -_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs", "__pycache__"} +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} -# Written into the context by the generator itself; a project file of the same -# name at the root would land on top of it. -_GENERATED_ROOT_FILES = {"requirements.txt", "Dockerfile"} - -def _sweep_project_files(project_dir): - """Recursively collect (abs_src, rel_dst) for every project file, preserving its directory structure. - - Not only modules: the editable install below reads the project's packaging - metadata, and that metadata routinely points at a README or a license file, - so a sweep that took `.py` alone would leave nothing installable. Hidden - files are skipped -- `.env` holds credentials and has no business in an - image. - """ +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): dirs[:] = [ @@ -295,18 +284,12 @@ def _sweep_project_files(project_dir): for d in dirs if not d.startswith(".") and not (root == project_dir and d in _GENERATED_DIRS) - and d != "__pycache__" ] - at_root = os.path.abspath(root) == os.path.abspath(project_dir) for fname in files: - if fname.startswith("."): - continue - if at_root and fname in _GENERATED_ROOT_FILES: - continue abs_src = os.path.join(root, fname) - if os.path.islink(abs_src): - continue - swept.append((abs_src, os.path.relpath(abs_src, project_dir))) + if fname.endswith(".py") and not os.path.islink(abs_src): + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) return swept @@ -314,6 +297,30 @@ def _sweep_project_files(project_dir): _PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") +def _packaging_files(project_dir): + """The metadata `-e .` reads, plus the files that metadata points at. + + The sweep above takes `.py` and nothing else, so a project's pyproject.toml + would never reach the build context and `uv pip install -e .` would fail + with "does not appear to be a Python project". Packaging metadata also + routinely names a README or a license, and the install fails when that + target is missing, so root-level ones come along too. + + Metadata pointing somewhere the sweep does not reach -- a readme under + docs/ -- is not handled here, and surfaces as uv's own error at build time. + """ + if not project_dir or not os.path.isdir(project_dir): + return [] + picked = [] + for name in sorted(os.listdir(project_dir)): + upper = name.upper() + if name in _PACKAGING_FILES or upper.startswith(("README", "LICENSE", "LICENCE")): + path = os.path.join(project_dir, name) + if os.path.isfile(path): + picked.append((path, name)) + return picked + + def _install_step(project_dir): """The Dockerfile lines that install requirements, plus the project itself. @@ -343,32 +350,18 @@ def _install_step(project_dir): ) -def _stub_destinations(stub_file, stub_entrypoints): - """Every path a stub is copied to, flat name first. - - The flat copy is what callers import -- `from joke_agent import JokeAgent` - resolves against /app, which is sys.path[0]. The entrypoint copy overwrites - the real implementation the sweep placed there, so importing a peer by its - path gives the caller a stub rather than the peer's own code. An agent's own - entrypoint is copied after this and wins its flat name back. - """ +def _stub_destination(stub_file, stub_entrypoints): + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) - destinations = [basename] - entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") - if normalized.startswith("/") or ".." in normalized.split("/"): - print( - f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat only" - ) - elif normalized != basename: - destinations.append(normalized) + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") elif stub_entrypoints: - print( - f" Warning: no entrypoint mapping for stub {basename}, placing flat only" - ) - return destinations + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") + return basename def _copy_files(output_dir, files_to_copy): @@ -393,9 +386,9 @@ def generate_docker( output_dir=None, grpc_stubs_dir=None, stub_files=None, - requirements=None, project_dir=None, stub_entrypoints=None, + requirements=None, ): """ Generate a minimal Docker build context for an agent. @@ -409,9 +402,9 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. - requirements: Optional list of extra pip packages this agent needs. project_dir: Optional project root to sweep for extra .py helper files. stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. + requirements: Optional list of extra pip packages this agent needs. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -437,7 +430,8 @@ def generate_docker( # Sweep the project for extra .py helper files not on the explicit list below. files_to_copy = [] if project_dir: - files_to_copy += _sweep_project_files(project_dir) + files_to_copy += _sweep_py_files(project_dir) + files_to_copy += _packaging_files(project_dir) # Copy general agent files files_to_copy += [ @@ -464,8 +458,12 @@ def generate_docker( # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: - for dst in _stub_destinations(stub_file, stub_entrypoints or {}): - files_to_copy.append((os.path.abspath(stub_file), dst)) + files_to_copy.append( + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) + ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -513,9 +511,9 @@ def generate_workflow_docker( output_dir=None, grpc_stubs_dir=None, api_port=8080, - requirements=None, project_dir=None, stub_entrypoints=None, + requirements=None, ): """ Generate a Docker build context for a workflow. @@ -529,9 +527,9 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - requirements: Optional list of extra pip packages this workflow needs. project_dir: Optional project root to sweep for extra .py helper files. stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. + requirements: Optional list of extra pip packages this workflow needs. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -554,7 +552,11 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = _sweep_project_files(project_dir) if project_dir else [] + files_to_copy = ( + _sweep_py_files(project_dir) + _packaging_files(project_dir) + if project_dir + else [] + ) files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -576,11 +578,15 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - for dst in _stub_destinations(stub_file, stub_entrypoints or {}): - files_to_copy.append((os.path.abspath(stub_file), dst)) + files_to_copy.append( + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): From 91e23c470254587266dfb8ae62dc7ece4b495bcb Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 22:32:09 -0700 Subject: [PATCH 37/43] porting-to-ventis: name the two imports a workflow needs The skill said where the stub lands only obliquely and never gave the import line, so a porter reads the worked example instead -- and the example's `from joke_agent import JokeAgent` raises ModuleNotFoundError in the workflow image. The build copies the stub to exactly one path, and for the workflow that path is agents/.py. Both imports are now written down, with the two traps that sit on them: the flat form the examples use, and the class name the build log announces. The log says 'Generated stub class Stub' while the code writes -- the message is computed separately from the class -- and importing what the message names raises ImportError. Both cost this session real time. Verified in the image: the flat form raises ModuleNotFoundError, the agents. form imports, no __init__.py needed since agents/ is a namespace package. A memory-agent port rewritten to the documented form deployed and served. --- .claude/skills/porting-to-ventis/SKILL.md | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-ventis/SKILL.md index 4c887cb..d34c019 100644 --- a/.claude/skills/porting-to-ventis/SKILL.md +++ b/.claude/skills/porting-to-ventis/SKILL.md @@ -29,9 +29,12 @@ config/policy.yaml optional — only to restrict access NOT EDITED — copied whole into every image ``` -The two `agents/` files share one basename, as every example does — the stub the -build generates lands where it cannot collide with either. Pick a basename that -is not a module the adapter imports. +The two `agents/` files share one basename, as every example does. The build +generates a stub from the yaml and copies it to `agents/.py` in every +image; in the agent's own image the adapter is copied afterwards and wins the +flat name back, so the two never collide. Pick a basename that is not a module +the adapter imports — an adapter beside a source package called `memory_agent` +is named something else, or it shadows the package it exists to import. Everything the source already does — prompts, tools, schemas, parsing, retries, its LLM client — is reached with an `import`. **A port that contains a prompt @@ -165,6 +168,25 @@ What each method has to do is Step 1's table. **workflow** — a top-level function named `main`, taking a single `query: str`, plus `deploy(main, port=...)` at the end. +Its two imports are fixed, and neither is guessable: + +```python +from deploy import deploy # flat: deploy.py is copied to /app +from agents. import # the stub, under agents/ +``` + +**The stub is only at `agents/`, and its class carries the agent's own name.** +Two traps sit here, and the build walks you into both: + +- `ventis build` prints `Generated stub class 'Stub'`, but the class + it writes is ``. The message is computed separately from the code. + Importing what it names raises `ImportError`. +- The flat form `from import ` is what the examples in + this repository use, and in the workflow image it raises + `ModuleNotFoundError`: the stub is copied to one path, and for the workflow + that path is `agents/.py`. No `__init__.py` is needed — `agents/` + resolves as a namespace package. + Ventis itself is permissive here: it serves `POST /` and splats the request body in as kwargs, so any name and any arguments run. The deployment platform's test endpoint is not. It posts to a hardcoded `/main`, and its body From 9f4dfa0357bffc481edb724479670c709c39b16e Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 22:40:24 -0700 Subject: [PATCH 38/43] porting-to-ventis: check the stub import, and stop W006 lying about -e . Two rules the skill states and the validator did not check. V023 is new: the workflow must import a stub as `from agents. import `. It catches both ways of getting this wrong, because the project walks a reader into both -- the flat form, which is what examples/joke_writer uses and which raises ModuleNotFoundError in the workflow image, and `Stub`, which is the name ventis build prints while writing the class without the suffix. Run against joke_writer it flags the example, which is the point: the example is wrong. W006 now knows what the editable install resolves. It flagged langgraph and langchain_core on a project whose pyproject.toml requires both, while printing editable_install: yes in its own header three lines above. A false warning about a dependency is worse than none -- it teaches the reader to dismiss the check. Metadata it cannot read (setup.py, or no tomllib) still warns, but says so, because silence there would hide the real case. Also corrected a provenance line that has been false since the branch aligned to #51: stub_two_destinations is not on PR #51. #51 places a stub at one path; the fix that also places it flat is 01a70f2 on the skill branch and nobody has proposed merging it. --- .claude/skills/porting-to-ventis/SKILL.md | 1 + .claude/skills/porting-to-ventis/validate.py | 154 +++++++++++++++++-- 2 files changed, 146 insertions(+), 9 deletions(-) diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-ventis/SKILL.md index d34c019..920d676 100644 --- a/.claude/skills/porting-to-ventis/SKILL.md +++ b/.claude/skills/porting-to-ventis/SKILL.md @@ -245,6 +245,7 @@ not a ceiling. | M12 | The workflow MUST NEVER carry an `if __name__ == "__main__":` block | V017 | | M13 | A fan-out MUST dispatch every call before resolving any | V018 | | M14 | No project module MUST take the flat name of a runtime file or a stub | V019, V020 | +| M14b | The workflow MUST import a stub as `from agents. import ` | V023 | | M15 | `policy.yaml` MUST be absent, or MUST carry a non-empty `rules:` list | V021 | | M16 | An EC2 entry MUST declare `instance_type`, and `ec2:` MUST be complete | V022 | | M17 | NEVER copy a prompt, tool, or schema that exists in the source | W001 | diff --git a/.claude/skills/porting-to-ventis/validate.py b/.claude/skills/porting-to-ventis/validate.py index 86646d1..3647111 100755 --- a/.claude/skills/porting-to-ventis/validate.py +++ b/.claude/skills/porting-to-ventis/validate.py @@ -130,7 +130,9 @@ "env_file": "PR #53 (jiajunh/can-232-...), open against main", "editable_install": "no PR -- only on jiajunh/can-228-create-a-skill-...", "sweeps_all_files": "no PR -- only on jiajunh/can-228-create-a-skill-...", - "stub_two_destinations": "PR #51 (feature/all-the-files), open against main", + # Not on PR #51: its _stub_destination places a stub at one path. The fix + # that also puts it flat lives on the skill branch and has not been proposed. + "stub_two_destinations": "no PR -- 01a70f2 on jiajunh/can-228-porting-to-ventis-skill", } @@ -884,8 +886,77 @@ def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, met # ------------------------------------------------------------------ # -def check_workflow(report, workflow_path): - """V016 V017 V018.""" +def check_stub_imports(report, workflow_path, tree, stub_classes): + """V023 -- the workflow must import a stub as `from agents. import `. + + The build copies each stub to exactly one path, and for the workflow image + that path is agents/.py. Two ways of writing this line fail, and + the project walks you into both: the flat form is what examples/ uses, and + the class name is the one `ventis build` prints, which is not the one it + writes. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + base = alias.name.split(".")[0] + if base in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`import {alias.name}` -- the stub is at " + f"agents/{base}.py, not flat", + "The build copies a stub to one path, and for the " + "workflow that path is under agents/. This is a " + "ModuleNotFoundError the moment the workflow runs. " + f"Write `from agents.{base} import {stub_classes[base]}`.", + ) + continue + + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + + module = node.module + if module in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`from {module} import ...` -- the stub is at " + f"agents/{module}.py, not flat", + "The build copies a stub to one path, and for the workflow " + "that path is under agents/. The flat form is what this " + "repository's own examples use and it raises " + "ModuleNotFoundError in the workflow image. Write " + f"`from agents.{module} import {stub_classes[module]}`.", + ) + continue + + if not module.startswith("agents."): + continue + base = module.split(".", 1)[1] + expected = stub_classes.get(base) + if expected is None: + continue + for alias in node.names: + if alias.name == expected: + continue + if alias.name == f"{expected}Stub": + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is the name the build prints, not the " + f"class it writes", + "generate_agent_stub sets class_name = agent_config['name'] " + "and then recomputes it with a 'Stub' suffix for the log " + "line only. The message names a class that does not exist; " + f"the class is `{expected}`.", + ) + else: + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is not a class the stub for {base} defines", + f"The stub's class carries the agent's own name: `{expected}`.", + ) + + +def check_workflow(report, workflow_path, stub_classes=None): + """V016 V017 V018 V023.""" tree, error = parse_python(workflow_path) if tree is None: report.error("V016", workflow_path, 0, f"does not parse: {error}", "") @@ -919,6 +990,9 @@ def check_workflow(report, workflow_path): else: check_main_signature(report, workflow_path, main) + if stub_classes: + check_stub_imports(report, workflow_path, tree, stub_classes) + # V016 -- deploy() is what starts Flask. if not any( isinstance(node, ast.Call) @@ -1491,6 +1565,32 @@ def git(*args): ) +def _pyproject_dependencies(project_dir): + """What `-e .` installs alongside `requirements:`, or None if unreadable. + + None and the empty set mean different things here: empty means the project + declares no dependencies, None means we could not find out -- a setup.py, or + a tomllib this interpreter does not have. The caller must not treat the + second as the first, or it warns about imports the install would satisfy. + """ + path = os.path.join(project_dir, "pyproject.toml") + if not os.path.isfile(path): + return None + try: + import tomllib + except ImportError: # < 3.11 + return None + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except Exception: # noqa: BLE001 - malformed metadata is uv's error to give + return None + deps = (data.get("project") or {}).get("dependencies") + if not isinstance(deps, list): + return None + return {_normalize_distribution(d) for d in deps if isinstance(d, str)} + + def check_requirements_coverage( report, project_dir, entry, entrypoint_path, config_path ): @@ -1504,6 +1604,23 @@ def check_requirements_coverage( for item in (entry.get("requirements") or []) if isinstance(item, str) } + + # Where the editable install exists, `-e .` resolves the project's own + # [project.dependencies] in the same pass as `requirements:`. Warning about + # those is a false positive, and a false warning about a dependency is worse + # than none: it teaches the reader to dismiss this check. + editable = report.capabilities.get("editable_install") + metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + unreadable_metadata = False + if editable and metadata: + project_deps = _pyproject_dependencies(project_dir) + if project_deps is None: + unreadable_metadata = True + else: + declared |= project_deps base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} stdlib = getattr(sys, "stdlib_module_names", frozenset()) @@ -1519,17 +1636,30 @@ def check_requirements_coverage( distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) if distribution in base or distribution in declared: continue + if unreadable_metadata: + mechanism = ( + "The container installs the base list, `requirements:`, and -- " + "since this project declares packaging metadata -- whatever " + "`-e .` resolves from it. That metadata could not be read here, " + f"so if it already requires `{name}` this line is noise; " + "otherwise it is a ModuleNotFoundError inside _load_agent and " + "'No agent loaded' on the first request." + ) + else: + mechanism = ( + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside " + "_load_agent and 'No agent loaded' on the first request. If the " + f"distribution is named something other than `{name}`, declare " + f"that name in {report.rel(config_path)}." + ) report.warn( "W006", entrypoint_path, lineno, f"`import {name}` is in neither the runtime's base list nor this " "entry's `requirements:`", - "The container installs the base list plus `requirements:` and " - "nothing else, so this is a ModuleNotFoundError inside _load_agent " - "and 'No agent loaded' on the first request. If the distribution is " - f"named something other than `{name}`, declare that name in " - f"{report.rel(config_path)}.", + mechanism, ) @@ -1569,10 +1699,16 @@ def validate(project_dir, config_path, capabilities): } agents_by_name = {} + # basename of agents/.yaml -> the class its stub defines, which is + # the agent's own name. + stub_classes = {} for path in yaml_paths: data = check_agent_yaml_loads(report, path) if data is not None: agents_by_name[data["agent"]["name"]] = (path, data["agent"]) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = data["agent"][ + "name" + ] entries = config.get("agents") or [] check_config_entries(report, config, config_path, project_dir, set(agents_by_name)) @@ -1602,7 +1738,7 @@ def validate(project_dir, config_path, capabilities): continue workflow_path = os.path.join(project_dir, workflow_file) if os.path.isfile(workflow_path): - check_workflow(report, workflow_path) + check_workflow(report, workflow_path, stub_classes) check_flat_collisions(report, project_dir, yaml_paths, entrypoints) check_policy(report, config, config_path) From 59225245a0b654db0ea1190fa8b2110873682e15 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 10:33:36 -0700 Subject: [PATCH 39/43] iterate the skill --- .claude/skills/porting-to-ventis/SKILL.md | 25 +++++++++++++++++-- .../joke_writer/workflow/joke_workflow.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-ventis/SKILL.md index 920d676..f72e403 100644 --- a/.claude/skills/porting-to-ventis/SKILL.md +++ b/.claude/skills/porting-to-ventis/SKILL.md @@ -134,6 +134,20 @@ not. The nodes those edges connected are imported, unchanged. | `Crew(...)` / `GroupChat(...)` assembly | rewrite as Python control flow | | node functions, prompts, tools, schemas, parsers, clients | **import** | | the source's model provider and SDK | **keep** | +| a runtime object the nodes read services off | **construct one** — see below | + +**A framework runtime supplies two things, and only one of them is edges.** It +also injects services the nodes read at call time: LangGraph hands each node a +`Runtime` and the node reads `runtime.store`, `runtime.context`; other +frameworks pass a memory, a callback manager, a session. Ventis injects none of +it, so the adapter builds the object and passes it in — that is part of +re-expressing the runtime, not a liberty taken with the source. + +**Configure it from what the project already declares, never from taste.** A +LangGraph project states its store in `langgraph.json`; copy those values rather +than choosing your own, because an invented embedding model or dimension is a +silent change to what the project does. Where the project declares nothing, say +in the port report what you chose and why. ## Rule 2 — Split only to scale @@ -299,8 +313,9 @@ in this order. Neither covers the other.** # is ever reached, so probing the entrypoint alone will miss it. docker run --rm ventis- python -c "import local_controller" -# 2. The agent, loaded the way _load_agent loads it. -docker run --rm ventis- python -c " +# 2. The agent, loaded the way _load_agent loads it. --env-file because the +# constructor reads the environment, and the deployment gives it one. +docker run --rm --env-file ventis- python -c " import importlib.util, sys spec = importlib.util.spec_from_file_location('m', '.py') m = importlib.util.module_from_spec(spec); sys.modules['m'] = m @@ -321,6 +336,12 @@ Probe 2 exists because `_load_agent` catches every exception, logs it and return arguments, or a broken import inside the source tree are all invisible until the first request answers `"No agent loaded"`. +It takes `--env-file` because `__init__` reads the environment and a container +started by `ventis deploy` gets one. Without it a correct port fails its own +probe on a missing credential — an adapter that builds an embeddings client in +its constructor raises `OpenAIError: Missing credentials` and passes unchanged +the moment the file is passed. + Then `ventis deploy`, which needs Docker and an importable `grpc_stubs/` **on this host** (it aborts if they were cleaned after the build). It starts its own Redis container — do not run one. diff --git a/examples/joke_writer/workflow/joke_workflow.py b/examples/joke_writer/workflow/joke_workflow.py index 4b60c61..9fdf3cc 100644 --- a/examples/joke_writer/workflow/joke_workflow.py +++ b/examples/joke_writer/workflow/joke_workflow.py @@ -19,7 +19,7 @@ import json from deploy import deploy -from joke_agent import JokeAgent +from agents.joke_agent import JokeAgent def main(query): From 563997722b40853aadf120368c2eea045d18ce33 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 11:47:49 -0700 Subject: [PATCH 40/43] Rename porting skill to CanyonOS Core --- .../SKILL.md | 241 ++++-- .../canyonos-core-contract.md} | 125 ++- .../traps.md | 46 +- .../validate.py | 788 ++---------------- examples/joke_writer/README.md | 4 +- 5 files changed, 399 insertions(+), 805 deletions(-) rename .claude/skills/{porting-to-ventis => porting-to-canyonos-core}/SKILL.md (65%) rename .claude/skills/{porting-to-ventis/ventis-contract.md => porting-to-canyonos-core/canyonos-core-contract.md} (72%) rename .claude/skills/{porting-to-ventis => porting-to-canyonos-core}/traps.md (60%) rename .claude/skills/{porting-to-ventis => porting-to-canyonos-core}/validate.py (61%) diff --git a/.claude/skills/porting-to-ventis/SKILL.md b/.claude/skills/porting-to-canyonos-core/SKILL.md similarity index 65% rename from .claude/skills/porting-to-ventis/SKILL.md rename to .claude/skills/porting-to-canyonos-core/SKILL.md index f72e403..3102c5d 100644 --- a/.claude/skills/porting-to-ventis/SKILL.md +++ b/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -1,34 +1,54 @@ --- -name: porting-to-ventis -description: Use when porting an existing agent project (LangChain, LangGraph, CrewAI, AutoGen, or a hand-rolled pipeline) onto Ventis +name: porting-to-canyonos-core +description: Use when porting an existing agent project (LangChain, LangGraph, CrewAI, AutoGen, or a hand-rolled pipeline) onto CanyonOS Core --- -# Porting an agent project to Ventis +# Porting an agent project to CanyonOS Core + +CanyonOS Core is the product name. Its current compatibility interface remains +unchanged: the executable is `ventis`, the Python package is `ventis`, runtime +environment variables use `VENTIS_*`, and Docker resources use `ventis-*`. +Treat those as protocol identifiers, not branding strings; do not rename them +while porting. ## How to read the rules in this file Set in capitals, **MUST** and **NEVER** mark a rule whose violation breaks the port: the build skips an image, `ventis deploy` dies, or the first request -fails. Every one is indexed in [The MUST list](#the-must-list), and every one a -machine can decide is checked by `validate.py`. Nothing else in this file is -written in capitals, so `grep -nE '\b(MUST|NEVER)\b' SKILL.md` returns the -rules and only the rules. +fails. Every one is indexed in [The MUST list](#the-must-list), whose last +column says whether `ventis build`, deploy preflight, or `validate.py` decides +it. `validate.py` intentionally covers only failures a green image build hides. +Nothing else in this file is written in capitals, so +`grep -nE '\b(MUST|NEVER)\b' SKILL.md` returns the rules and only the rules. -Everything else is stated as fact, in the indicative — how Ventis behaves, and +Everything else is stated as fact, in the indicative — how CanyonOS Core behaves, and what follows from it. There is no "should", and nothing is left to taste that does not have to be. -## A port is four files beside an untouched source tree +## A port is thin scaffolding beside an untouched source tree ``` -agents/.yaml declares the callable surface -agents/.py the thinnest class that satisfies Ventis +agents/.yaml one callable surface per CanyonOS Core service +agents/.py one thin adapter per service, when needed workflow/_workflow.py entry point; calls deploy() config/global_controller.yaml deployment manifest config/policy.yaml optional — only to restrict access +pyproject.toml conditional — only to expose a nested import root NOT EDITED — copied whole into every image ``` +The file count follows the deployment design. A single adapted service normally +adds one yaml/adapter pair, one workflow, and one config. A multi-agent port adds +one yaml/adapter pair for each service that Rule 2 justifies splitting out. If a +source class already satisfies the CanyonOS Core contract, its config can point to that +source file directly and no adapter copy is needed. + +The project root is the directory from which `ventis build` runs. The source +remains untouched below it. A root `pyproject.toml` is additional conditional +scaffolding when the source is nested, its original imports do not resolve from +`/app`, and the target CanyonOS Core supports an editable install. Metadata inside the +nested source tree does not trigger that install. + The two `agents/` files share one basename, as every example does. The build generates a stub from the yaml and copies it to `agents/.py` in every image; in the agent's own image the adapter is copied afterwards and wins the @@ -41,12 +61,12 @@ its LLM client — is reached with an `import`. **A port that contains a prompt string, a tool body, or a model call that already exists in the source is a rewrite of the project, not a port of it.** -Mechanism and evidence for every claim here: `ventis-contract.md`. +Mechanism and evidence for every claim here: `canyonos-core-contract.md`. Symptom-to-cause lookup once something breaks: `traps.md`. ## Step 1 — Survey the source before writing anything -Ventis loads an agent by doing exactly this: +CanyonOS Core loads an agent by doing exactly this: ```python module = @@ -84,16 +104,38 @@ container starts at `/app`, so only what landed flat imports on its own. whole list, not the one bad item: `_normalize_requirements` logs one warning and returns `[]`, and the build still succeeds with none of them installed. -- **The import root** — *needs a Ventis change that has no PR.* An editable - install (`-e .`) driven by a `pyproject.toml`, `setup.py` or `setup.cfg` at the - root is what makes a `src/` layout importable, and the project's own packaging - metadata is what decides the root. `_install_step` exists only on - `jiajunh/can-228-create-a-skill-to-convert-a-langchain-project-to-ventis`, which - nobody has proposed merging. Until it lands, **only modules that land flat at - `/app` import at all** — an adapter reaching into `src/pkg/` raises - `ModuleNotFoundError` inside `_load_agent`, and the first request answers - `"No agent loaded"`. `validate.py` probes for the feature and reports which - rule is in force. +- **The import root** — run `validate.py` first and read its + `editable_install` capability. When available, a `pyproject.toml`, `setup.py` + or `setup.cfg` at the **port root** adds `-e .`; metadata inside the nested + source does not. Add a minimal root `pyproject.toml` only when an original + import cannot resolve from `/app`. It names the existing source directory and + package, declares no dependencies, and does not reference a README or license: + + ```toml + [build-system] + requires = ["setuptools>=64"] + build-backend = "setuptools.build_meta" + + [project] + name = "ventis-port" + version = "0.0.0" + dependencies = [] + + [tool.setuptools.packages.find] + where = [""] + include = ["*"] + namespaces = true + ``` + + Set `where` from the actual tree; for a wrapped project with + `source/pyproject.toml` and `source/src/pkg/`, it is `source/src`, not + `source`. Keep dependencies where the source declared them. Because nested + metadata is not installed, repeat its runtime distributions under each + config entry's `requirements:` without editing or deleting the source list. + Without editable-install support, report an import that cannot resolve from + `/app` and stop. A directory rooted directly at `/app` can already resolve as + a Python namespace package even without `__init__.py`; do not add packaging + metadata merely because that file is absent. - **`env_file:`** — *needs PR #53, open against main.* A path relative to the project root pointing at a local `.env`, handed to every container as @@ -103,6 +145,56 @@ container starts at `/app`, so only what landed flat imports on its own. sets `env_file:` is setting a key nothing reads — the credential is silently dropped and the failure surfaces as a provider error on the first request. +### When the target includes `llm_proxy` + +The proxy is an endpoint redirect, not a provider conversion. Keep the source's +OpenAI, Anthropic, or boto3 client and its request format; put the corresponding +SDK variable in the runtime env file: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Use only the lines for each provider the source actually uses. Its caller-side +credentials are placeholders, for example: + +```dotenv +OPENAI_API_KEY=proxy-placeholder +ANTHROPIC_API_KEY=proxy-placeholder +AWS_ACCESS_KEY_ID=proxy-placeholder +AWS_SECRET_ACCESS_KEY=proxy-placeholder +AWS_REGION=us-east-1 +``` + +OpenAI and Anthropic SDKs still require an API-key variable and boto3 still +requires credentials with which to sign the request, even though the proxy +replaces or reissues those credentials upstream. Launch the proxy in a separate +environment holding the real credentials. Do not put real proxy credentials in +the port's `env_file`, which is given to every agent and workflow container. + +The current proxy is local and non-streaming. Start it on the Docker host with a +non-loopback bind and a port different from the workflow API's usual 8080: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers can resolve `host.docker.internal` because their +`docker run` includes `--add-host=host.docker.internal:host-gateway`. An EC2 +container resolves that name to its own EC2 Docker host, not the machine running +`ventis deploy`; a local-only proxy therefore does not support a distributed +port. A reachable proxy address or one proxy per host is deployment work to +report, not an adapter rewrite. + +Survey the source for streaming before choosing this route: OpenAI +`stream=True`, Anthropic stream APIs, Bedrock `invoke_model_with_response_stream`, +Converse, and Converse Stream are outside this implementation. Do not silently +turn streaming off. Report the unsupported call and stop. Exact mechanics and +failure signatures are in `canyonos-core-contract.md` and `traps.md`. + **And one thing to report rather than fix.** Where the editable install exists, `-e .` installs `[project.dependencies]` in the same resolve as `requirements:`, and workshop projects routinely put their whole toolchain there so that one @@ -123,7 +215,7 @@ all. Then let the owner decide, including deciding not to. ## Rule 1 — Rewrite orchestration, import everything else One kind of source code genuinely cannot be reused: **control flow owned by a -framework runtime.** Ventis has no runtime to execute a LangGraph `StateGraph`, a +framework runtime.** CanyonOS Core has no runtime to execute a LangGraph `StateGraph`, a CrewAI `Crew` or an AutoGen `GroupChat`, so their wiring is re-expressed as ordinary Python — in the workflow when it fans out, in the adapter when it does not. The nodes those edges connected are imported, unchanged. @@ -139,7 +231,7 @@ not. The nodes those edges connected are imported, unchanged. **A framework runtime supplies two things, and only one of them is edges.** It also injects services the nodes read at call time: LangGraph hands each node a `Runtime` and the node reads `runtime.store`, `runtime.context`; other -frameworks pass a memory, a callback manager, a session. Ventis injects none of +frameworks pass a memory, a callback manager, a session. CanyonOS Core injects none of it, so the adapter builds the object and passes it in — that is part of re-expressing the runtime, not a liberty taken with the source. @@ -152,7 +244,7 @@ in the port report what you chose and why. ## Rule 2 — Split only to scale **Splitting into multiple agents is a scaling decision, not a format -requirement.** A single agent holding the whole pipeline is a valid Ventis +requirement.** A single agent holding the whole pipeline is a valid CanyonOS Core project. Start there, and hoist a loop into the workflow only when each iteration fans out to more than one node: @@ -162,7 +254,7 @@ fans out to more than one node: - a supervisor handing out N tasks, or a `Send` fan-out, is **hoisted** — N independent runs per request with no shared state is what replicas pay for. -An agent with `replicas: 1` and no distinct resource profile is a node Ventis +An agent with `replicas: 1` and no distinct resource profile is a node CanyonOS Core does nothing for. When you do split, say plainly what it buys. ## Step 2 — Write the files @@ -201,7 +293,7 @@ Two traps sit here, and the build walks you into both: that path is `agents/.py`. No `__init__.py` is needed — `agents/` resolves as a namespace package. -Ventis itself is permissive here: it serves `POST /` and splats the +CanyonOS Core itself is permissive here: it serves `POST /` and splats the request body in as kwargs, so any name and any arguments run. The deployment platform's test endpoint is not. It posts to a hardcoded `/main`, and its body schema is `{query: string}` under a strict validator, so a differently named @@ -250,53 +342,57 @@ not a ceiling. | M3 | A yaml `arguments[].name` MUST equal the Python parameter name exactly | V008 | | M4 | A yaml `type` MUST be a bare builtin | V010 | | M5 | A method backing a yaml function MUST be synchronous | V009 | -| M6 | Every config entry `name` MUST match some yaml `agent.name` | V003, V005 | -| M7 | Two config entry `name`s MUST differ by more than case | V004 | -| M8 | `provider` MUST be lowercase `local` (EC2 takes any casing) | V012 | -| M9 | `replicas` MUST be an integer | V013 | -| M10 | `requirements:` MUST be a list of strings | V014 | -| M11 | The workflow MUST expose `main(query)`; other parameters MUST have defaults | V015, V016 | +| M6 | Every config entry `name` MUST match some yaml `agent.name` | build | +| M7 | Two config entry `name`s MUST differ by more than case | build output | +| M8 | `provider` MUST be lowercase `local` (EC2 takes any casing) | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements:` MUST be a list of strings | build | +| M11 | The workflow MUST expose `main(query)`; other parameters MUST have defaults | build + V016 | | M12 | The workflow MUST NEVER carry an `if __name__ == "__main__":` block | V017 | | M13 | A fan-out MUST dispatch every call before resolving any | V018 | | M14 | No project module MUST take the flat name of a runtime file or a stub | V019, V020 | | M14b | The workflow MUST import a stub as `from agents. import ` | V023 | -| M15 | `policy.yaml` MUST be absent, or MUST carry a non-empty `rules:` list | V021 | -| M16 | An EC2 entry MUST declare `instance_type`, and `ec2:` MUST be complete | V022 | -| M17 | NEVER copy a prompt, tool, or schema that exists in the source | W001 | +| M15 | `policy.yaml` MUST be absent, or MUST carry a non-empty `rules:` list | deploy preflight | +| M16 | An EC2 entry MUST declare `instance_type`, and `ec2:` MUST be complete | deploy preflight | +| M17 | NEVER copy a prompt, tool, or schema that exists in the source | review | | M18 | NEVER hardcode a credential, or ship one in the build context | W003 | -| M19 | NEVER edit the source tree, and NEVER vendor it into `agents/` | W002 | -| M20 | NEVER swap the LLM provider the source uses | -- | +| M19 | NEVER edit the source tree, and NEVER vendor it into `agents/` | `git status` | +| M20 | NEVER swap the LLM provider the source uses; an LLM proxy only redirects its endpoint | -- | | M21 | NEVER move or drop a declared dependency — report it and stop | -- | | M22 | Framework control flow MUST be rewritten; everything else MUST be imported | -- | -`validate.py` reports more than this list — V001, V002 and W005, W006 catch -files that do not parse and imports the container cannot satisfy — but every row -here has a check behind it. +Build owns YAML parsing, required paths, stub generation, and Dockerfile/package +installation errors. Deploy preflight owns provider, replica, policy, and EC2 +shape. `validate.py` does not repeat those checks; it focuses on adapter loading, +stub imports, workflow execution, copy collisions, credentials, import roots, +and dependencies that fail only inside a built container. -Two more rules apply only where the Ventis you are targeting supports them, +Two more rules apply only where the CanyonOS Core you are targeting supports them, which `validate.py` probes for rather than assumes: | # | The rule | Needs | Check | | --- | ----------------------------------------------------------------- | ----------------------- | ----- | -| M23 | `env_file:` MUST resolve to a readable file, and MUST be the only way a credential enters | PR #53 | V030 | -| M24 | An adapter import from outside the project root MUST have packaging metadata behind it | no PR yet | V031 | +| M23 | `env_file:` MUST resolve to a readable file, and MUST be the only way a credential enters | PR #53 | deploy preflight; support V030 | +| M24 | A source import that does not resolve from `/app` MUST have usable packaging metadata at the port root | editable install | V031 | -## Step 3 — Validate +## Step 3 — Preflight hidden runtime failures ```bash python /validate.py . ``` -Run it before building. `ventis build` never imports your agent, and the -controller writes `healthy` to Redis *before* `_load_agent` runs — so a green -build and a healthy replica are both compatible with a container that can serve -nothing. This script is the only stage that reads what you actually wrote. +Run it before building. It does not duplicate errors `ventis build` or deploy +preflight already reports. Instead it catches what those stages do not execute: +the adapter class contract, generated-stub import path, workflow behavior, flat +copy collisions, container credentials, package import roots, and undeclared +runtime imports. -It parses; it never imports the port, so it is safe to run on a tree whose -dependencies are not installed. Errors are provable contract violations and exit -1. Warnings are the rewrite smells — M16, M17, M18 — and exit 0 on their own, -because a heuristic cannot be allowed to block a correct port; `--strict` -promotes them for CI. `--json` emits the findings as data. +It parses Python but never imports the port, so it is safe on a tree whose +dependencies are not installed. Errors are deterministic runtime contract +violations and exit 1. Heuristic warnings exit 0 unless `--strict` is used. +`--json` emits the findings as data. A malformed config or agent yaml is reported +by `ventis build`; when it prevents runtime inspection, the validator emits only +a `BUILD` informational finding and stops. The header prints which capability-gated rules are in force. A rule whose feature is missing is reported `UNAVAILABLE`, never silently skipped. @@ -346,6 +442,37 @@ Then `ventis deploy`, which needs Docker and an importable `grpc_stubs/` **on this host** (it aborts if they were cleaned after the build). It starts its own Redis container — do not run one. +## Step 5 — Clean up every build and deployment product + +Do this after recording the probe and request results, including on failure +paths. First stop the foreground `ventis deploy` with Ctrl+C and wait for +`GlobalController cleanup` to remove its agent, workflow, and Redis containers. +If deploy crashed before its cleanup handler ran, remove the exact container +names created by this deployment; do not delete another project's containers. + +Then remove generated files and the exact images built from the config: + +```bash +ventis clean # removes stubs/, grpc_stubs/, and docker_container/ + +docker image rm \ + ventis- \ + ventis- +``` + +Repeat the image argument for every config entry. `ventis clean` does not remove +containers or images. Confirm that the project root no longer contains the three +generated directories and that no container from this deployment remains: + +```bash +test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container +docker ps -a --format '{{.Names}}' +``` + +Keep all agent declarations and adapters, the workflow, config, conditional +root `pyproject.toml`, untouched source tree, and any requested logs or port +report. Those are source and evidence, not build products. + ## Never do these Each turns a port into a rewrite. They are not judgment calls, and the middle @@ -354,7 +481,7 @@ column is the thought that gets you there. | Move | The rationalization | Why it is wrong | | ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------- | | Copy a prompt, tool, or schema into the adapter | "so the adapter stands alone" | It exists in the source. Import it — the whole tree is in the image, and a copy drifts. | -| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider. | -| Hardcode a key, or ship it in a file you add | "there is no other way in" | The build sweeps the project into every image. Where `env_file:` exists it is the way in; where it does not, say so and stop. | +| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider; `llm_proxy` redirects that SDK rather than converting its request. | +| Hardcode a key, or ship it in a file you add | "there is no other way in" | The build sweeps the project into every image. Where `env_file:` exists it is the way in; with `llm_proxy`, it contains routing plus dummy caller credentials while the proxy receives real credentials separately. | | Drop or move a dependency | "this one is obviously dev-only" | Obvious to you, not yours to decide. Declare it under `requirements:`; report the rest and let the owner classify. | | Edit files in the source tree, or vendor it into `agents/` | "just this one line" | The port leaves `git status` on the source clean, and vendoring is copying. | diff --git a/.claude/skills/porting-to-ventis/ventis-contract.md b/.claude/skills/porting-to-canyonos-core/canyonos-core-contract.md similarity index 72% rename from .claude/skills/porting-to-ventis/ventis-contract.md rename to .claude/skills/porting-to-canyonos-core/canyonos-core-contract.md index 3d83165..18b9d44 100644 --- a/.claude/skills/porting-to-ventis/ventis-contract.md +++ b/.claude/skills/porting-to-canyonos-core/canyonos-core-contract.md @@ -1,9 +1,13 @@ -# The Ventis contract +# The CanyonOS Core contract + +The product is CanyonOS Core; its compatibility CLI, Python package, environment +prefix, and Docker resource prefix remain `ventis`, `ventis`, `VENTIS_*`, and +`ventis-*` respectively. Mechanism behind every rule in `SKILL.md`. Validate against [CanyonCodeCoreAI/canyoncodecore](https://github.com/CanyonCodeCoreAI/canyoncodecore). -**Which Ventis this describes.** Two sections below hold for a branch rather than +**Which CanyonOS Core this describes.** Two sections below hold for a branch rather than for `main`, and each says so where it starts. `validate.py` probes the importable `ventis` package for them instead of assuming: @@ -25,7 +29,7 @@ Everything not marked holds on `main` today. | `config/policy.yaml` | `global_controller.py` `_load_policy_rules` — optional | | the workflow file | the `workflow_file` key on the `type: workflow` config entry | | the project root | `cli.py` passes `project_dir=os.getcwd()`; build and deploy run from it | -| `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .`. **No PR carries this** | +| root `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .`; nested metadata is ignored. **No PR carries this** | ## Agent yaml @@ -110,7 +114,7 @@ The workflow file is **not imported — it is `exec`'d**. - Module-level code runs **once** at container start; the workflow function runs **per request**, on a Flask worker thread. - The REST route is `fn.__name__` — rename the function and the endpoint renames - with it. There is no fixed `/main` **in Ventis**. + with it. There is no fixed `/main` **in CanyonOS Core**. - The request body is splatted in as kwargs after `_context` is popped off. Any shape of body works. @@ -164,22 +168,33 @@ own* stub by name — the entrypoint shadows it flat. It can still reach it at > **No PR carries this.** `_install_step` lives only on > `jiajunh/can-228-create-a-skill-...`. On `main`, and on both open PRs, the > agent Dockerfile is `COPY requirements.txt` → `uv pip install -r -> requirements.txt` → `COPY . .`, with no `-e .` and no packaging detection — -> so **only modules that land flat at `/app` import at all**, whatever metadata -> the project declares. `validate.py` V031 enforces whichever rule is in force. +> requirements.txt` → `COPY . .`, with no `-e .` and no packaging detection. +> Python then resolves only names rooted at `/app`: flat modules, regular +> packages, and namespace-package directories. A package below another source +> directory is not a top-level import merely because it was copied. +> `validate.py` V031 enforces whichever rule is in force. `_install_step` writes -`RUN uv pip install --system -r requirements.txt -e .` when the project root has -a `pyproject.toml`, `setup.py` or `setup.cfg`. That editable install is what -makes a `src/` layout importable, and the source's own packaging metadata is what -decides it — `[tool.setuptools.package-dir] "" = "src"` is a typical case. Ventis -never guesses a directory name. - -Without packaging metadata the install is skipped — silently, no warning. The -tree is still copied, but `sys.path[0]` is `/app`, so only modules that landed -flat resolve. `examples/helloworld`, `finance` and `text2sql` are all in this -state; they work because their entrypoints import nothing from the project tree, -only stubs, which land flat. +`RUN uv pip install --system -r requirements.txt -e .` when the **project root** +has a `pyproject.toml`, `setup.py` or `setup.cfg`. It does not search below that +root. This distinction is observable in the test harness: a repository kept +untouched under `source/` can have `source/pyproject.toml`, but `_install_step` +still skips it. + +When the nested source's original imports need another directory on `sys.path`, +the port supplies a minimal root `pyproject.toml` that points setuptools at the +existing package directory. It declares no dependencies and references no +README or license. That file is port scaffolding, not a source edit. For example, +if `source/src/pkg/` backs `import pkg`, package discovery uses +`where = ["source/src"]`, `include = ["pkg*"]`, and `namespaces = true` when the +package omits `__init__.py`. + +Without root packaging metadata the editable install is skipped silently. Some +nested-looking imports still work: `/app/src/agents/kyc_agent.py` is importable +as `src.agents.kyc_agent` through namespace packages even when neither directory +has `__init__.py`. It is not importable as `agents.kyc_agent`; that spelling +would require `/app/src` as an import root. Packaging is conditional on the +actual import spelling, not on whether `__init__.py` exists. **One resolve, not two.** Where `_install_step` exists, requirements and `-e .` go to a single `uv pip install` so the runtime's list and the source's own @@ -205,10 +220,14 @@ list of strings — a bare string, a mapping, or a list with a non-string in it each logs one warning and becomes `[]`, so a malformed entry costs the whole list rather than the one item. Nothing is deduplicated against the base either. -**The source's own `pyproject.toml` is installed in the same resolve**, so -`requirements:` covers only what the adapter imports and the source does not -declare. The whole dependency list comes along, dev extras included — a workshop -project's can carry jupyter, matplotlib and pandas into a 1GB agent image. +**Only a `pyproject.toml` at the port root is installed in the same resolve.** +If that is the source's own metadata, `requirements:` covers only imports it +does not declare, and its whole dependency list comes along. If the source is +nested and the root metadata is minimal port scaffolding, the nested dependency +list is not installed; repeat its runtime distributions under the relevant +config entries' `requirements:` while leaving the source declaration untouched. +This duplication is required by the current root-only packaging probe, not a +license to reclassify or drop dependencies. ### The gRPC stack is unpinned @@ -271,6 +290,68 @@ Consequences for a port: - `load_dotenv(".env")` in the source still does nothing: the file is not in the image and `load_dotenv` is silent about a missing one. +## The optional `llm_proxy` endpoint contract + +The `llm_proxy` implementation is a separate Flask process, not an agent and not +part of `ventis deploy`. It preserves each caller's SDK protocol under a provider +prefix and funnels all completed calls through `llm_proxy.core.proxy_request`: + +| Source client | Container variable | Proxy path | Upstream behavior | +| --- | --- | --- | --- | +| OpenAI SDK | `OPENAI_BASE_URL=http://host.docker.internal:/openai/v1` | `/openai/...` | HTTP request forwarded; caller authorization removed and proxy key inserted | +| Anthropic SDK | `ANTHROPIC_BASE_URL=http://host.docker.internal:/anthropic` | `/anthropic/...` | HTTP request forwarded; caller key removed and proxy key inserted | +| boto3 Bedrock Runtime | `AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:/bedrock` | `/bedrock/model//invoke` | body reissued through the proxy's boto3 client and its AWS identity | + +This is why using the proxy does not relax the no-provider-swap rule: model IDs, +request bodies, response bodies, and the source SDK remain provider-specific. +The only port artifact is endpoint configuration in the runtime environment. +OpenAI and Anthropic client constructors still validate that their normal key +variables exist. Botocore still signs the request it sends to its custom +endpoint. Dummy caller credentials satisfy those clients; the proxy process gets +real `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or AWS credentials from its own +process environment. Putting the real values in the project's `env_file` gives +them to every CanyonOS Core container and defeats the credential boundary. + +The proxy defaults to `127.0.0.1:8080`. Both defaults are wrong beside a local +CanyonOS Core deployment: a container cannot reach the host's loopback, and the +workflow normally publishes host port 8080. Bind the proxy to `0.0.0.0` on a +different port. Local and EC2 CanyonOS Core `docker run` commands add +`host.docker.internal:host-gateway`; that name means the Docker host of each +container. Consequently a proxy on the controller's machine serves local +containers, while remote EC2 containers require a reachable network address or +a proxy running on each EC2 host. + +`GET /healthz` proves only that Flask is listening and lists the registered +providers. It does not validate any upstream credential. OpenAI and Anthropic +upstream 4xx/5xx responses pass through with their status and body. Exceptions +in routing or provider code become a JSON `502` with `error: proxy_error`. +Bedrock `ClientError` responses are reconstructed as JSON with the upstream +status; they are not byte-for-byte passthrough. Although `Config` reads +`BEDROCK_UPSTREAM_HOST`, `BedrockProvider` does not pass it as `endpoint_url` +when constructing boto3, so that variable has no effect in this implementation. + +The implementation buffers the full request and response. It has no OpenAI or +Anthropic streaming path, and Bedrock accepts only the final `invoke` operation; +`invoke-with-response-stream`, `converse`, and `converse-stream` raise +`NotImplementedError` and surface as 502. A port cannot preserve a source that +uses those calls through this proxy today. + +## Cleanup boundaries + +`ventis deploy` registers `GlobalController.cleanup` for Ctrl+C, SIGTERM, and +normal process exit. That cleanup terminates the controller's recorded agent and +workflow instances and its Redis containers. A hard kill or an exception before +a runtime is recorded can leave Docker containers behind, so cleanup must also +be verified from Docker state. + +`ventis clean` is narrower: `cmd_clean` removes only the project-root `stubs/`, +`grpc_stubs/`, and `docker_container/` directories. It removes neither running +containers nor the `ventis-` images produced by the +build. Image removal therefore happens explicitly after containers stop. The +agent declarations and adapters, workflows, config, conditional root packaging +metadata, untouched source, and recorded evidence are not generated build +products and remain. + ## `config/policy.yaml` is optional `_load_policy_rules` logs `No policy file found ..., skipping policy setup` and diff --git a/.claude/skills/porting-to-ventis/traps.md b/.claude/skills/porting-to-canyonos-core/traps.md similarity index 60% rename from .claude/skills/porting-to-ventis/traps.md rename to .claude/skills/porting-to-canyonos-core/traps.md index 4c25940..2cd2239 100644 --- a/.claude/skills/porting-to-ventis/traps.md +++ b/.claude/skills/porting-to-canyonos-core/traps.md @@ -1,7 +1,7 @@ # Traps Symptom-to-cause lookup for a port that is already written. The mechanism behind -each row is in `ventis-contract.md`. Rows marked with a check id are decided +each row is in `canyonos-core-contract.md`. Rows marked with a check id are decided before any of this happens by `validate.py`. ## Before any container starts @@ -10,13 +10,13 @@ before any of this happens by `validate.py`. | Symptom | Cause | | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | | `env_file does not exist` on deploy | the path is resolved against the project root you run from; `cmd_deploy` fails before launching anything (V030) | -| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved (V012) | -| `TypeError: int() argument ...` naming `replicas` | `replicas:` is a list; `_get_replica_placements` accepts that shape but `InstanceManager` calls `int()` on it (V013) | -| `AttributeError` inside `GlobalController.__init__` | `config/policy.yaml` exists but is empty, or its `rules:` is null. Absent would have been fine (V021) | -| `EC2 deploy preflight failed: missing ec2 config keys`| no top-level `ec2:` block, or an incomplete one. `ssh_user` passes the CLI's shorter list and fails later at provision (V022) | +| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved | +| `TypeError: int() argument ...` naming `replicas` | `replicas:` is a list; `_get_replica_placements` accepts that shape but `InstanceManager` calls `int()` on it | +| `AttributeError` inside `GlobalController.__init__` | `config/policy.yaml` exists but is empty, or its `rules:` is null. Absent would have been fine | +| `EC2 deploy preflight failed: missing ec2 config keys`| no top-level `ec2:` block, or an incomplete one. `ssh_user` passes the CLI's shorter list and fails later at provision | | `generated grpc_stubs are missing or not importable` | `ventis build` has not run on this host, or its output was cleaned | -| An agent missing from the deployment | its config `name` matched no yaml, or its entry has no `entrypoint`; the build logged a warning and exited 0 (V003, V005) | -| Two agents, one image | two config `name`s differing only in case — both tag `ventis-` and the second overwrites the first (V004) | +| An agent missing from the deployment | its config `name` matched no yaml, or its entry has no `entrypoint`; inspect the build warnings | +| Two agents, one image | two config `name`s differing only in case — both tag `ventis-` and the second overwrites the first | ## The container dies or serves nothing @@ -28,12 +28,38 @@ before any of this happens by `validate.py`. | `"No agent loaded"` on the first request | anything below — the agent container's stdout is the only place the cause exists | | A replica reports `healthy` but answers nothing | same; `healthy` is written before `_load_agent` runs and is never revised | | `Missing credentials` loading the agent | no `env_file:`, or the key the source reads is not in it | -| `ModuleNotFoundError` for the source's own modules | only modules that land flat at `/app` import; on a Ventis with `-e .`, the project also declares no packaging metadata (V031) | +| `ModuleNotFoundError` for the source's own modules | the original import does not resolve from `/app`; when editable installs are supported, add minimal packaging metadata at the port root (nested source metadata is ignored) (V031) | | `ModuleNotFoundError` for a third-party package | an import the source needs is missing from the entry's `requirements:` (W006) | | `ModuleNotFoundError` for nothing in particular | `requirements:` was not a list of strings, so the whole list was dropped with one warning (V014) | | `NameError` importing a stub | a yaml `type` that is not a builtin (V010) | | A peer's real code behaves like an empty stub | a project module at the root shares a basename with an `agents/*.yaml`, and the stub is copied over it (V020) | -| The container dies on a Ventis module name | a project module at the root is called `local_controller.py`, `deploy.py`, `future.py` ... — the runtime is copied flat over it (V019) | +| The container dies on a CanyonOS Core module name | a project module at the root is called `local_controller.py`, `deploy.py`, `future.py` ... — the runtime is copied flat over it (V019) | + + +## Through `llm_proxy` + +| Symptom | Cause | +| --- | --- | +| `Connection refused` at `127.0.0.1:8080` | that address is the agent container itself, not the Docker host; use `host.docker.internal`, bind the proxy to `0.0.0.0`, and avoid the workflow's host port 8080 | +| Proxy `/healthz` works, but the agent cannot connect | the health probe ran on the host; check the base URL from inside the agent image and whether a remote agent host can reach the proxy | +| OpenAI or Anthropic client says its key is missing before making a request | its SDK still requires the normal key variable; give the container a dummy value and give the proxy process the real value separately | +| boto3 says it cannot locate credentials | botocore signs even a custom endpoint request; give the caller dummy AWS credentials while the proxy process retains its own real AWS identity | +| Upstream answers 401 through the proxy | the proxy process has no real provider key; `/healthz` checks registration, not credentials | +| `BEDROCK_UPSTREAM_HOST` appears to be ignored | it is read into `Config` but never passed to the proxy's boto3 client in this implementation | +| JSON `502` with `proxy_error` | proxy routing or provider code raised; inspect `detail` and proxy logs | +| JSON `502` naming `invoke-with-response-stream`, `converse`, or `converse-stream` | the Bedrock adapter implements only `invoke` | +| A streaming OpenAI or Anthropic call hangs or returns a buffered response | this proxy has no streaming implementation; the port is unsupported without changing source behavior | +| Local agents work but EC2 agents cannot connect | `host.docker.internal` on EC2 names each EC2 Docker host, not the deploying machine; expose a reachable proxy or run one per host | +| Proxy fails to bind port 8080 | the workflow API normally publishes the same host port; run the proxy on another port | + + +## During cleanup + +| Symptom | Cause | +| --- | --- | +| `ventis clean` succeeds but containers still run | the command removes generated directories only; stop `ventis deploy` and remove any exact leftovers | +| `ventis clean` succeeds but `ventis-*` images remain | image deletion is not part of `cmd_clean`; remove the exact tags after their containers stop | +| The next deploy says a port or container name is already in use | the previous deploy crashed or was killed before `GlobalController.cleanup` completed | ## The request is accepted and then goes wrong @@ -42,7 +68,7 @@ before any of this happens by `validate.py`. | Symptom | Cause | | --------------------------------------------------- | ------------------------------------------------------------------------------------- | | `TypeError: unexpected keyword argument` | yaml `arguments[].name` != the Python parameter name (V008) | -| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched (V021) | +| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched | | `.value()` returns a `str` of a dict | expected — `json.loads` it | | `Object of type ... is not JSON serializable` | the adapter returned framework objects; serialize with the framework's own serializer | | Redis holds `` | the method is `async def`; keep the signature sync and `asyncio.run` inside (V009) | diff --git a/.claude/skills/porting-to-ventis/validate.py b/.claude/skills/porting-to-canyonos-core/validate.py similarity index 61% rename from .claude/skills/porting-to-ventis/validate.py rename to .claude/skills/porting-to-canyonos-core/validate.py index 3647111..d854121 100755 --- a/.claude/skills/porting-to-ventis/validate.py +++ b/.claude/skills/porting-to-canyonos-core/validate.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 -"""Deterministic checks for a Ventis port. +"""Preflight the runtime traps that `ventis build` cannot see. -Every rule in SKILL.md marked MUST or NEVER that a machine can decide is decided -here. Nothing in this file imports the port -- YAML is parsed, Python is parsed -to an AST, and neither is executed. A port that fails here fails at build, at -deploy, or on its first request; `ventis build` will not tell you, because it -never imports your agent, and a replica will not tell you either, because the -controller writes `healthy` to Redis before `_load_agent` runs. +This deliberately does not duplicate build-time validation such as malformed +YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those +checks. This script parses Python without importing it and catches failures that +otherwise stay hidden until a container loads an agent, starts a workflow, or +serves its first request. A replica is not evidence: the controller writes +`healthy` to Redis before `_load_agent` runs. python validate.py [project_dir] [-c config/global_controller.yaml] [--json] [--strict] Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. -Some rules depend on Ventis features that are not on `main`. Rather than assume, +Some rules depend on CanyonOS Core features that are not on `main`. Rather than assume, this script probes the importable `ventis` package and reports each capability with the PR that carries it. A check whose capability is absent is reported as UNAVAILABLE, never silently skipped. @@ -25,13 +25,12 @@ import json import os import re -import subprocess import sys from typing import ClassVar try: import yaml -except ImportError: # pragma: no cover - pyyaml is a Ventis dependency +except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") raise SystemExit(2) from None @@ -68,22 +67,6 @@ "ipython", "boto3", ] -BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + [ - "flask", - "sqlalchemy", - "psycopg[binary]", -] - -# ventis/cli.py EC2_REQUIRED_CONFIG_KEYS is shorter than what EC2/_runtime.py -# actually demands; the CLI preflight passes and provisioning then fails. -EC2_REQUIRED_CONFIG_KEYS = ( - "ami_id", - "subnet_id", - "security_group_ids", - "region", - "ssh_user", -) - # Import name -> distribution name, for the handful where they differ and the # mismatch would otherwise be reported as a missing requirement. IMPORT_TO_DISTRIBUTION = { @@ -112,8 +95,6 @@ ] SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) -MIN_COPIED_LITERAL = 80 - ERROR = "ERROR" WARN = "WARN" INFO = "INFO" @@ -224,7 +205,6 @@ def __init__(self, project_dir, capabilities): self.project_dir = project_dir self.capabilities = capabilities self.findings = [] - self.reported_ec2_block = False # A peer agent is imported by the name of its generated stub, which the # build copies flat into every image. Those are not project modules and # need no requirement. @@ -334,364 +314,7 @@ def toplevel_import_names(tree): # ------------------------------------------------------------------ # -# V001-V002 the files parse at all # -# ------------------------------------------------------------------ # - - -def check_config_loads(report, config_path): - """V001 -- cli.py _load_config, then cmd_build's config.get("agents", []).""" - if not os.path.isfile(config_path): - report.error( - "V001", - config_path, - 0, - "config/global_controller.yaml is missing", - "cli.py cmd_build logs 'Config file not found' and exits 1.", - ) - return None - config, error = load_yaml(config_path) - if error is not None: - report.error("V001", config_path, 0, f"unparseable YAML: {error}", "") - return None - if not isinstance(config, dict): - report.error( - "V001", - config_path, - 0, - "the config is empty or is not a mapping", - "cmd_build calls config.get('agents', []) on it -- AttributeError.", - ) - return None - if "agents" not in config: - report.error( - "V001", - config_path, - line_of(config), - "no `agents:` key", - "Nothing is built and nothing is deployed.", - ) - return config - agents = config.get("agents") - if agents is None or not isinstance(agents, list): - report.error( - "V001", - config_path, - line_of(config, "agents"), - "`agents:` is null or is not a list", - "cmd_build iterates it as a list -- TypeError before any image.", - ) - config["agents"] = [] - return config - - -def check_agent_yaml_loads(report, path): - """V002 -- stub_generator reads agent/name with [], not .get().""" - data, error = load_yaml(path) - if error is not None: - report.error("V002", path, 0, f"unparseable YAML: {error}", "") - return None - if not isinstance(data, dict): - report.error( - "V002", - path, - 0, - "the file is empty or is not a mapping", - "cmd_build does yaml.safe_load(f).get('agent', {}) -- AttributeError.", - ) - return None - agent = data.get("agent") - if not isinstance(agent, dict): - report.error( - "V002", - path, - line_of(data, "agent"), - "`agent:` is missing or null", - "stub_generator does config['agent'] -- KeyError, or AttributeError " - "in cmd_build's name index.", - ) - return None - name = agent.get("name") - if not isinstance(name, str) or not name: - report.error( - "V002", - path, - line_of(agent, "name") or line_of(agent), - "`agent.name` is missing or is not a string", - "It becomes the generated class name and ENV VENTIS_AGENT_NAME.", - ) - return None - if "functions" in agent and agent.get("functions") is None: - report.error( - "V002", - path, - line_of(agent, "functions"), - "`functions:` is present but null", - "stub_generator iterates it -- TypeError: 'NoneType' is not iterable. " - "Omit the key instead.", - ) - for func in agent.get("functions") or []: - if not isinstance(func, dict) or not isinstance(func.get("name"), str): - report.error( - "V002", - path, - line_of(agent, "functions"), - "a function entry has no string `name`", - "stub_generator does func_config['name'] -- KeyError.", - ) - continue - if "arguments" in func and func.get("arguments") is None: - report.error( - "V002", - path, - line_of(func, "arguments"), - f"`{func['name']}.arguments:` is present but null", - "stub_generator iterates it -- TypeError. Omit the key instead.", - ) - return data - - -# ------------------------------------------------------------------ # -# V003-V005, V012-V015, V022 the config entries # -# ------------------------------------------------------------------ # - - -def check_config_entries(report, config, config_path, project_dir, yaml_by_name): - """V003 V004 V005 V012 V013 V014 V015 V022.""" - agents = config.get("agents") or [] - seen_lower = {} - workflow_entries = [] - - for entry in agents: - if not isinstance(entry, dict): - report.error( - "V001", - config_path, - line_of(config, "agents"), - f"an `agents:` item is not a mapping: {entry!r}", - "cmd_build does agent_cfg['name'] on it.", - ) - continue - name = entry.get("name") - if not isinstance(name, str) or not name: - report.error( - "V003", - config_path, - line_of(entry), - "an `agents:` entry has no string `name`", - "cmd_build does agent_cfg['name'] -- KeyError.", - ) - continue - - # V004 -- the image tag is ventis-. - previous = seen_lower.get(name.lower()) - if previous is not None: - report.error( - "V004", - config_path, - line_of(entry, "name"), - f"`{name}` and `{previous}` differ only in case", - "Both build to the image tag ventis-" - f"{name.lower()}; the second overwrites the first.", - ) - seen_lower[name.lower()] = name - - entry_type = entry.get("type", "agent") - if entry_type == "workflow": - workflow_entries.append((name, entry)) - check_workflow_entry(report, entry, config_path, project_dir) - else: - check_agent_entry( - report, name, entry, config_path, project_dir, yaml_by_name - ) - - check_provider(report, name, entry, config_path) - check_replicas(report, name, entry, config_path) - check_requirements(report, name, entry, config_path) - check_ec2_entry(report, name, entry, config, config_path) - - # V015 -- without a workflow entry nothing serves HTTP. - if not workflow_entries: - report.error( - "V015", - config_path, - line_of(config, "agents"), - "no entry has `type: workflow`", - "Nothing builds a Flask container, so the port has no HTTP surface.", - ) - elif len(workflow_entries) > 1: - names = ", ".join(name for name, _ in workflow_entries) - report.error( - "V015", - config_path, - line_of(config, "agents"), - f"more than one `type: workflow` entry: {names}", - "Every workflow builds into docker_container/Workflow; the last wins.", - ) - return workflow_entries - - -def check_agent_entry(report, name, entry, config_path, project_dir, yaml_by_name): - """V003 V005.""" - entrypoint = entry.get("entrypoint") - if not entrypoint: - report.error( - "V005", - config_path, - line_of(entry), - f"agent `{name}` has no `entrypoint`", - "cmd_build warns 'Skipping agent', builds no image, and exits 0.", - ) - elif not os.path.isfile(os.path.join(project_dir, entrypoint)): - report.error( - "V005", - config_path, - line_of(entry, "entrypoint"), - f"agent `{name}`: entrypoint `{entrypoint}` does not exist", - "cmd_build logs 'Agent file not found', skips it, and exits 0.", - ) - if name not in yaml_by_name: - report.error( - "V003", - config_path, - line_of(entry, "name"), - f"no agents/*.yaml declares `agent.name: {name}`", - "cmd_build warns 'No YAML definition found', builds no image for it, " - "and exits 0 -- the agent is simply absent from the deployment.", - ) - - -def check_workflow_entry(report, entry, config_path, project_dir): - """V015.""" - workflow_file = entry.get("workflow_file") - if not workflow_file: - report.error( - "V015", - config_path, - line_of(entry), - "the workflow entry has no `workflow_file`", - "cmd_build warns 'Skipping workflow' and exits 0.", - ) - elif not os.path.isfile(os.path.join(project_dir, workflow_file)): - report.error( - "V015", - config_path, - line_of(entry, "workflow_file"), - f"`workflow_file: {workflow_file}` does not exist", - "cmd_build logs 'Workflow file not found', skips it, and exits 0.", - ) - - -def check_provider(report, name, entry, config_path): - """V012 -- provider == "local" is compared case-sensitively; EC2 is not.""" - provider = entry.get("provider", "local") - if not isinstance(provider, str): - report.error( - "V012", - config_path, - line_of(entry, "provider"), - f"`{name}`: provider must be a string, got {provider!r}", - "InstanceManager compares it to the literal 'local'.", - ) - return - if provider == "local" or provider.upper() == "EC2": - return - if provider.lower() == "local": - report.error( - "V012", - config_path, - line_of(entry, "provider"), - f"`{name}`: `provider: {provider}` must be lowercase `local`", - "InstanceManager.ensure_instances tests provider == 'local' to " - "reserve a host port. Any other casing leaves reserved_port None and " - "Local/_runtime.py dies on int(None) before a container starts.", - ) - else: - report.error( - "V012", - config_path, - line_of(entry, "provider"), - f"`{name}`: unknown `provider: {provider}`", - "Only 'local' (exact) and 'EC2' (any casing) are recognised.", - ) - - -def check_replicas(report, name, entry, config_path): - """V013 -- InstanceManager does int(replicas).""" - if "replicas" not in entry: - return - replicas = entry.get("replicas") - if isinstance(replicas, bool) or not isinstance(replicas, int): - report.error( - "V013", - config_path, - line_of(entry, "replicas"), - f"`{name}`: `replicas` must be an int, got {replicas!r}", - "InstanceManager.ensure_instances does range(int(replicas)); the " - "list form GlobalController._get_replica_placements accepts raises " - "TypeError here.", - ) - elif replicas < 1: - report.error( - "V013", - config_path, - line_of(entry, "replicas"), - f"`{name}`: `replicas: {replicas}` launches nothing", - "range(0) -- the agent is deployed with no instances.", - ) - - -def check_requirements(report, name, entry, config_path): - """V014 -- one bad item drops the whole list, with only a warning.""" - if "requirements" not in entry: - return - requirements = entry.get("requirements") - if requirements is None: - return - if not isinstance(requirements, list) or not all( - isinstance(item, str) for item in requirements - ): - report.error( - "V014", - config_path, - line_of(entry, "requirements"), - f"`{name}`: `requirements` must be a list of strings", - "_normalize_requirements logs one warning and returns [] -- the " - "whole list is dropped, not the bad item, and the build still " - "succeeds with none of them installed.", - ) - - -def check_ec2_entry(report, name, entry, config, config_path): - """V022 -- the CLI preflight list is shorter than what provisioning needs.""" - provider = entry.get("provider", "local") - if not isinstance(provider, str) or provider.upper() != "EC2": - return - if not entry.get("instance_type"): - report.error( - "V022", - config_path, - line_of(entry), - f"`{name}`: EC2 entry has no `instance_type`", - "EC2/_runtime.py does spec['instance_type'] -- KeyError at provision.", - ) - if report.reported_ec2_block: - return - report.reported_ec2_block = True - ec2 = config.get("ec2") or {} - missing = [key for key in EC2_REQUIRED_CONFIG_KEYS if not ec2.get(key)] - if missing: - report.error( - "V022", - config_path, - line_of(config, "ec2") or line_of(config), - f"top-level `ec2:` is missing {', '.join(missing)}", - "cli.py's preflight checks only four of these; ssh_user is demanded " - "later by EC2/_runtime.py, after preflight has already passed.", - ) - - -# ------------------------------------------------------------------ # -# V006-V010, W005 the adapter against its yaml # +# V006-V010 adapter failures hidden by _load_agent # # ------------------------------------------------------------------ # BUILTIN_TYPE_NAMES = frozenset( @@ -702,7 +325,7 @@ def check_ec2_entry(report, name, entry, config, config_path): def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): - """V006 V007 V008 V009 V010 W005.""" + """V006 V007 V008 V009 V010.""" name = agent_block["name"] functions = agent_block.get("functions") or [] check_argument_types(report, agent_yaml_path, functions) @@ -759,15 +382,7 @@ def check_argument_types(report, agent_yaml_path, functions): continue declared = arg.get("type") if not isinstance(declared, str): - report.error( - "V010", - agent_yaml_path, - line_of(arg, "type"), - f"`type: {declared!r}` is not a string", - "ast.Name(id=) then ast.unparse -- a non-string raises " - "while the stub is generated.", - ) - continue + continue # stub generation reports malformed type values if declared in BUILTIN_TYPE_NAMES: continue report.error( @@ -802,7 +417,7 @@ def check_constructor(report, entrypoint_path, name, methods): def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): - """V008 V009 W005.""" + """V008 V009.""" func_name = func["name"] method = methods.get(func_name) if method is None: @@ -863,22 +478,6 @@ def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, met "in the signature.", ) - # W005 -- `returns` is read by nothing, but it is how the workflow author - # learns the call site needs json.loads. - returns = func.get("returns") - declared_return = returns.get("type") if isinstance(returns, dict) else None - annotation = method.returns - annotated = annotation.id if isinstance(annotation, ast.Name) else None - if annotated in ("dict", "list") and declared_return != annotated: - report.warn( - "W005", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` returns {annotated} but the yaml " - f"declares `returns.type: {declared_return}`", - "`returns` is read by nothing; its only job is telling whoever " - "writes the workflow that .value() hands back a string to json.loads.", - ) # ------------------------------------------------------------------ # @@ -982,7 +581,7 @@ def check_workflow(report, workflow_path, stub_classes=None): workflow_path, 1, f"no top-level function named `main` (found: {found})", - "Ventis serves POST /, but the deployment platform's " + "CanyonOS Core serves POST /, but the deployment platform's " "test endpoint posts to a hardcoded /main. A differently named " "workflow builds, deploys and stays unreachable -- 404, container " "healthy.", @@ -1111,7 +710,7 @@ def check_fused_fanout(report, workflow_path, tree): ".value() blocks, so each call completes before the " "next is dispatched. It does not error -- the fan-out " "is just silently serial, and with it the reason to be " - "on Ventis. Dispatch every call first, then resolve: " + "on CanyonOS Core. Dispatch every call first, then resolve: " "futures = [a.work(i) for i in items] then " "[f.value() for f in futures].", ) @@ -1137,8 +736,8 @@ def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): path, 1, f"a project module named `{entry}` sits at the project root", - "The shared Ventis runtime is copied flat into the image after " - "the project sweep, so this file is overwritten by Ventis's own " + "The shared CanyonOS Core runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by CanyonOS Core's own " f"{entry}. Rename it or move it into a package directory.", ) @@ -1164,120 +763,6 @@ def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): ) -# ------------------------------------------------------------------ # -# V021 policy.yaml # -# ------------------------------------------------------------------ # - - -def check_policy(report, config, config_path): - """V021 -- absent is fine; present-but-empty kills deploy.""" - policy_path = os.path.join( - os.path.dirname(os.path.abspath(config_path)), "policy.yaml" - ) - if not os.path.isfile(policy_path): - # _load_policy_rules logs and returns [], and _check_policy allows - # everything when the rule list is empty. Nothing to check. - return - - policy, error = load_yaml(policy_path) - if error is not None: - report.error("V021", policy_path, 0, f"unparseable YAML: {error}", "") - return - if not isinstance(policy, dict): - report.error( - "V021", - policy_path, - 0, - "policy.yaml exists but is empty", - "_load_policy_rules does policy_config.get('rules', []) on None -- " - "AttributeError inside GlobalController.__init__, so `ventis deploy` " - "dies before any container starts. Delete the file instead; absent " - "means everything is allowed.", - ) - return - - rules = policy.get("rules") - if rules is None or not isinstance(rules, list): - report.error( - "V021", - policy_path, - line_of(policy, "rules") or line_of(policy), - "`rules:` is null or is not a list", - "_load_policy_rules calls .sort() on it -- AttributeError inside " - "GlobalController.__init__, before any container starts.", - ) - return - - declared = [ - entry.get("name") - for entry in config.get("agents") or [] - if isinstance(entry, dict) and isinstance(entry.get("name"), str) - ] - fallback = None - for rule in rules: - if not isinstance(rule, dict): - report.error( - "V021", - policy_path, - line_of(policy, "rules"), - f"a rule is not a mapping: {rule!r}", - "_check_policy does rule.get('match', {}) on it.", - ) - continue - match = rule.get("match") - if match is None or (isinstance(match, dict) and not match): - fallback = rule - - if fallback is None: - report.error( - "V021", - policy_path, - line_of(policy, "rules"), - "no rule with an empty `match: {}`", - "_check_policy denies access when no rule matches the request " - "context, so every service answers Unauthorized after its request " - "was already accepted with a 202.", - ) - return - - if not isinstance(fallback.get("access"), (list, str)): - report.error( - "V021", - policy_path, - line_of(fallback, "access") or line_of(fallback), - "the `match: {}` rule's `access` is neither a list nor `all`", - "_check_policy does `service in access`.", - ) - return - - # Reachable under *some* context. A service deliberately restricted to one - # caller -- text2sql keeps ProductionExecutorAgent out of the fallback and - # reaches it only through an `access: all` rule -- is correct policy, not a - # defect, so only a service no rule can ever reach is worth reporting. - reachable = set() - for rule in rules: - if not isinstance(rule, dict): - continue - access = rule.get("access") - if access == "all": - reachable.update(declared) - elif isinstance(access, list): - reachable.update(item for item in access if isinstance(item, str)) - - unreachable = [name for name in declared if name not in reachable] - if unreachable: - report.warn( - "V021", - policy_path, - line_of(policy, "rules"), - f"no rule grants access to {', '.join(unreachable)}", - "The first matching rule decides, and a service named in none of " - "them answers Unauthorized on /status after the request was " - "already accepted with a 202. Intentional if the service is meant " - "to be unreachable.", - ) - - # ------------------------------------------------------------------ # # V030-V031 capability-gated rules # # ------------------------------------------------------------------ # @@ -1294,7 +779,7 @@ def check_env_file(report, config, config_path, project_dir): "V030", config_path, line_of(config, "env_file"), - f"`env_file: {declared}` is set, but this Ventis never reads it", + f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", "No resolve_env_file in the importable ventis package, so the " "key is silently dropped and the container answers a provider " "credential error on the first request. It arrives with " @@ -1303,7 +788,7 @@ def check_env_file(report, config, config_path, project_dir): else: report.unavailable( "V030", - "env_file is not supported by the importable Ventis " + "env_file is not supported by the importable `ventis` runtime " f"({CAPABILITY_SOURCE['env_file']}). Credentials have no " "declared path into a container on this tree.", ) @@ -1321,27 +806,8 @@ def check_env_file(report, config, config_path, project_dir): ) return - resolved = os.path.expanduser(str(declared)) - if not os.path.isabs(resolved): - resolved = os.path.join(project_dir, resolved) - if not os.path.isfile(resolved): - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` does not resolve to a file", - "resolve_env_file raises before GlobalController exists, so " - "`ventis deploy` fails with one error line. The path is resolved " - "against the project root you run from.", - ) - elif not os.access(resolved, os.R_OK): - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` is not readable", - "resolve_env_file raises on an unreadable file.", - ) + # Path existence and readability are deploy-preflight checks. Do not + # duplicate them here. def check_import_root(report, project_dir, entrypoint_paths): @@ -1370,7 +836,7 @@ def check_import_root(report, project_dir, entrypoint_paths): report.unavailable( "V031", "the editable install (`-e .`) is not supported by the importable " - f"Ventis ({CAPABILITY_SOURCE['editable_install']}). Only modules " + f"`ventis` runtime ({CAPABILITY_SOURCE['editable_install']}). Only modules " "that land flat at /app import inside a container.", ) for path, lineno, name, location in non_flat: @@ -1380,7 +846,7 @@ def check_import_root(report, project_dir, entrypoint_paths): lineno, f"`import {name}` resolves to {location}, which is not at the " "project root", - "sys.path[0] is /app and this Ventis runs no editable install, " + "sys.path[0] is /app and this CanyonOS Core runs no editable install, " "so only modules swept to the root import. The adapter raises " "ModuleNotFoundError inside _load_agent and the first request " "answers 'No agent loaded'.", @@ -1395,81 +861,41 @@ def check_import_root(report, project_dir, entrypoint_paths): lineno, f"`import {name}` resolves to {location}, and the project root " "has no packaging metadata", - "A pyproject.toml, setup.py or setup.cfg at the root is what " - "adds `-e .`, and the project's own metadata is what decides the " - "import root. Without it the install is skipped silently and " - "only flat modules import.", + "A pyproject.toml, setup.py or setup.cfg at the port root is " + "what adds `-e .`; metadata nested inside the untouched source " + "tree is ignored. Add minimal root metadata pointing at the " + "existing package directory. Without it the install is skipped " + "silently.", ) def _resolves_flat(project_dir, name): - return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isfile( - os.path.join(project_dir, name, "__init__.py") + """Whether Python can resolve `name` with /app as its import root. + + A directory does not need __init__.py: PEP 420 namespace packages resolve + from sys.path just like regular packages. + """ + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( + os.path.join(project_dir, name) ) def _resolves_nested(project_dir, name): - """Where inside the tree `name` lives, if it is a project module at all.""" + """Where below /app `name` lives but cannot resolve as a top-level name.""" for root, dirs, files in os.walk(project_dir): dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] if root == project_dir: continue if f"{name}.py" in files: return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) - if name in dirs and os.path.isfile(os.path.join(root, name, "__init__.py")): + if name in dirs: return os.path.relpath(os.path.join(root, name), project_dir) return None # ------------------------------------------------------------------ # -# W001-W006 the rewrite smells # +# W003, W006 secrets and imports a green build does not reject # # ------------------------------------------------------------------ # -# -# Warnings, not errors: each is a heuristic, and a false positive must never -# block a correct port. --strict promotes them for CI. - - -def _string_literals(tree, minimum): - for node in ast.walk(tree): - if ( - isinstance(node, ast.Constant) - and isinstance(node.value, str) - and len(node.value.strip()) >= minimum - ): - yield node.value, node.lineno - - -def check_copied_literals(report, project_dir, port_paths, source_paths): - """W001 -- a prompt that exists in the source and again in the adapter.""" - source_text = {} - for path in source_paths: - try: - with open(path, "r", encoding="utf-8") as handle: - source_text[path] = handle.read() - except OSError: - continue - if not source_text: - return - - for port_path in port_paths: - tree, _ = parse_python(port_path) - if tree is None: - continue - for literal, lineno in _string_literals(tree, MIN_COPIED_LITERAL): - for source_path, text in source_text.items(): - if literal in text: - report.warn( - "W001", - port_path, - lineno, - f"a {len(literal)}-character string literal also appears " - f"in {report.rel(source_path)}", - "It exists in the source. Import it -- the whole tree is " - "in the image, and a copy drifts the moment the source " - "changes. A port that restates a prompt has rewritten " - "the project, not ported it.", - ) - break def check_secrets(report, port_paths): @@ -1518,53 +944,6 @@ def check_secrets(report, port_paths): ) -def check_source_tree_clean(report, project_dir): - """W002 -- the port must leave `git status` on the source clean.""" - - def git(*args): - try: - result = subprocess.run( - ["git", *args], - cwd=project_dir, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - return result.stdout if result.returncode == 0 else None - - # `git status` prints paths relative to the repo root, so a port nested - # inside a larger repo needs that prefix stripped before the port's own - # directories can be recognised. - prefix = git("rev-parse", "--show-prefix") - if prefix is None: - return - prefix = prefix.strip() - status = git("status", "--porcelain", "--", ".") - if status is None: - return - - port_prefixes = ("agents/", "workflow/", "config/") - generated = ("docker_container/", "stubs/", "grpc_stubs/") - for line in status.splitlines(): - path = line[3:].strip().strip('"') - if prefix and path.startswith(prefix): - path = path[len(prefix) :] - if not path or path.startswith(port_prefixes) or path.startswith(generated): - continue - report.warn( - "W002", - os.path.join(project_dir, path), - 0, - f"`{path}` is modified or untracked outside the port's own files", - "A port adds agents/, workflow/ and config/ beside an untouched " - "source tree. If this is an edit to the source, it is a rewrite; if " - "it is unrelated local work, ignore this line.", - ) - - def _pyproject_dependencies(project_dir): """What `-e .` installs alongside `requirements:`, or None if unreadable. @@ -1675,43 +1054,46 @@ def _normalize_distribution(name): def validate(project_dir, config_path, capabilities): + """Inspect only failures hidden behind a successful image build.""" report = Report(project_dir, capabilities) - config = check_config_loads(report, config_path) - if config is None: + # The build owns config/YAML syntax and shape validation. We read only enough + # valid structure to locate code for the deeper checks below. + config, error = load_yaml(config_path) + if error is not None or not isinstance(config, dict): + report.unavailable( + "BUILD", + "runtime preflight skipped because the config cannot be read; " + "ventis build owns and reports this error.", + ) return report import glob yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) - if not yaml_paths: - report.error( - "V002", - os.path.join(project_dir, "agents"), - 0, - "no agents/*.yaml files", - "cmd_build warns 'No agent YAML files found'; no stubs are generated " - "and no agent image is built.", - ) - report.stub_module_names = { os.path.splitext(os.path.basename(path))[0] for path in yaml_paths } agents_by_name = {} - # basename of agents/.yaml -> the class its stub defines, which is - # the agent's own name. stub_classes = {} for path in yaml_paths: - data = check_agent_yaml_loads(report, path) - if data is not None: - agents_by_name[data["agent"]["name"]] = (path, data["agent"]) - stub_classes[os.path.splitext(os.path.basename(path))[0]] = data["agent"][ - "name" - ] - - entries = config.get("agents") or [] - check_config_entries(report, config, config_path, project_dir, set(agents_by_name)) + data, yaml_error = load_yaml(path) + agent = data.get("agent") if isinstance(data, dict) else None + name = agent.get("name") if isinstance(agent, dict) else None + if yaml_error is not None or not isinstance(name, str): + continue # ventis build reports malformed agent declarations + agents_by_name[name] = (path, agent) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = name + + entries = config.get("agents") + if not isinstance(entries, list): + report.unavailable( + "BUILD", + "runtime preflight skipped because `agents:` is not a list; " + "ventis build owns and reports this error.", + ) + return report entrypoints = [] for entry in entries: @@ -1719,13 +1101,13 @@ def validate(project_dir, config_path, capabilities): continue name = entry.get("name") entrypoint = entry.get("entrypoint") - if entrypoint: + if isinstance(entrypoint, str): entrypoints.append(entrypoint) if name in agents_by_name: yaml_path, agent_block = agents_by_name[name] check_adapter(report, yaml_path, agent_block, entry, project_dir) entrypoint_path = os.path.join(project_dir, entrypoint or "") - if entrypoint and os.path.isfile(entrypoint_path): + if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): check_requirements_coverage( report, project_dir, entry, entrypoint_path, config_path ) @@ -1734,14 +1116,15 @@ def validate(project_dir, config_path, capabilities): if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": continue workflow_file = entry.get("workflow_file") - if not workflow_file: + if not isinstance(workflow_file, str): continue workflow_path = os.path.join(project_dir, workflow_file) if os.path.isfile(workflow_path): check_workflow(report, workflow_path, stub_classes) + # These survive a green build and otherwise surface only in a container or + # on its first request. check_flat_collisions(report, project_dir, yaml_paths, entrypoints) - check_policy(report, config, config_path) check_env_file(report, config, config_path, project_dir) entrypoint_paths = [ @@ -1753,40 +1136,17 @@ def validate(project_dir, config_path, capabilities): port_paths = list(entrypoint_paths) for entry in entries: - if isinstance(entry, dict) and entry.get("workflow_file"): + if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): candidate = os.path.join(project_dir, entry["workflow_file"]) if os.path.isfile(candidate): port_paths.append(candidate) - source_paths = _source_paths(project_dir, port_paths) - check_copied_literals(report, project_dir, port_paths, source_paths) + # Secret detection remains because a green image build would permanently + # bake the credential into every image. check_secrets(report, port_paths) - check_source_tree_clean(report, project_dir) return report -def _source_paths(project_dir, port_paths): - """Every project .py that is not one of the port's own four files.""" - excluded = {os.path.abspath(p) for p in port_paths} - found = [] - for root, dirs, files in os.walk(project_dir): - dirs[:] = [ - d - for d in dirs - if not d.startswith(".") - and d != "__pycache__" - and not ( - root == project_dir and d in ("docker_container", "stubs", "grpc_stubs") - ) - ] - for name in files: - if not name.endswith(".py"): - continue - path = os.path.join(root, name) - if os.path.abspath(path) not in excluded: - found.append(path) - return found - # ------------------------------------------------------------------ # # Output # @@ -1817,7 +1177,7 @@ def print_report(report, project_dir): print("ventis is not importable here -- capability-gated rules are") print("reported UNAVAILABLE rather than checked.\n") else: - print("Ventis capabilities detected:") + print("CanyonOS Core capabilities detected:") for key, source in CAPABILITY_SOURCE.items(): mark = "yes" if caps.get(key) else "no " print(f" {mark} {key:<22} {source}") @@ -1849,7 +1209,7 @@ def print_report(report, project_dir): def main(argv=None): parser = argparse.ArgumentParser( - description="Check a Ventis port against the rules in SKILL.md." + description="Check a CanyonOS Core port against the rules in SKILL.md." ) parser.add_argument( "project_dir", nargs="?", default=".", help="the port's project root" diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md index 7161d28..3ba7930 100644 --- a/examples/joke_writer/README.md +++ b/examples/joke_writer/README.md @@ -80,7 +80,7 @@ converse API, so each node asks for JSON in its prompt and validates the reply through the same pydantic schema upstream used. `_extract_json` exists only because `with_structured_output` used to do that work. -That rewrite is not something the `porting-to-ventis` skill should do on a +That rewrite is not something the `porting-to-canyonos-core` skill should do on a user's project — it is the credential wall, and the skill's instruction is to report it. It was done here deliberately, so that this example is one that actually deploys. @@ -110,7 +110,7 @@ $EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... > **`env_file:` needs PR #53** (`jiajunh/can-232-...`), still open against main. > Until it merges nothing in `ventis/` reads the key, so the steps below leave > the container without a credential and every request answers a Bedrock -> credential error. `python ../../.claude/skills/porting-to-ventis/validate.py .` +> credential error. `python ../../.claude/skills/porting-to-canyonos-core/validate.py .` > reports this as V030 and stops reporting it the day the PR lands. `config/global_controller.yaml` points `env_file:` at that file, and every From b826fa16fa909723c7d9c71208ce2e5586115abf Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 11:51:04 -0700 Subject: [PATCH 41/43] Remove testing porting skill from branch --- .../skills/testing-porting-to-ventis/SKILL.md | 320 ------------------ .../testing-porting-to-ventis/record.py | 75 ---- .../testing-porting-to-ventis/schema.sql | 38 --- 3 files changed, 433 deletions(-) delete mode 100644 .claude/skills/testing-porting-to-ventis/SKILL.md delete mode 100644 .claude/skills/testing-porting-to-ventis/record.py delete mode 100644 .claude/skills/testing-porting-to-ventis/schema.sql diff --git a/.claude/skills/testing-porting-to-ventis/SKILL.md b/.claude/skills/testing-porting-to-ventis/SKILL.md deleted file mode 100644 index d4a627b..0000000 --- a/.claude/skills/testing-porting-to-ventis/SKILL.md +++ /dev/null @@ -1,320 +0,0 @@ ---- -name: testing-porting-to-ventis -description: Use when running a repository through the porting-to-ventis skill to find out where the port stops, or when building a corpus of such results across many repositories ---- - -# Testing `porting-to-ventis` against a repository - -One repository per run. The output is a row in `.ventis-tests/results.sqlite` -saying how far the port got and what stopped it, plus a directory of every -command's raw output. - -**A pass rate is not the deliverable.** The deliverable is attribution: for each -repository, whether the blame lies with the skill, with Ventis, or with the -repository itself. A run that ends `blocked` because the repo needs a vector -store nobody configured is not evidence about the skill, and recording it as a -failure makes the whole corpus mean less than it appears to. - -## Two rules the run is built on - -**Never edit the source tree.** M19 and M20 are rules the skill is being tested -on. Rewriting a repo's model calls, or setting its model config to a different -provider, tests the rewrite instead — and every later failure becomes -unattributable. Add files beside the source; leave `git status` on the source -clean. - -**Never edit `porting-to-ventis` during a run.** Its git tree hash is pinned into -every row. A skill edited between repo 1 and repo 100 means the two were not -given the same test. Fix it between runs, as a new pinned version, and re-run -what it affects. - -## What you judge, and what you only record - -You do not do the port. A subagent does (step 4), and you stay the observer — -otherwise the run is graded by the person who wrote it. - -You judge two things: **whether the repo is in scope** (step 2) and **what the -result means** (the write-up). Everything else is a command whose exit code and -output you record verbatim. - -**Never decide that a build "basically worked".** `ventis build` prints -`Build complete.` and exits 0 for a project whose container dies on startup, so -its exit code is not evidence on its own — that is what the two probes in step 6 -are for. If a command failed, the stage failed, whatever you think of the reason. - -## The run - -Work in `.ventis-tests//`, with the clone at `src/` and every command's -output written into `artifacts/`. - -```bash -mkdir -p .ventis-tests//artifacts -git clone --depth 1 .ventis-tests//src -git -C .ventis-tests//src rev-parse HEAD # repo_sha -git rev-parse HEAD:.claude/skills/porting-to-ventis # skill_sha -git rev-parse HEAD:ventis # ventis_sha -``` - -| # | `farthest_step` | What runs | -| --- | --- | --- | -| 1 | `fetched` | the clone above | -| 2 | `screened` | your read of the repo — see below | -| 3 | `wired` | write `.env` beside the source with the keys the repo needs | -| 4 | `ported` | a **`general-purpose` subagent** running the `porting-to-ventis` skill | -| 5 | `validated` | `python .claude/skills/porting-to-ventis/validate.py .` | -| 6 | `built` | `ventis build -c config/global_controller.yaml`, then both probes | -| 7 | `deployed` | `ventis deploy -c config/global_controller.yaml`, backgrounded | -| 8 | `served` | `POST /main`, then poll `GET /status/` | -| 9 | — | tear down; not a stage, but the run is not over without it | - -`farthest_step` is the furthest stage reached. Step 5 is the exception: it does -**not** gate what follows. - -### Step 2 — read the repo before spending anything on it - -Answer these from the source. Each rejection below was learned by paying an -agent's full budget to rediscover it. - -| Question | Reject when | -| --- | --- | -| Is there a module the adapter can import from the project root? | No root-level `.py` **and** no `pyproject.toml`/`setup.py`/`setup.cfg`. Without packaging metadata nothing is importable at `/app`. This is M24, and it rejects most tutorial repos. | -| Which provider will it actually call? | It needs one whose key you do not have. | -| Does it need something to reach? | It reads the address or credentials of a service you are not standing up. Ventis provides Redis; everything else is on you. | -| Is there Python at all? | Notebooks only, or no LLM call anywhere. | -| Is it small to medium? | Hundreds of modules, or a framework rather than a project. | - -**Reading the imports is not enough to answer the provider question.** Every -LangGraph template reaches its model through `init_chat_model("anthropic/…")` or -a config default string, so a repo can depend entirely on Anthropic while -importing nothing named `anthropic` — and can import `langchain_openai` for its -embeddings while its chat model is Claude. Grep the string literals as well as -the imports, and take the union: - -```bash -grep -rnoE '"(openai|anthropic|google_genai|bedrock|cohere|mistralai)[:/][^"]+"' -``` - -A repo needing a provider you cannot serve is `blocked`, not `failed`. Record -which provider and stop — that count is the argument for obtaining the key. - -**Ask the backing-service question as a principle, not as a list.** Enumerating -prefixes reads as a checklist and lets everything unlisted through: a list -naming `ELASTICSEARCH_*` and `PINECONE_*` passed a repo whose first node calls -`int(os.getenv("SSH_PORT"))` against a remote host that does not exist. Read the -env vars the source actually reads, and for each one ask **what would have to be -running for this to work**: - -```bash -grep -rhoE "getenv\(\s*[\"'][A-Z_]{3,}|environ\[[\"'][A-Z_]{3,}" \ - | grep -oE "[A-Z_]{3,}" | sort -u -``` - -An LLM key you hold is fine. A host to SSH into, a vector store, a database, a -search API, an object store — anything the source must connect to and you are -not providing — is out of scope. A repo whose work is *reaching* such a service -stays out of scope even when a port of it builds and serves: the request returns -the source's own failure, and the run proves nothing about the skill. - -### Step 3 — the credential goes beside the source, never inside it - -Write `.env` at the project root with the real keys, and let the port's -`config/global_controller.yaml` point `env_file:` at it. Never bake a key into -the build context: `ventis build` sweeps the project into every image, and -`_sweep_project_files` skips dotfiles precisely so `.env` cannot ride along. - -### Step 4 — dispatch the port to a subagent. Do not port it yourself. - -**Send it to a `general-purpose` subagent — never a `fork`.** A fork inherits -this conversation, which is the one thing that must not reach the porter: by the -time you are running repo twenty you know that the build log misnames the stub -class, that probe 2 needs `--env-file`, that `-e .` installs the project's own -dependencies. None of that came from the skill. A porter carrying it will get -past traps the skill never warned it about, and the corpus will report a skill -that is better than the one a new reader actually gets. - -Porting it yourself is the same mistake wearing a second hat: you would be -grading a port whose every decision you made, knowing what you meant rather than -what the skill said. - -Give it the repo and the skill, and nothing else: - -``` -Port the project at onto Ventis. - -Use the porting-to-ventis skill and follow it as written, including the steps -that tell you to validate and to probe the built images. - -Do not ask for confirmation; there is nobody to answer. Where the skill tells -you to report something rather than fix it, write PORT_REPORT.md in the project -root and stop -- that counts as following it. - -Report back: what you wrote, what you ran and what it said, anything the skill -left you guessing about, and anything you had to work out that the skill could -have told you. -``` - -No hints, no warnings, no "watch out for". A trap you spare it is a trap the -skill gets credit for warning about. - -**Let it run the skill's own Step 3 and Step 4** — validate, build, probe. Those -are instructions in the artifact under test; a porter that skips them is not -following the skill, and stopping it would be measuring something else. - -**Then verify from scratch. Its report is a claim, not a result.** You have no -transcript of what it did, so what you record must come from what you can see -yourself: - -| Check | How | -| --- | --- | -| Did it edit the source? (M19) | `git status --porcelain` in `src/` — only new directories should appear | -| What did it actually write? | read the four files; the report is not evidence for them | -| Does validate pass? | run step 5 yourself; do not take the report's word | -| Does it build and load? | run step 6 yourself, both probes | - -A subagent reporting a green build where yours fails is a finding, not a -discrepancy to reconcile. Record what your own commands said. - -**Deploy and serve stay here.** A subagent holding a fleet of containers has no -clear owner for step 9, and the leak lands on the next repo. - -When the subagent reports and stops rather than porting — **that is the skill -working, and the run is `blocked`, not `failed`.** Those paths fire on things -Ventis cannot do, so the finding belongs in `core_issue`. - -Its answer to *"anything the skill left you guessing about"* is the most -valuable thing it returns: unlike a defect you find by tripping over it, that is -the skill's silence reported by the only reader who did not already know. - -### Step 6 — build, then probe twice - -```bash -ventis build -c config/global_controller.yaml - -# 1. the runtime, which fails before your agent is reached -docker run --rm ventis- python -c "import local_controller" - -# 2. the agent, loaded the way _load_agent loads it -docker run --rm ventis- python -c " -import importlib.util, sys -spec = importlib.util.spec_from_file_location('m', '.py') -m = importlib.util.module_from_spec(spec); sys.modules['m'] = m -spec.loader.exec_module(m); m.(); print('ok')" -``` - -Probe 1 catches the protobuf/gRPC wall — a `core_issue`, since the fix belongs in -`generate_docker`. Probe 2 catches everything `_load_agent` swallows, which -otherwise surfaces only as `"No agent loaded"` at step 8. - -### Step 7 — check the ports are free before deploying - -```bash -docker ps -a --format '{{.Names}}' | grep -i '^ventis-' || echo "clean" -lsof -nP -iTCP:8080 -sTCP:LISTEN -``` - -Both must come back empty. A container the previous repo left behind still holds -`:8080`, and `ventis deploy` fails on it with `Failed to launch -ventis-local-workflow-0` and nothing else — it drops docker's stderr, so the -message names the symptom and not the cause. Checking first costs a second; -diagnosing it afterwards costs `docker inspect` and a confused half hour. - -### Step 8 — served means the port answered - -`POST /main {"query": ...}` and poll `/status/`. **Send a query the -repo can actually act on** — read its README first. Asking an SSH operations -agent about "animals" tests nothing. - -An outer `"status": "done"` with an inner `"status": "failed"` means the port -worked and the project's own logic did not. That is `passed`, because the port -did what the skill promises — carry a request to the source and return the -source's own result — but say so in `analysis`. A bare missing env var -(`'ELASTICSEARCH_API_KEY'`) is `blocked`: nobody configured it. - -### Step 9 — tear down, especially when the run failed - -`ventis deploy` blocks and holds a fleet: one container per replica, one for the -workflow, and a Redis it started itself. None of it stops when the request -finishes, and a failed launch leaves a container behind in `Created` state that -the next run's own stale-container sweep does not clear. - -Run this at the end of every repo, on the failure paths too — a leak does not -break the repo that leaked, it breaks the next one: - -```bash -pkill -f "ventis deploy" -docker ps -aq --filter 'name=^ventis-local-' | xargs -r docker rm -f -docker ps -aq --filter 'name=^ventis-redis-' | xargs -r docker rm -f -ventis clean # stubs/, grpc_stubs/, docker_container/ -``` - -`xargs -r` rather than `docker rm -f $(...) 2>/dev/null`: the substitution form -runs `docker rm` with no arguments when nothing is left, which is an error, and -the `2>/dev/null` that hides it would hide a real removal failure just as well. - -Then confirm it worked, because believing it did is how the next repo fails: - -```bash -docker ps -a --format '{{.Names}}' | grep -i '^ventis-' && echo "STILL THERE" -``` - -Leave the clone and `artifacts/` in place. They are the row's evidence, and the -database only stores a path to them. - -**Known gap:** a subagent has no spend cap. The Python harness this replaced -passed `--max-budget-usd` per repo and recorded exhaustion as its own outcome; -there is no equivalent here, so a repo that sends a porter in circles costs -whatever it costs. Watch the first runs of any new repo shape. - -| `status` | When | -| --- | --- | -| `passed` | Step 8 returned the source's own result. | -| `blocked` | Nothing was tested: out of scope at step 2, a missing key or backing service, or the skill correctly reported-and-stopped. | -| `failed` | The port was attempted and something about it did not work. | - -`blocked` is not a soft `failed`. It means this repository produced no evidence -about the skill, and rows that produced no evidence must not be counted as if -they had. - -## Recording - -Always record, including for `blocked` runs — a rejection is the datum. - -```bash -python .claude/skills/testing-porting-to-ventis/record.py \ - --db .ventis-tests/results.sqlite <<'JSON' -{ - "repo": "https://github.com/owner/name", - "repo_sha": "…", "skill_sha": "…", "ventis_sha": "…", - "stars": 128, "framework": "langgraph", "is_multiagent": 1, - "description": "what it does, technically", - "farthest_step": "built", "status": "failed", "validate_ok": 1, - "core_issue": [{"kind": "runtime_import", "detail": "…"}], - "skill_issue": [{"kind": "no_fanout", "detail": "…"}], - "analysis": "what happened and why, in a few sentences", - "artifacts": ".ventis-tests//artifacts", - "started_at": "2026-08-28T00:00:00Z", "ended_at": "2026-08-28T00:10:00Z" -} -JSON -``` - -`core_issue` is what a Ventis owner must fix; `skill_issue` is what the skill -file must say better. Keep them apart — collapsing them loses the distinction the -whole exercise exists to produce. Leave both empty when the run had no findings. - -## Common mistakes - -| Mistake | What it costs | -| --- | --- | -| Screening on imports alone | Anthropic-only repos reach step 4 and burn a budget before failing | -| Skipping the `provider/model` grep | Same, and it is the default shape of every LangGraph template | -| Letting `validate.py` gate the build | The one case where a validation was wrong to block becomes unobservable | -| Treating `Build complete.` as evidence | A green build and a healthy replica are both compatible with a container that serves nothing | -| Running one probe instead of two | Probe 1's failure is a Ventis bug; probe 2's is the port's; neither covers the other | -| Scoring report-and-stop as `failed` | Counts the skill working as the skill failing, and buries the Ventis gap that caused it | -| Sending `{"query": "animals"}` to everything | `served` stops meaning anything | -| Screening backing services against a list of prefixes | Whatever is not on the list gets in — ask what must be running, not what matches | -| Editing the skill mid-corpus | The pass rate loses its denominator | -| Skipping teardown after a failed run | The next repo fails on a port this one still holds, and its error names the wrong thing | -| Dispatching the port to a `fork` | It inherits everything you have learned, and the skill gets credit for warning about traps it never mentions | -| Doing the port yourself | You grade a port whose decisions you made, knowing what you meant rather than what the skill said | -| Recording the subagent's claims | Its report is a claim. Run validate and build yourself and record what *your* commands said | diff --git a/.claude/skills/testing-porting-to-ventis/record.py b/.claude/skills/testing-porting-to-ventis/record.py deleted file mode 100644 index c3d25aa..0000000 --- a/.claude/skills/testing-porting-to-ventis/record.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -"""Write one test result into the results database. - -Reads a JSON object on stdin so that findings and analysis -- which contain -quotes, newlines and error text -- reach SQLite as data. Hand-quoting them into -a `sqlite3` heredoc is how a run's own error message ends up truncating the row -that was supposed to record it. - - python record.py --db .ventis-tests/results.sqlite <<'JSON' - {"repo": "...", "repo_sha": "...", ..., "core_issue": [...]} - JSON -""" - -from __future__ import annotations - -import argparse -import json -import sqlite3 -import sys -from pathlib import Path - -REPO_FIELDS = ("stars", "framework", "is_multiagent", "description") -TEST_FIELDS = ("repo", "repo_sha", "skill_sha", "ventis_sha", "farthest_step", - "status", "validate_ok", "core_issue", "skill_issue", "analysis", - "artifacts", "started_at", "ended_at") -REQUIRED = ("repo", "repo_sha", "skill_sha", "ventis_sha", "farthest_step", - "status", "artifacts", "started_at") -STATUSES = {"passed", "failed", "blocked"} - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--db", required=True) - ap.add_argument("--schema", default=str(Path(__file__).with_name("schema.sql"))) - args = ap.parse_args() - - row = json.load(sys.stdin) - - missing = [f for f in REQUIRED if not row.get(f)] - if missing: - print(f"missing required field(s): {', '.join(missing)}", file=sys.stderr) - return 2 - if row["status"] not in STATUSES: - print(f"status must be one of {sorted(STATUSES)}", file=sys.stderr) - return 2 - - db = Path(args.db) - db.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(db) - conn.executescript(Path(args.schema).read_text(encoding="utf-8")) - - with conn: - conn.execute("INSERT OR IGNORE INTO repos (repo) VALUES (?)", (row["repo"],)) - cols = {f: row[f] for f in REPO_FIELDS if row.get(f) is not None} - if cols: - assigns = ", ".join(f"{k} = ?" for k in cols) - conn.execute(f"UPDATE repos SET {assigns} WHERE repo = ?", - (*cols.values(), row["repo"])) - - test = {} - for f in TEST_FIELDS: - v = row.get(f) - test[f] = json.dumps(v) if isinstance(v, (list, dict)) else v - names = ", ".join(test) - marks = ", ".join("?" for _ in test) - cur = conn.execute(f"INSERT INTO tests ({names}) VALUES ({marks})", - tuple(test.values())) - - print(f"recorded test #{cur.lastrowid}: {row['repo']} -> " - f"{row['status']} at {row['farthest_step']}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.claude/skills/testing-porting-to-ventis/schema.sql b/.claude/skills/testing-porting-to-ventis/schema.sql deleted file mode 100644 index 41abbc9..0000000 --- a/.claude/skills/testing-porting-to-ventis/schema.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Results of running `porting-to-ventis` against a repository. --- --- Everything a machine can produce is a column; everything that needs judgement --- is written by the agent that ran the port. No analysis is stored that could be --- recomputed later from the artifacts directory -- running the pipeline is the --- expensive part, and reading its output afterwards is not. - -CREATE TABLE IF NOT EXISTS repos ( - id INTEGER PRIMARY KEY, - repo TEXT UNIQUE NOT NULL, -- github url - stars INTEGER, -- gh api - framework TEXT, -- langchain|langgraph|crewai|autogen|adk|plain - is_multiagent INTEGER, -- does one request fan out to independent work? - description TEXT -- technical, written by the agent -); - -CREATE TABLE IF NOT EXISTS tests ( - id INTEGER PRIMARY KEY, - repo TEXT NOT NULL REFERENCES repos(repo), - - -- The three pins. Nothing else here can be reconstructed once a run is over: - -- which source, which skill, and which Ventis produced this result. - repo_sha TEXT NOT NULL, - skill_sha TEXT NOT NULL, - ventis_sha TEXT NOT NULL, - - farthest_step TEXT NOT NULL, -- the furthest stage reached - status TEXT NOT NULL, -- passed|failed|blocked - validate_ok INTEGER, -- stage 5's verdict, kept apart from the outcome - - core_issue TEXT, -- json: findings a Ventis owner must fix - skill_issue TEXT, -- json: findings the skill file must fix - analysis TEXT, -- the agent's recap: what happened and why - - artifacts TEXT NOT NULL, -- directory holding every command's output - started_at TEXT NOT NULL, - ended_at TEXT -); From 7ae76ab55f6191c260d1fc9efa934035b768ab8b Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 18:06:27 -0700 Subject: [PATCH 42/43] Refine CanyonOS Core porting skill --- .../skills/porting-to-canyonos-core/SKILL.md | 601 ++++++------------ .../canyonos-core-contract.md | 405 ------------ .../references/ec2.md | 41 ++ .../references/llm-proxy.md | 64 ++ .../references/packaging.md | 81 +++ .../references/runtime-contract.md | 187 ++++++ .../references/troubleshooting.md | 60 ++ .../skills/porting-to-canyonos-core/traps.md | 86 --- .../porting-to-canyonos-core/validate.py | 41 +- 9 files changed, 628 insertions(+), 938 deletions(-) delete mode 100644 .claude/skills/porting-to-canyonos-core/canyonos-core-contract.md create mode 100644 .claude/skills/porting-to-canyonos-core/references/ec2.md create mode 100644 .claude/skills/porting-to-canyonos-core/references/llm-proxy.md create mode 100644 .claude/skills/porting-to-canyonos-core/references/packaging.md create mode 100644 .claude/skills/porting-to-canyonos-core/references/runtime-contract.md create mode 100644 .claude/skills/porting-to-canyonos-core/references/troubleshooting.md delete mode 100644 .claude/skills/porting-to-canyonos-core/traps.md diff --git a/.claude/skills/porting-to-canyonos-core/SKILL.md b/.claude/skills/porting-to-canyonos-core/SKILL.md index 3102c5d..fe6dd49 100644 --- a/.claude/skills/porting-to-canyonos-core/SKILL.md +++ b/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -1,487 +1,240 @@ --- name: porting-to-canyonos-core -description: Use when porting an existing agent project (LangChain, LangGraph, CrewAI, AutoGen, or a hand-rolled pipeline) onto CanyonOS Core +description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. +compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. --- -# Porting an agent project to CanyonOS Core +# Port an agent project to CanyonOS Core -CanyonOS Core is the product name. Its current compatibility interface remains -unchanged: the executable is `ventis`, the Python package is `ventis`, runtime -environment variables use `VENTIS_*`, and Docker resources use `ventis-*`. -Treat those as protocol identifiers, not branding strings; do not rename them -while porting. +CanyonOS Core is the product name. Its compatibility executable and Python +package remain `ventis`; environment variables and Docker resources retain the +`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding +strings. Do not rename them. -## How to read the rules in this file +## Load references only when needed -Set in capitals, **MUST** and **NEVER** mark a rule whose violation breaks the -port: the build skips an image, `ventis deploy` dies, or the first request -fails. Every one is indexed in [The MUST list](#the-must-list), whose last -column says whether `ventis build`, deploy preflight, or `validate.py` decides -it. `validate.py` intentionally covers only failures a green image build hides. -Nothing else in this file is written in capitals, so -`grep -nE '\b(MUST|NEVER)\b' SKILL.md` returns the rules and only the rules. +- Read [references/packaging.md](references/packaging.md) when a source import + does not resolve from `/app`, the source is nested, or packaging metadata is + involved. +- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target + includes `llm_proxy`. +- Read [references/ec2.md](references/ec2.md) only when any config entry uses + `provider: EC2`. +- Read [references/troubleshooting.md](references/troubleshooting.md) after a + failed build, image probe, deploy, or request. +- Read [references/runtime-contract.md](references/runtime-contract.md) when a + validator finding needs explanation or the runtime mechanism is unclear. -Everything else is stated as fact, in the indicative — how CanyonOS Core behaves, and -what follows from it. There is no "should", and nothing is left to taste that -does not have to be. +## Goal: thin scaffolding beside untouched source -## A port is thin scaffolding beside an untouched source tree - -``` -agents/.yaml one callable surface per CanyonOS Core service +```text +agents/.yaml one callable surface per service agents/.py one thin adapter per service, when needed -workflow/_workflow.py entry point; calls deploy() +workflow/_workflow.py HTTP entry point; calls deploy() config/global_controller.yaml deployment manifest -config/policy.yaml optional — only to restrict access -pyproject.toml conditional — only to expose a nested import root - NOT EDITED — copied whole into every image +config/policy.yaml optional access restriction +pyproject.toml conditional nested-import scaffolding + unchanged ``` -The file count follows the deployment design. A single adapted service normally -adds one yaml/adapter pair, one workflow, and one config. A multi-agent port adds -one yaml/adapter pair for each service that Rule 2 justifies splitting out. If a -source class already satisfies the CanyonOS Core contract, its config can point to that -source file directly and no adapter copy is needed. +The file count follows the deployment. A multi-agent port has one yaml/adapter +pair per service that is worth deploying separately. If a source class already +satisfies the runtime contract, point its config entry at that file and do not +copy it into an adapter. -The project root is the directory from which `ventis build` runs. The source -remains untouched below it. A root `pyproject.toml` is additional conditional -scaffolding when the source is nested, its original imports do not resolve from -`/app`, and the target CanyonOS Core supports an editable install. Metadata inside the -nested source tree does not trigger that install. +Everything the source already owns—prompts, tools, schemas, parsing, retries, +model clients, and node bodies—is imported. The port re-expresses only the +CanyonOS Core boundary and framework-owned orchestration. -The two `agents/` files share one basename, as every example does. The build -generates a stub from the yaml and copies it to `agents/.py` in every -image; in the agent's own image the adapter is copied afterwards and wins the -flat name back, so the two never collide. Pick a basename that is not a module -the adapter imports — an adapter beside a source package called `memory_agent` -is named something else, or it shadows the package it exists to import. +The port root is the existing repository root and the directory from which +`ventis build` runs. Write scaffolding there beside existing directories. If the +repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, +and `config/` beside it. Never move or copy the repository into a new `src/` +directory, and never create an outer wrapper merely for the port. -Everything the source already does — prompts, tools, schemas, parsing, retries, -its LLM client — is reached with an `import`. **A port that contains a prompt -string, a tool body, or a model call that already exists in the source is a -rewrite of the project, not a port of it.** +## 1. Survey before writing -Mechanism and evidence for every claim here: `canyonos-core-contract.md`. -Symptom-to-cause lookup once something breaks: `traps.md`. +Identify: -## Step 1 — Survey the source before writing anything +1. The source entry point and callable input/output. +2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, + `Send`, `Command`, interrupts). +3. Runtime-injected services nodes read: stores, context, memory, sessions, or + callback managers. +4. Sync versus async boundaries. +5. Imports and declared runtime dependencies. +6. Model provider, credential names, streaming use, and optional `llm_proxy`. +7. Whether independent work fans out and benefits from separate replicas. +8. Whether source imports resolve from the project root that becomes `/app`. -CanyonOS Core loads an agent by doing exactly this: +Run the validator once now. Its header detects capabilities directly from the +importable runtime rather than external development metadata: -```python -module = -agent = getattr(module, )() # no arguments -result = getattr(agent, )(**args) # synchronous +```bash +python /validate.py . ``` -Everything below is answerable by reading the source, and expensive to answer -after a green build. - -### What the adapter fixes - -Only the gap between that contract and what the source exposes. Nothing else -belongs in the file. - -| The source exposes | The adapter | -| ----------------------------------------------------- | ---------------------------------------------------------------------- | -| a no-argument class whose methods are synchronous | none — point `entrypoint` at the file it already lives in | -| module-level functions, or `@tool` objects (`StructuredTool` instances, not methods) | a class whose methods call them | -| a compiled graph, a `Crew`, a `GroupChat` | a class, plus the orchestration rewrite of Rule 1 | -| `async def` | a synchronous signature, with `asyncio.run(...)` inside the body | -| framework objects as results (messages, graph state) | the framework's own serializer — `json.dumps` runs on what you return | -| a model client built at import | nothing, once a credential can reach the container | - -Most LangChain and LangGraph projects are rows 2–5, and none of those rows is a -reason to touch the source. - -### What the config declares - -The whole tree is copied into the image at its own relative paths, but the -container starts at `/app`, so only what landed flat imports on its own. - -- **`requirements:`** on the config entry, a list of strings. It covers what the - source imports beyond the runtime's own base list. A malformed value costs the - whole list, not the one bad item: `_normalize_requirements` logs one warning - and returns `[]`, and the build still succeeds with none of them installed. - -- **The import root** — run `validate.py` first and read its - `editable_install` capability. When available, a `pyproject.toml`, `setup.py` - or `setup.cfg` at the **port root** adds `-e .`; metadata inside the nested - source does not. Add a minimal root `pyproject.toml` only when an original - import cannot resolve from `/app`. It names the existing source directory and - package, declares no dependencies, and does not reference a README or license: - - ```toml - [build-system] - requires = ["setuptools>=64"] - build-backend = "setuptools.build_meta" - - [project] - name = "ventis-port" - version = "0.0.0" - dependencies = [] - - [tool.setuptools.packages.find] - where = [""] - include = ["*"] - namespaces = true - ``` - - Set `where` from the actual tree; for a wrapped project with - `source/pyproject.toml` and `source/src/pkg/`, it is `source/src`, not - `source`. Keep dependencies where the source declared them. Because nested - metadata is not installed, repeat its runtime distributions under each - config entry's `requirements:` without editing or deleting the source list. - Without editable-install support, report an import that cannot resolve from - `/app` and stop. A directory rooted directly at `/app` can already resolve as - a Python namespace package even without `__init__.py`; do not add packaging - metadata merely because that file is absent. - -- **`env_file:`** — *needs PR #53, open against main.* A path relative to the - project root pointing at a local `.env`, handed to every container as - `docker run --env-file`. The file never enters the image, and `ventis deploy` - fails on a bad path before launching anything. Without that PR, the only - variables reaching a container are five `VENTIS_*` names, and a config that - sets `env_file:` is setting a key nothing reads — the credential is silently - dropped and the failure surfaces as a provider error on the first request. - -### When the target includes `llm_proxy` - -The proxy is an endpoint redirect, not a provider conversion. Keep the source's -OpenAI, Anthropic, or boto3 client and its request format; put the corresponding -SDK variable in the runtime env file: - -```dotenv -OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 -ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic -AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock -``` +If config or agent yaml is malformed, the validator defers to `ventis build`. +Capability-gated findings say which runtime behavior is available. -Use only the lines for each provider the source actually uses. Its caller-side -credentials are placeholders, for example: +## 2. Choose service boundaries -```dotenv -OPENAI_API_KEY=proxy-placeholder -ANTHROPIC_API_KEY=proxy-placeholder -AWS_ACCESS_KEY_ID=proxy-placeholder -AWS_SECRET_ACCESS_KEY=proxy-placeholder -AWS_REGION=us-east-1 -``` +Start with one service. Split only when it creates independent parallel work or +a distinct resource/replica profile. -OpenAI and Anthropic SDKs still require an API-key variable and boto3 still -requires credentials with which to sign the request, even though the proxy -replaces or reissues those credentials upstream. Launch the proxy in a separate -environment holding the real credentials. Do not put real proxy credentials in -the port's `env_file`, which is given to every agent and workflow container. +- Keep a ReAct loop together; every turn needs shared message history. +- Hoist supervisor task lists and `Send`-style fan-out into the workflow. +- Do not create a one-replica service with no distinct resource profile merely + to mirror every source graph node. -The current proxy is local and non-streaming. Start it on the Docker host with a -non-loopback bind and a port different from the workflow API's usual 8080: +Rewrite framework-owned edges as ordinary Python. Import the connected node +functions unchanged. Construct runtime-injected service objects from source +configuration; do not invent models, dimensions, stores, or defaults silently. +Report any choice the source does not specify. -```bash -PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy -curl http://127.0.0.1:8081/healthz -``` +## 3. Write declarations and adapters -Local CanyonOS Core containers can resolve `host.docker.internal` because their -`docker run` includes `--add-host=host.docker.internal:host-gateway`. An EC2 -container resolves that name to its own EC2 Docker host, not the machine running -`ventis deploy`; a local-only proxy therefore does not support a distributed -port. A reachable proxy address or one proxy per host is deployment work to -report, not an adapter rewrite. - -Survey the source for streaming before choosing this route: OpenAI -`stream=True`, Anthropic stream APIs, Bedrock `invoke_model_with_response_stream`, -Converse, and Converse Stream are outside this implementation. Do not silently -turn streaming off. Report the unsupported call and stop. Exact mechanics and -failure signatures are in `canyonos-core-contract.md` and `traps.md`. - -**And one thing to report rather than fix.** Where the editable install exists, -`-e .` installs `[project.dependencies]` in the same resolve as `requirements:`, -and workshop projects routinely put their whole toolchain there so that one -install sets a laptop up. Compare each declared name against the source's -imports: +### Agent yaml -```bash -grep -rl "import \|from " / -``` - -**Report the mismatch and stop there.** A grep finds names, not requirements: a -package loaded from a string at runtime is imported nowhere and still required. -Hand over the list, the cost (Step 4's protobuf wall, a full image build away), -and the two places entries can move to — `[project.optional-dependencies]`, which -`-e .` skips, and `[dependency-groups]`, which never enters package metadata at -all. Then let the owner decide, including deciding not to. - -## Rule 1 — Rewrite orchestration, import everything else - -One kind of source code genuinely cannot be reused: **control flow owned by a -framework runtime.** CanyonOS Core has no runtime to execute a LangGraph `StateGraph`, a -CrewAI `Crew` or an AutoGen `GroupChat`, so their wiring is re-expressed as -ordinary Python — in the workflow when it fans out, in the adapter when it does -not. The nodes those edges connected are imported, unchanged. - -| Source code | Treatment | -| ---------------------------------------------------------- | ------------------------------ | -| `StateGraph` / `add_edge` / `Send` / `Command(goto=...)` | rewrite as Python control flow | -| `Crew(...)` / `GroupChat(...)` assembly | rewrite as Python control flow | -| node functions, prompts, tools, schemas, parsers, clients | **import** | -| the source's model provider and SDK | **keep** | -| a runtime object the nodes read services off | **construct one** — see below | - -**A framework runtime supplies two things, and only one of them is edges.** It -also injects services the nodes read at call time: LangGraph hands each node a -`Runtime` and the node reads `runtime.store`, `runtime.context`; other -frameworks pass a memory, a callback manager, a session. CanyonOS Core injects none of -it, so the adapter builds the object and passes it in — that is part of -re-expressing the runtime, not a liberty taken with the source. - -**Configure it from what the project already declares, never from taste.** A -LangGraph project states its store in `langgraph.json`; copy those values rather -than choosing your own, because an invented embedding model or dimension is a -silent change to what the project does. Where the project declares nothing, say -in the port report what you chose and why. - -## Rule 2 — Split only to scale - -**Splitting into multiple agents is a scaling decision, not a format -requirement.** A single agent holding the whole pipeline is a valid CanyonOS Core -project. Start there, and hoist a loop into the workflow only when each iteration -fans out to more than one node: - -- a single-agent ReAct loop **stays whole in one agent** — every turn needs the - full message history, and hoisting pushes a growing message list through Redis - each turn. -- a supervisor handing out N tasks, or a `Send` fan-out, is **hoisted** — N - independent runs per request with no shared state is what replicas pay for. - -An agent with `replicas: 1` and no distinct resource profile is a node CanyonOS Core -does nothing for. When you do split, say plainly what it buys. - -## Step 2 — Write the files - -**yaml** — argument `type` is pasted into an AST unchecked, so `str` `int` -`float` `bool` `dict` `list` are the whole vocabulary; the generated stub imports -nothing else, and anything that is not a builtin raises `NameError` when the stub -is imported. Every declared argument is required at every call site — the -generator emits no defaults. `returns` is read by nothing; its value is as a -marker, where `type: dict` tells whoever writes the workflow that this call site -needs `json.loads`. - -**adapter** — the class name is `agent.name` and the constructor takes no -arguments: configuration comes from environment variables read in `__init__`. -What each method has to do is Step 1's table. - -**workflow** — a top-level function named `main`, taking a single `query: str`, -plus `deploy(main, port=...)` at the end. - -Its two imports are fixed, and neither is guessable: - -```python -from deploy import deploy # flat: deploy.py is copied to /app -from agents. import # the stub, under agents/ -``` +Use one yaml per deployed service. Argument types are bare builtins only: +`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is +required by the generated stub. `returns.type` is documentation; use `dict` or +`list` to signal that workflow callers must `json.loads` the returned string. -**The stub is only at `agents/`, and its class carries the agent's own name.** -Two traps sit here, and the build walks you into both: +### Adapter -- `ventis build` prints `Generated stub class 'Stub'`, but the class - it writes is ``. The message is computed separately from the code. - Importing what it names raises `ImportError`. -- The flat form `from import ` is what the examples in - this repository use, and in the workflow image it raises - `ModuleNotFoundError`: the stub is copied to one path, and for the workflow - that path is `agents/.py`. No `__init__.py` is needed — `agents/` - resolves as a namespace package. +The entrypoint exposes a module-level class named exactly `agent.name`. It +constructs with no arguments and its declared methods are synchronous. Read +configuration from the environment in `__init__`. Bridge source coroutines +inside a synchronous method with `asyncio.run(...)`. Serialize framework objects +with their own JSON-safe serializer before returning. -CanyonOS Core itself is permissive here: it serves `POST /` and splats the -request body in as kwargs, so any name and any arguments run. The deployment -platform's test endpoint is not. It posts to a hardcoded `/main`, and its body -schema is `{query: string}` under a strict validator, so a differently named -workflow is unreachable through it and any other key is rejected with 400 in the -control plane, before the request ever reaches the host. Pack richer input into -`query`; every other parameter needs a default, because nothing will ever send -it. +Do not duplicate source prompts, tools, schemas, or model calls. Keep the source +provider and SDK. -The file is `exec`'d rather than imported, so `__name__ == "__main__"` is true -and `if __name__ == "__main__":` blocks fire in production. `deploy()` blocks. +### Workflow -Dispatch every call before resolving any of them: +Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. +Import generated stubs by yaml basename and agent class name: ```python -futures = [agent.work(item=i) for i in items] # returns immediately -results = [json.loads(f.value()) for f in futures] # .value() blocks +from deploy import deploy +from agents. import ``` -Fused into one comprehension the calls run one after another. It does not error; -it is just silently serial, and the fan-out is gone. - -**config** — each entry's `name` matches a yaml's `agent.name`, or the build -warns, skips that image, and still exits 0. Write `provider: local` in -**lowercase**: the port reservation compares `provider == "local"` with no -normalization, so `Local` leaves the port unreserved and deploy dies. `replicas` -is an integer — the list form that `_get_replica_placements` accepts raises -`TypeError` in `InstanceManager`. - -**policy** — optional. Absent, every service is allowed. Present, it is read -strictly: an empty file, or a null `rules:`, is an `AttributeError` inside -`GlobalController.__init__` that kills `ventis deploy` before a container starts. -Write one only to restrict, and then remember that the first matching rule -decides — a service missing from the rule that matched is not a startup error but -an `Unauthorized` response after the request was accepted. - -## The MUST list - -Every hard rule in this file, and the check that decides it. `--` marks the ones -only a human can judge; they are the reason a clean validator run is a floor and -not a ceiling. - -| # | The rule | Check | -| --- | ------------------------------------------------------------------------- | ---------- | -| M1 | The entrypoint MUST define a class named exactly `agent.name` | V006 | -| M2 | That class MUST construct with no arguments | V007 | -| M3 | A yaml `arguments[].name` MUST equal the Python parameter name exactly | V008 | -| M4 | A yaml `type` MUST be a bare builtin | V010 | -| M5 | A method backing a yaml function MUST be synchronous | V009 | -| M6 | Every config entry `name` MUST match some yaml `agent.name` | build | -| M7 | Two config entry `name`s MUST differ by more than case | build output | -| M8 | `provider` MUST be lowercase `local` (EC2 takes any casing) | deploy preflight | -| M9 | `replicas` MUST be an integer | deploy preflight | -| M10 | `requirements:` MUST be a list of strings | build | -| M11 | The workflow MUST expose `main(query)`; other parameters MUST have defaults | build + V016 | -| M12 | The workflow MUST NEVER carry an `if __name__ == "__main__":` block | V017 | -| M13 | A fan-out MUST dispatch every call before resolving any | V018 | -| M14 | No project module MUST take the flat name of a runtime file or a stub | V019, V020 | -| M14b | The workflow MUST import a stub as `from agents. import ` | V023 | -| M15 | `policy.yaml` MUST be absent, or MUST carry a non-empty `rules:` list | deploy preflight | -| M16 | An EC2 entry MUST declare `instance_type`, and `ec2:` MUST be complete | deploy preflight | -| M17 | NEVER copy a prompt, tool, or schema that exists in the source | review | -| M18 | NEVER hardcode a credential, or ship one in the build context | W003 | -| M19 | NEVER edit the source tree, and NEVER vendor it into `agents/` | `git status` | -| M20 | NEVER swap the LLM provider the source uses; an LLM proxy only redirects its endpoint | -- | -| M21 | NEVER move or drop a declared dependency — report it and stop | -- | -| M22 | Framework control flow MUST be rewritten; everything else MUST be imported | -- | - -Build owns YAML parsing, required paths, stub generation, and Dockerfile/package -installation errors. Deploy preflight owns provider, replica, policy, and EC2 -shape. `validate.py` does not repeat those checks; it focuses on adapter loading, -stub imports, workflow execution, copy collisions, credentials, import roots, -and dependencies that fail only inside a built container. - -Two more rules apply only where the CanyonOS Core you are targeting supports them, -which `validate.py` probes for rather than assumes: - -| # | The rule | Needs | Check | -| --- | ----------------------------------------------------------------- | ----------------------- | ----- | -| M23 | `env_file:` MUST resolve to a readable file, and MUST be the only way a credential enters | PR #53 | deploy preflight; support V030 | -| M24 | A source import that does not resolve from `/app` MUST have usable packaging metadata at the port root | editable install | V031 | - -## Step 3 — Preflight hidden runtime failures +The deployment platform sends `{query: string}` to `/main`. Pack richer input +inside `query`; any additional workflow parameter has a default. -```bash -python /validate.py . +Dispatch every remote call before resolving any future: + +```python +futures = [agent.work(item=item) for item in items] +results = [json.loads(future.value()) for future in futures] ``` -Run it before building. It does not duplicate errors `ventis build` or deploy -preflight already reports. Instead it catches what those stages do not execute: -the adapter class contract, generated-stub import path, workflow behavior, flat -copy collisions, container credentials, package import roots, and undeclared -runtime imports. +Do not fuse dispatch and `.value()` in one comprehension; that silently +serializes fan-out. Do not add an `if __name__ == "__main__":` block: the +workflow is executed with `__name__ == "__main__"` in production. -It parses Python but never imports the port, so it is safe on a tree whose -dependencies are not installed. Errors are deterministic runtime contract -violations and exit 1. Heuristic warnings exit 0 unless `--strict` is used. -`--json` emits the findings as data. A malformed config or agent yaml is reported -by `ventis build`; when it prevents runtime inspection, the validator emits only -a `BUILD` informational finding and stops. +### Config -The header prints which capability-gated rules are in force. A rule whose feature -is missing is reported `UNAVAILABLE`, never silently skipped. +For each service, keep these names aligned: -## Step 4 — Build, then probe the image twice +```text +config entry name == yaml agent.name == entrypoint class name +``` -`ventis build` prints `Build complete.` and tags every image for a project whose -container dies on startup. So run the image — tagged -`ventis-` — and do what the container does. **Both probes, -in this order. Neither covers the other.** +Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a +list of distribution-name strings. Put `env_file` at config top level when the +runtime capability is available. Omit `policy.yaml` unless access must be +restricted; if present, give it a non-empty `rules` list. + +## Hard rules + +Capitalized **MUST** and **NEVER** are reserved for port-breaking or +source-integrity rules. The owner column states where each is decided. + +| ID | Rule | Owner | +|---|---|---| +| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | The class MUST construct with no arguments | V007 | +| M3 | yaml argument names MUST match Python parameter names | V008 | +| M4 | yaml argument types MUST be bare builtins | V010 | +| M5 | Declared adapter methods MUST be synchronous | V009 | +| M6 | Config names MUST match yaml agent names | build | +| M7 | Config names MUST not collide after lowercase normalization | build output | +| M8 | Local provider MUST be lowercase `local` | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements` MUST be a list of strings | build | +| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | +| M12 | Workflow MUST NEVER contain a main guard | V017 | +| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | +| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | +| M15 | Workflow MUST import stubs from `agents.` | V023 | +| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | +| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | +| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | +| M19 | NEVER hardcode or bake a real credential into an image | W003 | +| M20 | NEVER edit or vendor the source tree | `git status` | +| M21 | NEVER swap the source LLM provider | review | +| M22 | NEVER silently move, drop, or reclassify source dependencies | review | +| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | +| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | + +## 4. Validate, build, and probe + +Run static preflight, then let the build own build-time validation: ```bash -# 1. The runtime itself. This is what CMD runs, and it fails before your agent -# is ever reached, so probing the entrypoint alone will miss it. -docker run --rm ventis- python -c "import local_controller" - -# 2. The agent, loaded the way _load_agent loads it. --env-file because the -# constructor reads the environment, and the deployment gives it one. -docker run --rm --env-file ventis- python -c " -import importlib.util, sys -spec = importlib.util.spec_from_file_location('m', '.py') -m = importlib.util.module_from_spec(spec); sys.modules['m'] = m -spec.loader.exec_module(m); m.(); print('ok')" +python /validate.py . +ventis build -c config/global_controller.yaml ``` -Probe 1 exists because the gRPC stack is unpinned: `ventis build` runs -`grpc_tools.protoc` on the **host** and copies the generated `_pb2.py` in, where -a resolver that knows nothing about them picks the protobuf runtime. Protobuf -refuses gencode newer than its runtime, so a source whose dependencies hold -protobuf back kills the container on `import local_controller`. An image with few -requirements passes by coincidence. Report it — the fix belongs in -`generate_docker`, not in the port — and if Step 1 flagged declared-but-unimported -dependencies, name the culprit here. +A green build never imports the adapter. Probe each agent image in this order: -Probe 2 exists because `_load_agent` catches every exception, logs it and returns -`None`: a missing dependency, a wrong class name, a constructor that wants -arguments, or a broken import inside the source tree are all invisible until the -first request answers `"No agent loaded"`. +```bash +# Runtime startup path + docker run --rm ventis- \ + python -c "import local_controller" + +# Agent load path; include --env-file when configured + docker run --rm --env-file ventis- \ + python -c "import importlib.util,sys; \ +s=importlib.util.spec_from_file_location('m','.py'); \ +m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ +m.();print('ok')" +``` -It takes `--env-file` because `__init__` reads the environment and a container -started by `ventis deploy` gets one. Without it a correct port fails its own -probe on a missing credential — an adapter that builds an embeddings client in -its constructor raises `OpenAIError: Missing credentials` and passes unchanged -the moment the file is passed. +Also probe the workflow image with `python -c "import local_controller"`; it has +its own dependency resolve and generated-stub imports. -Then `ventis deploy`, which needs Docker and an importable `grpc_stubs/` **on -this host** (it aborts if they were cleaned after the build). It starts its own -Redis container — do not run one. +Then deploy, send a representative request, and poll its status: + +```bash +ventis deploy -c config/global_controller.yaml +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query":""}' +curl http://localhost:8080/status/ +``` -## Step 5 — Clean up every build and deployment product +A successful outer request with a source-level failure still proves the port +reached and returned the source behavior. Record the distinction. -Do this after recording the probe and request results, including on failure -paths. First stop the foreground `ventis deploy` with Ctrl+C and wait for -`GlobalController cleanup` to remove its agent, workflow, and Redis containers. -If deploy crashed before its cleanup handler ran, remove the exact container -names created by this deployment; do not delete another project's containers. +## 5. Clean up -Then remove generated files and the exact images built from the config: +After collecting evidence, stop foreground deploy with Ctrl+C and wait for +controller cleanup. Remove exact leftovers if startup crashed. Then remove build +products and exact images from this config: ```bash -ventis clean # removes stubs/, grpc_stubs/, and docker_container/ - -docker image rm \ - ventis- \ +ventis clean +docker image rm ventis- \ ventis- -``` - -Repeat the image argument for every config entry. `ventis clean` does not remove -containers or images. Confirm that the project root no longer contains the three -generated directories and that no container from this deployment remains: -```bash test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container docker ps -a --format '{{.Names}}' ``` -Keep all agent declarations and adapters, the workflow, config, conditional -root `pyproject.toml`, untouched source tree, and any requested logs or port -report. Those are source and evidence, not build products. - -## Never do these - -Each turns a port into a rewrite. They are not judgment calls, and the middle -column is the thought that gets you there. - -| Move | The rationalization | Why it is wrong | -| ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------- | -| Copy a prompt, tool, or schema into the adapter | "so the adapter stands alone" | It exists in the source. Import it — the whole tree is in the image, and a copy drifts. | -| Swap the LLM provider | "the image already has one, and the user wants something that runs" | A port that silently changed models does not run *their* project. `requirements:` installs the source's own provider; `llm_proxy` redirects that SDK rather than converting its request. | -| Hardcode a key, or ship it in a file you add | "there is no other way in" | The build sweeps the project into every image. Where `env_file:` exists it is the way in; with `llm_proxy`, it contains routing plus dummy caller credentials while the proxy receives real credentials separately. | -| Drop or move a dependency | "this one is obviously dev-only" | Obvious to you, not yours to decide. Declare it under `requirements:`; report the rest and let the owner classify. | -| Edit files in the source tree, or vendor it into `agents/` | "just this one line" | The port leaves `git status` on the source clean, and vendoring is copying. | +`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it +does not remove containers or images. Keep port scaffolding, untouched source, +and requested logs or reports. diff --git a/.claude/skills/porting-to-canyonos-core/canyonos-core-contract.md b/.claude/skills/porting-to-canyonos-core/canyonos-core-contract.md deleted file mode 100644 index 18b9d44..0000000 --- a/.claude/skills/porting-to-canyonos-core/canyonos-core-contract.md +++ /dev/null @@ -1,405 +0,0 @@ -# The CanyonOS Core contract - -The product is CanyonOS Core; its compatibility CLI, Python package, environment -prefix, and Docker resource prefix remain `ventis`, `ventis`, `VENTIS_*`, and -`ventis-*` respectively. - -Mechanism behind every rule in `SKILL.md`. Validate against -[CanyonCodeCoreAI/canyoncodecore](https://github.com/CanyonCodeCoreAI/canyoncodecore). - -**Which CanyonOS Core this describes.** Two sections below hold for a branch rather than -for `main`, and each says so where it starts. `validate.py` probes the importable -`ventis` package for them instead of assuming: - -| Behaviour | Carried by | -| ---------------------------------------------- | ------------------------------------------------------------------- | -| a stub lands at **two** paths | **PR #51** (`feature/all-the-files`), open against main | -| `env_file:` carries credentials to a container | **PR #53** (`jiajunh/can-232-...`), open against main | -| `-e .`, and a sweep that takes non-`.py` files | **no PR** — only on `jiajunh/can-228-create-a-skill-...` | - -Everything not marked holds on `main` today. - -## Project layout - -| Path | Where it comes from | -| ------------------------------------------- | ---------------------------------------------------------------------------- | -| `agents/*.yaml` | `cli.py` — `glob(agents_dir/*.yaml)` | -| `stubs/`, `grpc_stubs/` | `cli.py` — generated by `ventis build` | -| `config/global_controller.yaml` | `cli.py` — `DEFAULT_CONFIG_PATH`, overridable with `--config` | -| `config/policy.yaml` | `global_controller.py` `_load_policy_rules` — optional | -| the workflow file | the `workflow_file` key on the `type: workflow` config entry | -| the project root | `cli.py` passes `project_dir=os.getcwd()`; build and deploy run from it | -| root `pyproject.toml` / `setup.py` / `setup.cfg` | `_install_step` — its presence is what adds `-e .`; nested metadata is ignored. **No PR carries this** | - -## Agent yaml - -```yaml -agent: - name: # required - functions: # optional; absent -> stub class with only __init__ - - name: # required - description: # optional -> becomes the stub method's docstring - arguments: # optional; absent -> no-arg method - - name: # required - type: # optional -> pasted verbatim as an annotation - returns: - type: # read by nothing -``` - -Nothing else is read. Extra keys are ignored silently. - -- **`type` is pasted, never checked.** `_build_stub_method` does - `ast.Name(id=arg["type"])`, and the generated stub imports only `Future` and - `inspect`. Anything that is not a builtin raises `NameError` when the stub is - imported. Use `str` `int` `float` `bool` `dict` `list` — not `List[str]`, not - `Optional[int]`, not a class name. -- **No default values.** `ast.arguments(..., defaults=[])`. Every declared - argument is required at every call site. Optional configuration belongs in the - agent's `__init__`, read from the environment. -- **Parameter names must match exactly.** The controller invokes `method(**args)`. - Order is irrelevant; spelling is not, or the call raises `TypeError` at request - time. -- **`returns` is documentation.** The stub generator never reads it. Its value is - as a marker: `type: dict` tells whoever writes the workflow that this call site - needs `json.loads`. -- **The filename names the stub, not the agent.** `agents/x.yaml` generates - `stubs/x.py`, which the build copies to `/app/x.py` and `/app/agents/x.py`. - Sharing the entrypoint's basename is therefore fine and is the convention: the - entrypoint is copied last, so it wins `/app/x.py` while the stub keeps - `/app/agents/x.py`. What the basename must **not** match is a source module the - adapter imports — `joke_writer.yaml` beside a `joke_writer.py` puts a stub on - top of the source. - -## The three-way name binding - -``` -config entry `name` == agents/x.yaml `agent.name` == the class inside the .py - | - `entrypoint` on that config entry points at the .py -``` - -`cmd_build` looks up each config entry's `name` among the parsed yamls. No match -means a logged warning and **no image built for that agent** — the build still -exits 0. - -## Agent class - -| Requirement | Enforced by | -| ----------------------------------------------- | -------------------------------------------------------------------------------------------- | -| Class name equals `agent.name` | `generate_docker` writes `ENV VENTIS_AGENT_NAME`; `_load_agent` does `getattr(module, name)` | -| Instantiable with no arguments | `_load_agent` calls `agent_class()` | -| Methods are synchronous | the executor calls `method(**args)` — there is no `await` anywhere on this path | -| Return values survive `json.dumps` / `str()` | `_execute_locally` does `json.dumps(result)` for `dict`/`list`, `str(result)` otherwise | - -`self.tools = [...]` appears throughout `examples/` and is read by **nothing** in -`ventis/`. It is decoration. - -**`.value()` always returns a string.** The result is written into Redis as text -and handed back verbatim; there is no deserialization on the way out. - -## Workflow - -The workflow file is **not imported — it is `exec`'d**. -`generate_workflow_docker` writes a `workflow_launcher.py` whose last line is -`exec(open(".py").read())`, and the Dockerfile's CMD runs that launcher. - -- `__name__ == "__main__"` inside your workflow file, so - `if __name__ == "__main__":` blocks **execute in production**. -- `__file__` points at `workflow_launcher.py`. The `sys.path.insert(..., "..", - "stubs")` lines the examples carry resolve to nonexistent paths; imports work - anyway because the stubs and the runtime are placed flat at `/app`, which is - `sys.path[0]`. What makes the *project* tree importable is the editable - install, not `sys.path[0]`. -- `deploy()` ends in `app.run()` and blocks. Nothing after it runs. -- Module-level code runs **once** at container start; the workflow function runs - **per request**, on a Flask worker thread. -- The REST route is `fn.__name__` — rename the function and the endpoint renames - with it. There is no fixed `/main` **in CanyonOS Core**. -- The request body is splatted in as kwargs after `_context` is popped off. Any - shape of body works. - -Both of those are why the platform constraint has to be written down rather than -discovered: the control plane's test endpoint posts to a hardcoded `/main` with a -strictly validated `{query: string}` body, so a port must expose `main(query)` to -be reachable through it. Nothing in this repo enforces that or fails without it — -the constraint lives in the control plane (`deploy.routes.ts`, `deploy.agent.ts`, -`deploy.types.ts`), and the transport layer there is generic -(`Record`) while the route schema is not. - -The workflow container also runs its own `LocalController` on 50051 in a -background thread. That is what dispatches the Futures the workflow creates. - -## The build context - -`generate_docker` takes a `project_dir` and `cmd_build` passes it, so the whole -project reaches the image with its relative paths intact — structure is preserved -rather than flattened because packages need it (`src/tools/__init__.py` and -`src/tools/default/__init__.py` flatten to the same name). - -Copy order decides every collision: the swept tree first, then the shared -runtime, then every stub, then the entrypoint. Later writes land on earlier ones. - -| What | Lands where | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| the project tree | at its own relative paths (`agents/x.py`, `src/pkg/mod.py`) | -| the shared runtime | flat at the context root, winning over the swept tree — `local_controller.py` is the CMD, so a project file of that name breaks the container | -| every stub (PR #51) | **twice**: flat at the root (the copy imports resolve), and at `agents/.py`, landing on the real implementation so a peer's name gives the caller its stub | -| the entrypoint | flat, last, winning the flat name back — `VENTIS_AGENT_FILE` is a **basename**, loaded from `/app` | -| `requirements.txt` | written before anything is copied, so the sweep skips a project's own `requirements.txt` and root `Dockerfile` | - -`_sweep_py_files` takes **`.py` files only**, skipping symlinks, hidden files -and directories (`.env` holds credentials and the context is what ships), -`__pycache__`, and the three directories `ventis build` generates at the project -root: `docker_container`, `stubs`, `grpc_stubs`. - -That matters for a source whose packaging metadata points at a README or a -license: those files do not reach the image, so an editable install of it would -fail on the missing file. `_sweep_project_files`, which takes every file, exists -only on `jiajunh/can-228-create-a-skill-...` and has no PR — the same branch that -carries `_install_step`, which is the only reason the wider sweep is needed. - -**An agent is no longer one file**, and a yaml sharing the entrypoint's basename -no longer eats its own stub. What an agent loses is the ability to import *its -own* stub by name — the entrypoint shadows it flat. It can still reach it at -`agents/.py`, and nothing in `examples/` wants to. - -### The import root - -> **No PR carries this.** `_install_step` lives only on -> `jiajunh/can-228-create-a-skill-...`. On `main`, and on both open PRs, the -> agent Dockerfile is `COPY requirements.txt` → `uv pip install -r -> requirements.txt` → `COPY . .`, with no `-e .` and no packaging detection. -> Python then resolves only names rooted at `/app`: flat modules, regular -> packages, and namespace-package directories. A package below another source -> directory is not a top-level import merely because it was copied. -> `validate.py` V031 enforces whichever rule is in force. - -`_install_step` writes -`RUN uv pip install --system -r requirements.txt -e .` when the **project root** -has a `pyproject.toml`, `setup.py` or `setup.cfg`. It does not search below that -root. This distinction is observable in the test harness: a repository kept -untouched under `source/` can have `source/pyproject.toml`, but `_install_step` -still skips it. - -When the nested source's original imports need another directory on `sys.path`, -the port supplies a minimal root `pyproject.toml` that points setuptools at the -existing package directory. It declares no dependencies and references no -README or license. That file is port scaffolding, not a source edit. For example, -if `source/src/pkg/` backs `import pkg`, package discovery uses -`where = ["source/src"]`, `include = ["pkg*"]`, and `namespaces = true` when the -package omits `__init__.py`. - -Without root packaging metadata the editable install is skipped silently. Some -nested-looking imports still work: `/app/src/agents/kyc_agent.py` is importable -as `src.agents.kyc_agent` through namespace packages even when neither directory -has `__init__.py`. It is not importable as `agents.kyc_agent`; that spelling -would require `/app/src` as an import root. Packaging is conditional on the -actual import spelling, not on whether `__init__.py` exists. - -**One resolve, not two.** Where `_install_step` exists, requirements and `-e .` -go to a single `uv pip install` so the runtime's list and the source's own -dependencies resolve against each other; a genuine conflict fails the build -instead of the first request. It also forces `COPY . .` ahead of the install, so -the requirements layer no longer caches on its own. Without that branch the two -are separate layers, requirements first, and the source's own dependency list is -never installed at all. - -## Dependencies - -`generate_docker` and `generate_workflow_docker` both take a `requirements` -argument, and `cmd_build` passes `_normalize_requirements(agent_cfg)`. The -runtime's own list is unconditional and not declarable: - -``` -agent: grpcio grpcio-tools redis pyyaml psutil ipdb ipython boto3 -workflow: the same, plus flask sqlalchemy psycopg[binary] -``` - -The declared list is appended verbatim. `_normalize_requirements` takes only a -list of strings — a bare string, a mapping, or a list with a non-string in it -each logs one warning and becomes `[]`, so a malformed entry costs the whole list -rather than the one item. Nothing is deduplicated against the base either. - -**Only a `pyproject.toml` at the port root is installed in the same resolve.** -If that is the source's own metadata, `requirements:` covers only imports it -does not declare, and its whole dependency list comes along. If the source is -nested and the root metadata is minimal port scaffolding, the nested dependency -list is not installed; repeat its runtime distributions under the relevant -config entries' `requirements:` while leaving the source declaration untouched. -This duplication is required by the current root-only packaging probe, not a -license to reclassify or drop dependencies. - -### The gRPC stack is unpinned - -`cmd_build` runs `grpc_tools.protoc` on the **host** and copies the resulting -`_pb2.py` into the image, where a resolver that knows nothing about them picks -the protobuf runtime. Protobuf refuses to load gencode newer than its runtime, so -a source whose own dependencies drag protobuf down produces a container that dies -on `import local_controller` — before the agent is reached, with a green build -behind it: - -``` -google.protobuf.runtime_version.VersionError: Detected incompatible Protobuf -Gencode/Runtime versions ... gencode 7.35.1 runtime 6.33.6. -``` - -An image with few requirements resolves to the newest wheel, which happens to be -at least as new as the host's, and passes by coincidence. A fix means prepending -`grpcio==`, `grpcio-tools==` and `protobuf>=` at the host's own versions -(`importlib.metadata.version`) to the generated requirements — `>=` on protobuf -because the guarantee runs one way: a runtime at or above the gencode. - -**Check this first on any port that installs a large dependency tree.** Probing -the entrypoint module is not enough — it does not import `local_controller`, -which is what the container's CMD actually runs. - -## Credentials: `env_file` - -> **PR #53** (`jiajunh/can-232-...`) carries this, open against main. Without it -> nothing reads the key: `grep -rn env_file ventis/` finds no hits on `main`, so -> an `env_file:` line in the config is inert, the credential never reaches the -> container, and the failure surfaces as a provider error on the first request -> rather than as a config error at deploy. `validate.py` V030 probes for -> `resolve_env_file` and reports which of the two situations you are in. - -`_launch_locally` passes exactly five `-e` flags, all `VENTIS_*` -(`AGENT_PORT`, `AGENT_HOST`, `REDIS_HOST`, `REDIS_PORT`, `POLL_INTERVAL`), plus -`VENTIS_DATABASE_URL` and `VENTIS_PROJECT_ID` on a workflow entry when -configured. User secrets travel a separate road. - -`env_file:` in `config/global_controller.yaml` names a local `.env`. -`resolve_env_file` expands `~`, resolves a relative path against the project -root, and raises if the file is missing, is not a file, or is unreadable — -`cmd_deploy` calls it before `GlobalController` exists, so a bad path is one -error line rather than a fleet of agents with no keys. - -`env_file_args` then hands the file to `docker run` as `--env-file`. A container -on this machine reads the original; a container on a remote host gets a 0600 copy -under `/tmp`, deleted as soon as `docker run` returns. The explicit `VENTIS_*` -flags are appended first and still win, so a stray `VENTIS_*` line in someone's -`.env` cannot break agent wiring. - -Consequences for a port: - -- A source that constructs its model client at module scope **loads fine**. The - key is in the environment before the adapter imports the source. -- The file never enters the image — the sweep skips hidden files, and the - variables reach the container at run time. -- A missing key is no longer `"No agent loaded"`; it is a provider error on - `/status` after the request was accepted. -- `load_dotenv(".env")` in the source still does nothing: the file is not in the - image and `load_dotenv` is silent about a missing one. - -## The optional `llm_proxy` endpoint contract - -The `llm_proxy` implementation is a separate Flask process, not an agent and not -part of `ventis deploy`. It preserves each caller's SDK protocol under a provider -prefix and funnels all completed calls through `llm_proxy.core.proxy_request`: - -| Source client | Container variable | Proxy path | Upstream behavior | -| --- | --- | --- | --- | -| OpenAI SDK | `OPENAI_BASE_URL=http://host.docker.internal:/openai/v1` | `/openai/...` | HTTP request forwarded; caller authorization removed and proxy key inserted | -| Anthropic SDK | `ANTHROPIC_BASE_URL=http://host.docker.internal:/anthropic` | `/anthropic/...` | HTTP request forwarded; caller key removed and proxy key inserted | -| boto3 Bedrock Runtime | `AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:/bedrock` | `/bedrock/model//invoke` | body reissued through the proxy's boto3 client and its AWS identity | - -This is why using the proxy does not relax the no-provider-swap rule: model IDs, -request bodies, response bodies, and the source SDK remain provider-specific. -The only port artifact is endpoint configuration in the runtime environment. -OpenAI and Anthropic client constructors still validate that their normal key -variables exist. Botocore still signs the request it sends to its custom -endpoint. Dummy caller credentials satisfy those clients; the proxy process gets -real `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or AWS credentials from its own -process environment. Putting the real values in the project's `env_file` gives -them to every CanyonOS Core container and defeats the credential boundary. - -The proxy defaults to `127.0.0.1:8080`. Both defaults are wrong beside a local -CanyonOS Core deployment: a container cannot reach the host's loopback, and the -workflow normally publishes host port 8080. Bind the proxy to `0.0.0.0` on a -different port. Local and EC2 CanyonOS Core `docker run` commands add -`host.docker.internal:host-gateway`; that name means the Docker host of each -container. Consequently a proxy on the controller's machine serves local -containers, while remote EC2 containers require a reachable network address or -a proxy running on each EC2 host. - -`GET /healthz` proves only that Flask is listening and lists the registered -providers. It does not validate any upstream credential. OpenAI and Anthropic -upstream 4xx/5xx responses pass through with their status and body. Exceptions -in routing or provider code become a JSON `502` with `error: proxy_error`. -Bedrock `ClientError` responses are reconstructed as JSON with the upstream -status; they are not byte-for-byte passthrough. Although `Config` reads -`BEDROCK_UPSTREAM_HOST`, `BedrockProvider` does not pass it as `endpoint_url` -when constructing boto3, so that variable has no effect in this implementation. - -The implementation buffers the full request and response. It has no OpenAI or -Anthropic streaming path, and Bedrock accepts only the final `invoke` operation; -`invoke-with-response-stream`, `converse`, and `converse-stream` raise -`NotImplementedError` and surface as 502. A port cannot preserve a source that -uses those calls through this proxy today. - -## Cleanup boundaries - -`ventis deploy` registers `GlobalController.cleanup` for Ctrl+C, SIGTERM, and -normal process exit. That cleanup terminates the controller's recorded agent and -workflow instances and its Redis containers. A hard kill or an exception before -a runtime is recorded can leave Docker containers behind, so cleanup must also -be verified from Docker state. - -`ventis clean` is narrower: `cmd_clean` removes only the project-root `stubs/`, -`grpc_stubs/`, and `docker_container/` directories. It removes neither running -containers nor the `ventis-` images produced by the -build. Image removal therefore happens explicitly after containers stop. The -agent declarations and adapters, workflows, config, conditional root packaging -metadata, untouched source, and recorded evidence are not generated build -products and remain. - -## `config/policy.yaml` is optional - -`_load_policy_rules` logs `No policy file found ..., skipping policy setup` and -returns `[]`, which `_load_and_write_policies` publishes to every host Redis. -`LocalController._check_policy` returns `True` when the rule list is empty, so -**no policy file means everything is allowed.** - -**Absent is safe; present-and-empty is not.** Past the `os.path.isfile` guard the -read is unguarded: `policy_config.get("rules", [])` on an empty file's `None` is -an `AttributeError`, and a null `rules:` reaches `rules.sort()` as `None`. Either -one raises inside `GlobalController.__init__`, so `ventis deploy` dies before a -single container starts. Deleting the file is the safe state; a half-written one -is not. - -The path is derived from the **config file's own directory**, not from the -project root — `-c foo/gc.yaml` looks for `foo/policy.yaml`. - -When rules exist they are sorted most-specific-first (by number of `match` keys) -and the first rule whose `match` keys all equal the request context decides: -`access: all`, or membership in the `access` list. A service left out of the -matching rule answers `Unauthorized: Policy denied access to service 'X'` in the -`/status` response — after the request was accepted. If no rule matches at all, -access is denied. - -## `provider` is case-sensitive in one direction only - -`InstanceManager.ensure_instances` tests `provider == "local"` to decide whether -to reserve a host port. `Local` fails that test, `reserved_port` stays `None`, -and `Local/_runtime.py`'s -`int(spec.get("host_port", spec.get("port", next_host_port(host))))` raises -`int() argument must be a string, a bytes-like object or a real number, not -'NoneType'`. The EC2 test on the same value is `.upper() == "EC2"` everywhere, so -it accepts any casing. Every example that works writes lowercase `local`. - -## Failures are silent - -`_load_agent` catches every exception, logs it, and returns `None`. - -| Stage | A missing credential / dependency / wrong class name | -| --------------- | ------------------------------------------------------ | -| `ventis build` | passes — it never imports your agent | -| `ventis deploy` | passes — the container starts, gRPC listens | -| first request | `"No agent loaded"` | - -The real cause exists only in that container's stdout. - -Worse, the node still advertises itself as usable. `LocalController.__init__` -writes `healthy` to `controller:::status` **before** calling -`_load_agent`, and `_metrics_loop` re-writes `healthy` on every tick. Nothing -downgrades the status when the agent fails to load, so a replica that can serve -nothing keeps being routed to. diff --git a/.claude/skills/porting-to-canyonos-core/references/ec2.md b/.claude/skills/porting-to-canyonos-core/references/ec2.md new file mode 100644 index 0000000..e06daa0 --- /dev/null +++ b/.claude/skills/porting-to-canyonos-core/references/ec2.md @@ -0,0 +1,41 @@ +# EC2 deployment + +Read this only when at least one config entry uses `provider: EC2`. + +## Configuration + +Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The +top-level `ec2` block supplies the runtime's required infrastructure and SSH +settings. Read the target checkout's deploy preflight and EC2 runtime before +writing the block; do not copy values from an example environment. + +Typical required categories are: + +- AMI and instance type +- region and subnet +- security groups +- SSH user and credentials accepted by the runtime + +`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof +that provisioning, SSH, image transfer, or remote container startup works. + +## Networking + +A remote container's `host.docker.internal` names its own EC2 Docker host. It +does not name the local controller machine. Databases, model proxies, and other +services must use addresses reachable from every selected host. + +The environment file may be copied temporarily to a remote host by runtimes that +expose the `env_file` capability. Confirm behavior from the capability probe and +target runtime rather than assuming local Docker semantics. + +## Probes and cleanup + +Run the same runtime and adapter probes against the exact image before remote +deployment. After deploy, verify the remote container logs; controller health +can be green even when agent loading failed. + +Stop foreground deploy normally so the controller can terminate recorded EC2 +instances. If provisioning or startup fails before an instance is recorded, +inspect the cloud provider directly and remove exact leaked resources. Never use +a broad cleanup command against unrelated instances. diff --git a/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md new file mode 100644 index 0000000..f726bd7 --- /dev/null +++ b/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md @@ -0,0 +1,64 @@ +# LLM proxy integration + +Read this only when the target checkout contains `llm_proxy` or the deployment +explicitly routes model SDKs through it. + +## Preserve provider protocols + +The proxy redirects provider endpoints; it does not convert providers. Keep the +source SDK, model ID, request body, and response parsing unchanged. + +Configure only the provider variables the source uses: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Some SDKs refuse to initialize without caller credentials. Give agent containers +non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS +credentials in the separate proxy process, not in the port's `env_file`. + +## Start locally + +The proxy defaults conflict with a typical deployment: host loopback is not +reachable from a container, and port 8080 is normally used by the workflow API. +Use a non-loopback bind and a different port: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers resolve `host.docker.internal` through their +Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the +machine running `ventis deploy`. Distributed deployments need a reachable proxy +address or one proxy on each host. + +## Supported call shape + +The implementation buffers complete requests and responses: + +- OpenAI and Anthropic non-streaming HTTP calls are forwarded. +- Bedrock `invoke` is reissued through the proxy's boto3 client. +- OpenAI/Anthropic streaming is unsupported. +- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are + unsupported. + +Survey the source before selecting the proxy. Do not silently disable streaming; +report the unsupported behavior and stop. + +## Credential behavior + +- The OpenAI adapter removes caller authorization and inserts the proxy key. +- The Anthropic adapter removes caller key headers and inserts the proxy key. +- Botocore still signs requests sent to a custom endpoint, so a caller may need + placeholder AWS credentials even though the proxy reissues upstream with its + own identity. +- `/healthz` proves provider registration and Flask availability, not upstream + credential validity. + +OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return +JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed +with the upstream status and are not byte-for-byte passthrough. diff --git a/.claude/skills/porting-to-canyonos-core/references/packaging.md b/.claude/skills/porting-to-canyonos-core/references/packaging.md new file mode 100644 index 0000000..8520c4a --- /dev/null +++ b/.claude/skills/porting-to-canyonos-core/references/packaging.md @@ -0,0 +1,81 @@ +# Packaging and import roots + +Read this reference when an adapter imports nested source code, the source uses a +`src/` layout, or V031 reports an import-root problem. + +## What `/app` can import + +CanyonOS Core preserves project-relative paths in the image and starts Python at +`/app`. Without an editable install, Python resolves names rooted there: + +- `/app/tools.py` as `import tools` +- `/app/pkg/__init__.py` as `import pkg` +- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace + directories without `__init__.py` + +It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become +an import root first. + +## Detect support, do not infer it from release history + +Run: + +```bash +python /validate.py . +``` + +Read the `editable_install` capability. If it is unavailable and the original +import cannot resolve from `/app`, report a runtime capability blocker and stop. +Do not add a `sys.path` hack or relocate source files. + +## Root metadata is the trigger + +When editable install is supported, only packaging metadata at the **port root** +triggers `pip install -e .`: + +```text +port-root/pyproject.toml detected +port-root/source/pyproject.toml ignored as an install trigger +``` + +A nested source repository may remain untouched. Add minimal root scaffolding +that points package discovery at the existing source package: + +```toml +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "canyonos-port" +version = "0.0.0" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["source/src"] +include = ["pkg*"] +namespaces = true +``` + +Set `where` and `include` from the actual tree and original import spelling. Do +not reference a README or license from this wrapper metadata; file sweeps differ +by runtime capability and a missing referenced file makes the image build fail. + +## Dependencies in nested metadata + +A nested `pyproject.toml` is not installed merely because its Python files are +copied. Keep the source declaration unchanged and repeat its runtime +distributions in each relevant config entry's `requirements` list. This is +compatibility scaffolding, not permission to drop, move, or reclassify declared +dependencies. + +If source metadata is already at the port root, do not create a wrapper. Its +project dependencies participate in the same resolver as config requirements. +Report declared-but-unused toolchain dependencies and their image cost; let the +owner decide whether source metadata should change. + +## Validation boundary + +`ventis build` owns packaging syntax and installation errors. `validate.py` +checks only whether adapter imports appear to require a nested root that the +runtime will not expose. diff --git a/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md new file mode 100644 index 0000000..94f7401 --- /dev/null +++ b/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -0,0 +1,187 @@ +# CanyonOS Core runtime contract + +The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the +CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for +Docker resources. + +Read this reference when implementing an adapter or explaining a validator +finding. Runtime-dependent behavior is expressed as capabilities; run +`validate.py` against the target environment instead of inferring support from +release history. + +## Project root and discovery + +`ventis build` uses the current working directory as the project root. + +| Input | Discovery | +|---|---| +| `agents/*.yaml` | direct yaml glob under the project root | +| `config/global_controller.yaml` | default config, overridable with `-c` | +| workflow | `workflow_file` on a `type: workflow` entry | +| policy | `policy.yaml` beside the selected config file | +| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | + +The config name, yaml `agent.name`, and entrypoint class name form one binding: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +A missing match may skip an image while the command continues, so inspect build +output and generated image tags. + +## Agent yaml and generated stubs + +The consumed yaml shape is: + +```yaml +agent: + name: ExampleAgent + functions: + - name: work + arguments: + - name: query + type: str + returns: + type: dict +``` + +Argument annotations are generated from bare names without adding imports. Use +builtins. Generated methods have no defaults, so every declared argument is +required at the stub call site. `returns` does not control runtime conversion; +it documents whether workflow code should parse the returned string. + +Stub destinations differ by runtime capability and entrypoint layout. Workflow +code in this port convention imports the generated class from +`agents.`. The validator checks that import against declarations +before the workflow image starts. + +## Agent loading and execution + +The local controller effectively performs: + +```python +module = load(entrypoint) +agent_class = getattr(module, configured_name) +agent = agent_class() +result = getattr(agent, method_name)(**args) +``` + +Consequences: + +- The class is module-level and named exactly as configured. +- Construction takes no arguments. +- Declared methods accept yaml argument names as keyword arguments. +- Methods are synchronous; this path does not await a coroutine. +- Dicts and lists are JSON-encoded before entering Redis; other results become + strings. +- A remote Future's `.value()` returns text, not the original Python object. + +Agent import and construction exceptions are caught by the controller. A failed +agent may still advertise healthy because health is written independently of +successful agent loading. That is why image probes import both the runtime and +the entrypoint explicitly. + +## Workflow execution + +The workflow file is executed, not imported. Therefore: + +- module-level code runs at container startup; +- `__name__ == "__main__"`; +- `deploy()` blocks in the web server; +- the workflow function runs once per request; +- its function name determines the REST route exposed by the compatibility + runtime. + +The deployment platform additionally expects `/main` with a `{query: string}` +body. This platform constraint is stricter than the underlying transport. + +Each stub method returns a Future immediately. `.value()` blocks. Dispatching +and resolving inside one comprehension serializes work without raising an +error; dispatch all calls first, then resolve them. + +The workflow container also starts runtime controller code and has its own +package resolution. Probe it independently from agent images. + +## Build context and collisions + +The runtime copies project files while preserving relative paths, then writes +shared runtime modules, generated stubs, and entrypoints into the image. Later +writes can shadow project files. + +Avoid root project modules named like runtime files, including: + +```text +future.py +ventis_context.py +local_controller.py +local_controller_frontend.py +redis_client.py +grpc_options.py +bedrock.py +deploy.py +session_logging.py +workflow_launcher.py +``` + +Also avoid a yaml basename that shadows a different source module imported by an +adapter. The validator checks deterministic flat-name collisions. + +File sweep and editable-install behavior are runtime capabilities. For nested +imports, follow [packaging.md](packaging.md). + +## Dependencies and protobuf + +Agent and workflow images include a small runtime dependency set. Config +`requirements` adds source-specific distributions. A malformed requirements +value can be normalized away while image generation continues; missing imports +then surface only when the agent loads. + +The build compiles gRPC Python stubs on the host and copies them into images. +The image resolver does not necessarily know the generated-code version. A +source dependency that constrains protobuf below the host generator version can +produce a green image build that dies on: + +```text +import local_controller +``` + +Always run that probe before probing the entrypoint. Treat a generated-code / +runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter +source dependencies silently. + +## Credentials capability + +When `env_file` capability is available, the top-level config path is resolved +against the project root and passed at container start. Hidden env files are not +copied into images. Invalid paths are deploy-preflight errors. + +When the capability is unavailable, declaring `env_file` has no effect. If the +source needs credentials, report the capability blocker rather than hardcoding +or vendoring a secret. + +A source that constructs its client at import time works only when credentials +are already in the container environment. Image entrypoint probes therefore use +the same env file as deployment. + +For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). + +## Policy and provider behavior + +No policy file means unrestricted service access. If a policy exists, it needs a +non-empty rules list. Rules are evaluated by specificity and first match; +services excluded from the selected rule fail after request acceptance. + +Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior +and remote networking are covered in [ec2.md](ec2.md). + +## Cleanup boundary + +Stopping foreground deploy normally invokes controller cleanup for recorded +containers and Redis. Hard kills and failures before resource registration may +leave resources behind. + +`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and +`docker_container/`. It does not remove containers or images. Remove exact +leftovers explicitly and preserve source, port scaffolding, and requested +evidence. diff --git a/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md new file mode 100644 index 0000000..361acf9 --- /dev/null +++ b/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting + +Read this after a failed build, image probe, deploy, or request. For mechanisms, +read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, +read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). + +## Build or deploy stops early + +| Symptom | Likely cause | +|---|---| +| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | +| Two services produce one image | Config names collide after lowercase normalization | +| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | +| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | +| Replica conversion `TypeError` | `replicas` is not an integer | +| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | +| Port or container name already in use | A previous deployment did not complete cleanup | + +## Container exits or serves nothing + +| Symptom | Likely cause | +|---|---| +| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | +| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | +| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | +| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | +| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | +| Third-party module is missing | Distribution is absent from source metadata and config requirements | +| Stub import raises `NameError` | yaml argument type is not a bare builtin | +| Source module behaves like an empty stub | A generated stub basename shadowed the source module | +| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | + +## Request is accepted, then fails + +| Symptom | Likely cause | +|---|---| +| Unexpected keyword argument | yaml argument name differs from adapter parameter name | +| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | +| Unauthorized service | The first matching policy rule excludes that service | +| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | +| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | +| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | +| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | +| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | + +## Deployment platform endpoint + +| Symptom | Likely cause | +|---|---| +| 404 while workflow container is healthy | Workflow function is not named `main` | +| 400 before host receives request | Body is not the platform's `{query: string}` shape | +| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | + +## Cleanup + +| Symptom | Likely cause | +|---|---| +| `ventis clean` succeeds but containers remain | The command removes generated directories only | +| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | +| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/.claude/skills/porting-to-canyonos-core/traps.md b/.claude/skills/porting-to-canyonos-core/traps.md deleted file mode 100644 index 2cd2239..0000000 --- a/.claude/skills/porting-to-canyonos-core/traps.md +++ /dev/null @@ -1,86 +0,0 @@ -# Traps - -Symptom-to-cause lookup for a port that is already written. The mechanism behind -each row is in `canyonos-core-contract.md`. Rows marked with a check id are decided -before any of this happens by `validate.py`. - -## Before any container starts - - -| Symptom | Cause | -| ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| `env_file does not exist` on deploy | the path is resolved against the project root you run from; `cmd_deploy` fails before launching anything (V030) | -| `int() argument must be ... not 'NoneType'` on deploy | `provider:` is not lowercase `local`, so no host port was reserved | -| `TypeError: int() argument ...` naming `replicas` | `replicas:` is a list; `_get_replica_placements` accepts that shape but `InstanceManager` calls `int()` on it | -| `AttributeError` inside `GlobalController.__init__` | `config/policy.yaml` exists but is empty, or its `rules:` is null. Absent would have been fine | -| `EC2 deploy preflight failed: missing ec2 config keys`| no top-level `ec2:` block, or an incomplete one. `ssh_user` passes the CLI's shorter list and fails later at provision | -| `generated grpc_stubs are missing or not importable` | `ventis build` has not run on this host, or its output was cleaned | -| An agent missing from the deployment | its config `name` matched no yaml, or its entry has no `entrypoint`; inspect the build warnings | -| Two agents, one image | two config `name`s differing only in case — both tag `ventis-` and the second overwrites the first | - - -## The container dies or serves nothing - - -| Symptom | Cause | -| -------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| Container exits on `import local_controller` | protobuf gencode newer than the resolved runtime; nothing pins the gRPC stack | -| `"No agent loaded"` on the first request | anything below — the agent container's stdout is the only place the cause exists | -| A replica reports `healthy` but answers nothing | same; `healthy` is written before `_load_agent` runs and is never revised | -| `Missing credentials` loading the agent | no `env_file:`, or the key the source reads is not in it | -| `ModuleNotFoundError` for the source's own modules | the original import does not resolve from `/app`; when editable installs are supported, add minimal packaging metadata at the port root (nested source metadata is ignored) (V031) | -| `ModuleNotFoundError` for a third-party package | an import the source needs is missing from the entry's `requirements:` (W006) | -| `ModuleNotFoundError` for nothing in particular | `requirements:` was not a list of strings, so the whole list was dropped with one warning (V014) | -| `NameError` importing a stub | a yaml `type` that is not a builtin (V010) | -| A peer's real code behaves like an empty stub | a project module at the root shares a basename with an `agents/*.yaml`, and the stub is copied over it (V020) | -| The container dies on a CanyonOS Core module name | a project module at the root is called `local_controller.py`, `deploy.py`, `future.py` ... — the runtime is copied flat over it (V019) | - - -## Through `llm_proxy` - -| Symptom | Cause | -| --- | --- | -| `Connection refused` at `127.0.0.1:8080` | that address is the agent container itself, not the Docker host; use `host.docker.internal`, bind the proxy to `0.0.0.0`, and avoid the workflow's host port 8080 | -| Proxy `/healthz` works, but the agent cannot connect | the health probe ran on the host; check the base URL from inside the agent image and whether a remote agent host can reach the proxy | -| OpenAI or Anthropic client says its key is missing before making a request | its SDK still requires the normal key variable; give the container a dummy value and give the proxy process the real value separately | -| boto3 says it cannot locate credentials | botocore signs even a custom endpoint request; give the caller dummy AWS credentials while the proxy process retains its own real AWS identity | -| Upstream answers 401 through the proxy | the proxy process has no real provider key; `/healthz` checks registration, not credentials | -| `BEDROCK_UPSTREAM_HOST` appears to be ignored | it is read into `Config` but never passed to the proxy's boto3 client in this implementation | -| JSON `502` with `proxy_error` | proxy routing or provider code raised; inspect `detail` and proxy logs | -| JSON `502` naming `invoke-with-response-stream`, `converse`, or `converse-stream` | the Bedrock adapter implements only `invoke` | -| A streaming OpenAI or Anthropic call hangs or returns a buffered response | this proxy has no streaming implementation; the port is unsupported without changing source behavior | -| Local agents work but EC2 agents cannot connect | `host.docker.internal` on EC2 names each EC2 Docker host, not the deploying machine; expose a reachable proxy or run one per host | -| Proxy fails to bind port 8080 | the workflow API normally publishes the same host port; run the proxy on another port | - - -## During cleanup - -| Symptom | Cause | -| --- | --- | -| `ventis clean` succeeds but containers still run | the command removes generated directories only; stop `ventis deploy` and remove any exact leftovers | -| `ventis clean` succeeds but `ventis-*` images remain | image deletion is not part of `cmd_clean`; remove the exact tags after their containers stop | -| The next deploy says a port or container name is already in use | the previous deploy crashed or was killed before `GlobalController.cleanup` completed | - - -## The request is accepted and then goes wrong - - -| Symptom | Cause | -| --------------------------------------------------- | ------------------------------------------------------------------------------------- | -| `TypeError: unexpected keyword argument` | yaml `arguments[].name` != the Python parameter name (V008) | -| `Unauthorized: Policy denied access to service 'X'` | `X` is missing from the `access` list of the policy rule that matched | -| `.value()` returns a `str` of a dict | expected — `json.loads` it | -| `Object of type ... is not JSON serializable` | the adapter returned framework objects; serialize with the framework's own serializer | -| Redis holds `` | the method is `async def`; keep the signature sync and `asyncio.run` inside (V009) | -| No faster than the original | calls fused with `.value()`; dispatch all, then resolve all (V018) | -| Debug code runs in production | the workflow is `exec`'d, so `__name__ == "__main__"` (V017) | - - - -## Through the deployment platform's test endpoint - -| Symptom | Cause | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | -| 404 from the test endpoint, container healthy | the workflow function is not named `main`; the platform posts to a hardcoded `/main` (V016) | -| 400 before the request reaches the host | the body key is not `query`; the platform's schema is strict and rejects everything else (V016) | -| The workflow runs but an argument is missing | only `query` is ever sent; every other parameter needs a default (V016) | diff --git a/.claude/skills/porting-to-canyonos-core/validate.py b/.claude/skills/porting-to-canyonos-core/validate.py index d854121..04baf37 100755 --- a/.claude/skills/porting-to-canyonos-core/validate.py +++ b/.claude/skills/porting-to-canyonos-core/validate.py @@ -13,10 +13,9 @@ Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. -Some rules depend on CanyonOS Core features that are not on `main`. Rather than assume, -this script probes the importable `ventis` package and reports each capability -with the PR that carries it. A check whose capability is absent is reported as -UNAVAILABLE, never silently skipped. +Runtime capabilities vary across CanyonOS Core installations. This script probes +the importable `ventis` package directly. A capability-gated check reports +UNAVAILABLE when its behavior cannot be proven. """ import argparse @@ -104,16 +103,14 @@ # Capabilities # # ------------------------------------------------------------------ # # -# Each entry names what carries the capability. A rule gated on an absent -# capability is reported UNAVAILABLE so the gap is visible rather than assumed. +# Stable labels for behavior detected from the importable runtime. They contain +# no external development metadata. CAPABILITY_SOURCE = { - "env_file": "PR #53 (jiajunh/can-232-...), open against main", - "editable_install": "no PR -- only on jiajunh/can-228-create-a-skill-...", - "sweeps_all_files": "no PR -- only on jiajunh/can-228-create-a-skill-...", - # Not on PR #51: its _stub_destination places a stub at one path. The fix - # that also puts it flat lives on the skill branch and has not been proposed. - "stub_two_destinations": "no PR -- 01a70f2 on jiajunh/can-228-porting-to-ventis-skill", + "env_file": "runtime env-file injection", + "editable_install": "editable project installation", + "sweeps_all_files": "full project-file sweep", + "stub_two_destinations": "flat and package stub destinations", } @@ -769,7 +766,7 @@ def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): def check_env_file(report, config, config_path, project_dir): - """V030 -- gated on the env_file support that PR #53 carries.""" + """V030 -- gated on detected env-file injection support.""" declared = config.get("env_file") supported = report.capabilities.get("env_file") @@ -782,15 +779,14 @@ def check_env_file(report, config, config_path, project_dir): f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", "No resolve_env_file in the importable ventis package, so the " "key is silently dropped and the container answers a provider " - "credential error on the first request. It arrives with " - f"{CAPABILITY_SOURCE['env_file']}.", + "credential error on the first request. This port requires the " + "`env_file` runtime capability.", ) else: report.unavailable( "V030", - "env_file is not supported by the importable `ventis` runtime " - f"({CAPABILITY_SOURCE['env_file']}). Credentials have no " - "declared path into a container on this tree.", + "env_file is not supported by the importable `ventis` runtime. " + "Credentials have no declared path into a container on this tree.", ) return @@ -800,8 +796,8 @@ def check_env_file(report, config, config_path, project_dir): config_path, line_of(config), "no `env_file:` in the config", - "Only five VENTIS_* variables reach a container without it. If the " - "source reads any credential from the environment, the first " + "Only runtime-managed VENTIS_* variables are guaranteed without it. " + "If the source reads credentials from the environment, the first " "request fails on a provider error.", ) return @@ -811,7 +807,7 @@ def check_env_file(report, config, config_path, project_dir): def check_import_root(report, project_dir, entrypoint_paths): - """V031 -- gated on the editable install, which no PR carries today.""" + """V031 -- gated on detected editable-install support.""" supported = report.capabilities.get("editable_install") has_metadata = any( os.path.isfile(os.path.join(project_dir, name)) @@ -836,8 +832,7 @@ def check_import_root(report, project_dir, entrypoint_paths): report.unavailable( "V031", "the editable install (`-e .`) is not supported by the importable " - f"`ventis` runtime ({CAPABILITY_SOURCE['editable_install']}). Only modules " - "that land flat at /app import inside a container.", + "`ventis` runtime. Only names rooted at /app import inside a container.", ) for path, lineno, name, location in non_flat: report.error( From 7620b016473973d57dff298d2623b27110a1bfbb Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 18:13:17 -0700 Subject: [PATCH 43/43] Remove Ventis runtime changes from porting skill PR --- ventis/cli.py | 42 +--- .../cloud_provider_logic/EC2/_runtime.py | 12 +- .../cloud_provider_logic/Local/_runtime.py | 12 +- ventis/controller/global_controller.py | 101 +++------ ventis/controller/utils/env_file.py | 159 -------------- ventis/stub_generator.py | 198 ++++-------------- 6 files changed, 82 insertions(+), 442 deletions(-) delete mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index d5b75b6..b43a6b3 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,8 +16,6 @@ import subprocess import sys -from ventis.controller.utils.env_file import resolve_env_file - logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -228,23 +226,6 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) - import yaml - - # Looks up a config entry's YAML and to map stubs to entrypoints. - yaml_by_name = {} - for yaml_path in yaml_files: - with open(yaml_path) as f: - name = yaml.safe_load(f).get("agent", {}).get("name") - if name: - yaml_by_name[name] = yaml_path - - entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} - stub_entrypoints = { - f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] - for n, p in yaml_by_name.items() - if entrypoints_by_name.get(n) - } - stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -308,8 +289,6 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, ) else: @@ -327,7 +306,16 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = yaml_by_name.get(agent_name) + matching_yaml = None + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_name: + matching_yaml = yaml_path + break + if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", @@ -344,8 +332,6 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), - project_dir=project_dir, - stub_entrypoints=stub_entrypoints, ) bake_targets.append( @@ -411,14 +397,6 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() - # Fail here rather than after a fleet of containers is already up without - # the API keys they need. - try: - resolve_env_file(config, base_dir=project_dir) - except ValueError as e: - logger.error("%s", e) - sys.exit(1) - _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..4d5f766 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,7 +23,6 @@ import boto3 -from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -286,15 +285,8 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - - # User secrets from `env_file`. Explicit -e flags above still win over - # anything in the file. - with env_file_args( - _controller, host, ssh_user, container_name, is_local=False - ) as env_args: - cmd.extend(env_args) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index a387f7b..963eef3 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,8 +8,6 @@ import logging -from ventis.controller.utils.env_file import env_file_args - logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -112,15 +110,9 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) + cmd.append(image) - # User secrets from `env_file`. Explicit -e flags above still win, so a - # stray VENTIS_* line in someone's .env cannot break agent wiring. - with env_file_args( - _require_controller(), host, user, runtime_id, _is_local_host(host) - ) as env_args: - cmd.extend(env_args) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 0b30307..1e24f10 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,7 +4,6 @@ import atexit import logging -import shlex import signal import subprocess import threading @@ -17,7 +16,6 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs -from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -66,9 +64,6 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) - # Validate before launching anything: an agent that boots without its - # API keys fails deep inside a container, where it is expensive to debug. - self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -179,7 +174,6 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) - self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -648,28 +642,6 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # - def _ssh_args(self, host, user=None): - """Return the `ssh ... target` prefix used to reach a remote host.""" - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") - ) - return [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - f"{user}@{host}" if user else host, - ] - def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -684,52 +656,41 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run(cmd, capture_output=True, text=True, timeout=180) - - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - self._ssh_args(host, user) + [remote_cmd], - capture_output=True, - text=True, - timeout=180, - ) - - def _push_file(self, local_path, remote_path, host, user=None): - """ - Copy a local file to a remote host over SSH. - - Streams the bytes through `cat` under `umask 077` rather than using - `scp`, so a secrets file is never briefly world-readable on the far - side. - - Anything already sitting at the destination is removed first: `umask` - only governs files the shell creates, and `>` follows symlinks. Without - the `rm`, a local user on the remote host could pre-create the path - world-readable, or point it at a file of their own, and collect - whatever we write there. - - Returns: - subprocess.CompletedProcess - """ - quoted = shlex.quote(remote_path) - remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" - with open(local_path, "rb") as f: - result = subprocess.run( - self._ssh_args(host, user) + [remote_cmd], - stdin=f, + return subprocess.run( + cmd, capture_output=True, text=True, timeout=180 + ) + else: + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get( + "ssh_private_key_path", "~/.ssh/ventis_ec2" + ) + ) + ssh_target = f"{user}@{host}" if user else host + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + ssh_target, + remote_cmd, + ], capture_output=True, text=True, timeout=180, - check=False, - ) - if result.returncode != 0: - raise RuntimeError( - f"Failed to copy {local_path} to {host}:{remote_path}: " - f"{(result.stderr or result.stdout or '').strip()}" ) - return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py deleted file mode 100644 index b1f9381..0000000 --- a/ventis/controller/utils/env_file.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -Pass user secrets (API keys and friends) into agent containers. - -Two scopes, never mixed: `env_file` in the config (self-hosted), and -`DEFAULT_SECRETS_FILE` (managed -- the project's `.env` was never uploaded -there). Below `resolve_env_file` both are just a path: containers on this -machine read it directly, remote ones get a short-lived 0600 copy, and either -way it reaches Docker as `--env-file`. -""" - -import logging -import os -import re -from contextlib import contextmanager - -logger = logging.getLogger(__name__) - -REMOTE_ENV_DIR = "/tmp" -_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") - -# Where a managed deployment leaves the user's secrets. /var/run is tmpfs, so -# the file dies with the host instead of persisting on disk. -DEFAULT_SECRETS_FILE = "/var/run/ventis/secrets.env" - - -def platform_secrets_file(): - """ - The path a managed deployment leaves the user's secrets at. - - `VENTIS_SECRETS_FILE` overrides it for tests and for deployments that - cannot write under /var/run. Nothing sets it in normal operation. - """ - return os.environ.get("VENTIS_SECRETS_FILE", DEFAULT_SECRETS_FILE) - - -def resolve_env_file(config, base_dir=None): - """ - Return the absolute path of the env file to hand containers, or None. - - A file at `platform_secrets_file()` wins over `env_file`. A convention - beating an explicit setting only makes sense because the two describe - different machines: `env_file: .env` describes the user's own, and - carrying that config to a managed deployment does not make the `.env` - exist there. So it warns rather than raising. - - The platform must create that file unconditionally, empty when no secrets - are configured -- its presence means "managed", not "has secrets". - - Raises: - ValueError: the file is configured but unusable. Deploy should fail - here rather than start a fleet of agents with no API keys. - """ - platform_path = platform_secrets_file() - if os.path.isfile(platform_path): - if not os.access(platform_path, os.R_OK): - raise ValueError(f"platform secrets file is not readable: {platform_path}") - if config.get("env_file"): - logger.warning( - "env_file %r is ignored in a managed deployment: secrets come " - "from the platform. Configure them on the platform instead.", - config["env_file"], - ) - return platform_path - - return _resolve_project_env_file(config, base_dir) - - -def _resolve_project_env_file(config, base_dir): - """ - Resolve the `env_file` a self-hosted deployment points at, or None. - - Relative paths resolve against `base_dir` (default: the current working - directory), matching how `entrypoint` and `workflow_file` are resolved. - """ - raw = config.get("env_file") - if not raw: - return None - - path = os.path.expanduser(str(raw)) - if not os.path.isabs(path): - path = os.path.join(base_dir or os.getcwd(), path) - path = os.path.abspath(path) - - if not os.path.exists(path): - raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") - if not os.path.isfile(path): - raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") - if not os.access(path, os.R_OK): - raise ValueError(f"env_file is not readable: {path}") - return path - - -def remote_env_path(container_name): - """ - Where a remote host holds this container's copy of the env file. - - The name is scrubbed down to a shell-safe alphabet. This path is - interpolated into remote commands that `_run_cmd` joins with spaces and - hands to a shell unquoted, so a container name carrying a space would - split the cleanup `rm` into two harmless arguments -- it would exit 0 - while the secrets stayed on the host, with nothing in the log to say so. - """ - safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) - return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" - - -@contextmanager -def env_file_args(controller, host, user, container_name, is_local): - """ - Yield the `docker run` flags that hand the user's env file to a container. - - A container on this machine reads the original file. A container on a - remote host gets a 0600 copy, deleted as soon as the `with` body ends -- - success or failure, since by then the container holds the variables - itself. Keep that body tight around `docker run` so the copy is never - on the host longer than it has to be. - - Yields an empty list when no env file is configured, and when the one - configured is empty. - """ - env_file_path = getattr(controller, "env_file_path", None) - if not env_file_path or _has_no_variables(env_file_path): - yield [] - return - - if is_local: - yield ["--env-file", env_file_path] - return - - remote_path = remote_env_path(container_name) - controller._push_file(env_file_path, remote_path, host, user=user) - try: - yield ["--env-file", remote_path] - finally: - _remove_remote_copy(controller, remote_path, host, user) - - -def _has_no_variables(path): - """ - An empty platform file is routine, so skip it and spare a remote host a - copy pushed and deleted per container. A file that vanished since - resolution is not "empty" -- let it through so Docker reports it. - """ - try: - return os.path.getsize(path) == 0 - except OSError: - return False - - -def _remove_remote_copy(controller, remote_path, host, user): - """Delete a remote copy. Best effort -- never masks the caller's error.""" - try: - result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) - if getattr(result, "returncode", 0) != 0: - logger.warning("Failed to delete env file copy %s on %s", remote_path, host) - except Exception as e: - logger.warning( - "Failed to delete env file copy %s on %s: %s", remote_path, host, e - ) diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 472fe73..e408553 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,123 +271,12 @@ def _format_source(source): return "\n".join(formatted) + "\n" -# Directories ventis build itself generates inside a project -- never swept. -_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} - - -def _sweep_py_files(project_dir): - """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" - swept = [] - for root, dirs, files in os.walk(project_dir): - dirs[:] = [ - d - for d in dirs - if not d.startswith(".") - and not (root == project_dir and d in _GENERATED_DIRS) - ] - for fname in files: - abs_src = os.path.join(root, fname) - if fname.endswith(".py") and not os.path.islink(abs_src): - rel_dst = os.path.relpath(abs_src, project_dir) - swept.append((abs_src, rel_dst)) - return swept - - -# What a project must have at its root for `pip install -e .` to mean anything. -_PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") - - -def _packaging_files(project_dir): - """The metadata `-e .` reads, plus the files that metadata points at. - - The sweep above takes `.py` and nothing else, so a project's pyproject.toml - would never reach the build context and `uv pip install -e .` would fail - with "does not appear to be a Python project". Packaging metadata also - routinely names a README or a license, and the install fails when that - target is missing, so root-level ones come along too. - - Metadata pointing somewhere the sweep does not reach -- a readme under - docs/ -- is not handled here, and surfaces as uv's own error at build time. - """ - if not project_dir or not os.path.isdir(project_dir): - return [] - picked = [] - for name in sorted(os.listdir(project_dir)): - upper = name.upper() - if name in _PACKAGING_FILES or upper.startswith(("README", "LICENSE", "LICENCE")): - path = os.path.join(project_dir, name) - if os.path.isfile(path): - picked.append((path, name)) - return picked - - -def _install_step(project_dir): - """The Dockerfile lines that install requirements, plus the project itself. - - Sweeping the tree in is not enough to make it importable: the process starts - at the context root, so sys.path[0] is /app and only modules sitting there - resolve -- a src/ layout resolves to nothing. `-e .` hands the import root to - the project's own packaging metadata, so Ventis never has to guess a - directory name. A project that declares no metadata gets the plain install. - """ - installable = project_dir and any( - os.path.isfile(os.path.join(project_dir, name)) for name in _PACKAGING_FILES - ) - if not installable: - return ( - "COPY requirements.txt .\n" - "RUN --mount=type=cache,target=/root/.cache/uv " - "uv pip install --system -r requirements.txt\n" - "\n" - "COPY . .\n" - ) - # The project has to be in the context before it can be installed, so the - # copy moves ahead of the install and both resolve in one pass. - return ( - "COPY . .\n" - "RUN --mount=type=cache,target=/root/.cache/uv " - "uv pip install --system -r requirements.txt -e .\n" - ) - - -def _stub_destination(stub_file, stub_entrypoints): - """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" - basename = os.path.basename(stub_file) - entrypoint = stub_entrypoints.get(basename) - if entrypoint: - normalized = entrypoint.replace("\\", "/") - if not normalized.startswith("/") and ".." not in normalized.split("/"): - return normalized - print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") - elif stub_entrypoints: - print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") - return basename - - -def _copy_files(output_dir, files_to_copy): - """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" - real_output_dir = os.path.realpath(output_dir) - for src, dst in files_to_copy: - if not os.path.isfile(src): - print(f" Warning: source file not found, skipping: {src}") - continue - dest_path = os.path.join(output_dir, dst) - real_dest = os.path.realpath(dest_path) - if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: - print(f" Warning: destination escapes build context, skipping: {dst}") - continue - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - - def generate_docker( yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None, - project_dir=None, - stub_entrypoints=None, requirements=None, ): """ @@ -397,13 +286,11 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. - stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. requirements: Optional list of extra pip packages this agent needs. """ with open(yaml_path, "r") as f: @@ -427,14 +314,8 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements_txt) - # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) - files_to_copy += _packaging_files(project_dir) - # Copy general agent files - files_to_copy += [ + files_to_copy = [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -455,16 +336,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs, overwriting the swept real file at the same path + # Copy provided agent stubs if stub_files: for stub_file in stub_files: files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) + (os.path.abspath(stub_file), os.path.basename(stub_file)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -473,7 +351,11 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - _copy_files(output_dir, files_to_copy) + for src, dst in files_to_copy: + if os.path.isfile(src): + shutil.copy2(src, os.path.join(output_dir, dst)) + else: + print(f" Warning: source file not found, skipping: {src}") # Copy the YAML definition too shutil.copy2( @@ -483,14 +365,17 @@ def generate_docker( # ---- Dockerfile ------------------------------------------------------ agent_basename = os.path.basename(agent_file) - install_step = _install_step(project_dir) dockerfile = f"""# syntax=docker/dockerfile:1 FROM python:3.11-slim COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app -{install_step} +COPY requirements.txt . +RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt + +COPY . . + ENV VENTIS_AGENT_NAME={agent_name} ENV VENTIS_AGENT_FILE={agent_basename} @@ -511,8 +396,6 @@ def generate_workflow_docker( output_dir=None, grpc_stubs_dir=None, api_port=8080, - project_dir=None, - stub_entrypoints=None, requirements=None, ): """ @@ -523,12 +406,10 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. - stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). requirements: Optional list of extra pip packages this workflow needs. """ script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -551,14 +432,7 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = ( - _sweep_py_files(project_dir) + _packaging_files(project_dir) - if project_dir - else [] - ) - - files_to_copy += [ + files_to_copy = [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -578,15 +452,10 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - - # Copy stub files, overwriting the swept real file at the same path + + # Copy stub files for stub_file in stub_files: - files_to_copy.append( - ( - os.path.abspath(stub_file), - _stub_destination(stub_file, stub_entrypoints or {}), - ) - ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -594,7 +463,11 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - _copy_files(output_dir, files_to_copy) + for src, dst in files_to_copy: + if os.path.isfile(src): + shutil.copy2(src, os.path.join(output_dir, dst)) + else: + print(f" Warning: source file not found, skipping: {src}") # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading @@ -623,14 +496,17 @@ def start_lc(): f.write(launcher) # ---- Dockerfile ------------------------------------------------------ - install_step = _install_step(project_dir) dockerfile = f"""# syntax=docker/dockerfile:1 FROM python:3.11-slim COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ WORKDIR /app -{install_step} +COPY requirements.txt . +RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt + +COPY . . + EXPOSE 50051 EXPOSE {api_port}