From 4af43ae3abb8da914050f9f83c61cbf2badebf17 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Tue, 25 Aug 2026 20:40:54 -0700 Subject: [PATCH 1/5] [Feature] Pass env / secrets into agent containers Users had no way to get API keys (OpenAI, Anthropic, embedding models) into an agent container. Add a top-level `env_file` key to global_controller.yaml pointing at a local .env file, which reaches every container as `docker run --env-file`. - resolve_env_file validates the path before anything launches, so a missing .env fails at deploy time instead of deep inside a container. Relative paths resolve against the project root, matching entrypoint. - env_file_args is a context manager owning the local-vs-remote decision and the cleanup, so both runtimes share one code path. Local containers read the original file; remote containers get a copy that is deleted as soon as `docker run` returns, whether or not it succeeded. - GlobalController._push_file streams the file over ssh under `umask 077` rather than scp, so the copy is never briefly world-readable and the secret never lands in a command line. _run_cmd's ssh options moved to a shared _ssh_args. --env-file is appended after the explicit -e VENTIS_* flags; Docker gives those precedence regardless of order, so a stray VENTIS_* line in someone's .env cannot break agent wiring. Closes #50 --- ventis/cli.py | 10 ++ .../cloud_provider_logic/EC2/_runtime.py | 12 ++- .../cloud_provider_logic/Local/_runtime.py | 12 ++- ventis/controller/global_controller.py | 94 +++++++++++++------ ventis/controller/utils/env_file.py | 92 ++++++++++++++++++ 5 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index b43a6b3..6d85e1f 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,6 +16,8 @@ import subprocess import sys +from ventis.controller.utils.env_file import resolve_env_file + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -397,6 +399,14 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() + # Fail here rather than after a fleet of containers is already up without + # the API keys they need. + try: + resolve_env_file(config, base_dir=project_dir) + except ValueError as e: + logger.error("%s", e) + sys.exit(1) + _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..9955fa2 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,6 +23,7 @@ import boto3 +from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -285,8 +286,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + + # User secrets from `env_file`. Explicit -e flags above still win over + # anything in the file. + with env_file_args( + _controller, host, ssh_user, container_name, is_local=False + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 963eef3..a387f7b 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,6 +8,8 @@ import logging +from ventis.controller.utils.env_file import env_file_args + logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..241daff 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,6 +4,7 @@ import atexit import logging +import shlex import signal import subprocess import threading @@ -16,6 +17,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -64,6 +66,9 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) + # Validate before launching anything: an agent that boots without its + # API keys fails deep inside a container, where it is expensive to debug. + self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -174,6 +179,7 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) + self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -642,6 +648,28 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # + def _ssh_args(self, host, user=None): + """Return the `ssh ... target` prefix used to reach a remote host.""" + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + ) + return [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + f"{user}@{host}" if user else host, + ] + def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -656,41 +684,45 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=180 - ) - else: - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get( - "ssh_private_key_path", "~/.ssh/ventis_ec2" - ) - ) - ssh_target = f"{user}@{host}" if user else host - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - ssh_target, - remote_cmd, - ], + return subprocess.run(cmd, capture_output=True, text=True, timeout=180) + + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + capture_output=True, + text=True, + timeout=180, + ) + + def _push_file(self, local_path, remote_path, host, user=None): + """ + Copy a local file to a remote host over SSH. + + Streams the bytes through `cat` under `umask 077` rather than using + `scp`, so a secrets file is never briefly world-readable on the far + side. + + Returns: + subprocess.CompletedProcess + """ + remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + with open(local_path, "rb") as f: + result = subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + stdin=f, capture_output=True, text=True, timeout=180, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to copy {local_path} to {host}:{remote_path}: " + f"{(result.stderr or result.stdout or '').strip()}" ) + return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py new file mode 100644 index 0000000..c83f770 --- /dev/null +++ b/ventis/controller/utils/env_file.py @@ -0,0 +1,92 @@ +""" +Pass user secrets (API keys and friends) into agent containers. + +The user points `env_file` in `config/global_controller.yaml` at a local +`.env` file. Containers on this machine read that file directly; containers +on a remote host get a short-lived 0600 copy. Either way the file reaches +Docker as `--env-file`. +""" + +import logging +import os +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +REMOTE_ENV_DIR = "/tmp" + + +def resolve_env_file(config, base_dir=None): + """ + Return the absolute path of the configured env file, or None when unset. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + + Raises: + ValueError: the file is configured but unusable. Deploy should fail + here rather than start a fleet of agents with no API keys. + """ + raw = config.get("env_file") + if not raw: + return None + + path = os.path.expanduser(str(raw)) + if not os.path.isabs(path): + path = os.path.join(base_dir or os.getcwd(), path) + path = os.path.abspath(path) + + if not os.path.exists(path): + raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") + if not os.path.isfile(path): + raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") + if not os.access(path, os.R_OK): + raise ValueError(f"env_file is not readable: {path}") + return path + + +def remote_env_path(container_name): + """Where a remote host holds this container's copy of the env file.""" + return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + + +@contextmanager +def env_file_args(controller, host, user, container_name, is_local): + """ + Yield the `docker run` flags that hand the user's env file to a container. + + A container on this machine reads the original file. A container on a + remote host gets a 0600 copy, deleted as soon as the `with` body ends -- + success or failure, since by then the container holds the variables + itself. Keep that body tight around `docker run` so the copy is never + on the host longer than it has to be. + + Yields an empty list when no `env_file` is configured. + """ + env_file_path = getattr(controller, "env_file_path", None) + if not env_file_path: + yield [] + return + + if is_local: + yield ["--env-file", env_file_path] + return + + remote_path = remote_env_path(container_name) + controller._push_file(env_file_path, remote_path, host, user=user) + try: + yield ["--env-file", remote_path] + finally: + _remove_remote_copy(controller, remote_path, host, user) + + +def _remove_remote_copy(controller, remote_path, host, user): + """Delete a remote copy. Best effort -- never masks the caller's error.""" + try: + result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) + if getattr(result, "returncode", 0) != 0: + logger.warning("Failed to delete env file copy %s on %s", remote_path, host) + except Exception as e: + logger.warning( + "Failed to delete env file copy %s on %s: %s", remote_path, host, e + ) From 087ee15b66e967937297580fc551c121c7301a20 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 13:28:52 -0700 Subject: [PATCH 2/5] Harden the remote env file copy against a hostile /tmp Two holes in the remote staging path, both found reviewing the feature commit. `umask 077` only governs files the shell creates, and `>` follows symlinks -- so it did not actually guarantee a 0600 copy. The destination path is fully predictable (`/tmp/ventis-env-ventis-ec2--`), so a local user on the remote host could pre-create it world-readable, or point it at a file of their own, and collect the API keys. Remove whatever sits at the path before writing; `rm -f` unlinks a symlink rather than following it, so `cat >` then creates a fresh file under the umask. `_run_cmd` joins its argv with spaces and hands the result to a remote shell unquoted. `_push_file` quoted its path but the cleanup `rm` did not, so a container name containing a space split the `rm` into two arguments that matched nothing -- it exited 0 while the secrets file stayed on the host, and the returncode check logged nothing. Scrub the name down to [A-Za-z0-9_.-] in remote_env_path, which also closes the same gap in the `--env-file` argument and in any future use of that path. Still open, tracked separately: a push that dies mid-transfer can leave a copy behind, since the cleanup only covers the `docker run` that follows. On EC2 the instance is terminated on that path, which disposes of it. --- ventis/controller/global_controller.py | 9 ++++++++- ventis/controller/utils/env_file.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 241daff..0b30307 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -704,10 +704,17 @@ def _push_file(self, local_path, remote_path, host, user=None): `scp`, so a secrets file is never briefly world-readable on the far side. + Anything already sitting at the destination is removed first: `umask` + only governs files the shell creates, and `>` follows symlinks. Without + the `rm`, a local user on the remote host could pre-create the path + world-readable, or point it at a file of their own, and collect + whatever we write there. + Returns: subprocess.CompletedProcess """ - remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + quoted = shlex.quote(remote_path) + remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" with open(local_path, "rb") as f: result = subprocess.run( self._ssh_args(host, user) + [remote_cmd], diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index c83f770..6cd77b6 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -9,11 +9,13 @@ import logging import os +import re from contextlib import contextmanager logger = logging.getLogger(__name__) REMOTE_ENV_DIR = "/tmp" +_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") def resolve_env_file(config, base_dir=None): @@ -46,8 +48,17 @@ def resolve_env_file(config, base_dir=None): def remote_env_path(container_name): - """Where a remote host holds this container's copy of the env file.""" - return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + """ + Where a remote host holds this container's copy of the env file. + + The name is scrubbed down to a shell-safe alphabet. This path is + interpolated into remote commands that `_run_cmd` joins with spaces and + hands to a shell unquoted, so a container name carrying a space would + split the cleanup `rm` into two harmless arguments -- it would exit 0 + while the secrets stayed on the host, with nothing in the log to say so. + """ + safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) + return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" @contextmanager From 5d39ab270969f9790bc8aaeb605fe119cd456534 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 27 Aug 2026 15:30:47 -0700 Subject: [PATCH 3/5] Support managed-deployment secrets alongside self-hosted env_file --- README.md | 9 ++++ ventis/controller/utils/env_file.py | 74 +++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5236328..81d944f 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,15 @@ cp -r ../examples/* ./ #### Step 1: Configure the Global Controller Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +#### Step 1.1: Passing secrets to agents (optional) + +Agents that need API keys read them from environment variables. Point `env_file` at a `.env` file to have Ventis inject it into every agent container: + +```yaml +# config/global_controller.yaml +env_file: .env +``` + #### Step 2: Build the project ```bash ventis build diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index 6cd77b6..b1f9381 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -1,10 +1,11 @@ """ Pass user secrets (API keys and friends) into agent containers. -The user points `env_file` in `config/global_controller.yaml` at a local -`.env` file. Containers on this machine read that file directly; containers -on a remote host get a short-lived 0600 copy. Either way the file reaches -Docker as `--env-file`. +Two scopes, never mixed: `env_file` in the config (self-hosted), and +`DEFAULT_SECRETS_FILE` (managed -- the project's `.env` was never uploaded +there). Below `resolve_env_file` both are just a path: containers on this +machine read it directly, remote ones get a short-lived 0600 copy, and either +way it reaches Docker as `--env-file`. """ import logging @@ -17,18 +18,60 @@ REMOTE_ENV_DIR = "/tmp" _UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") +# Where a managed deployment leaves the user's secrets. /var/run is tmpfs, so +# the file dies with the host instead of persisting on disk. +DEFAULT_SECRETS_FILE = "/var/run/ventis/secrets.env" + + +def platform_secrets_file(): + """ + The path a managed deployment leaves the user's secrets at. + + `VENTIS_SECRETS_FILE` overrides it for tests and for deployments that + cannot write under /var/run. Nothing sets it in normal operation. + """ + return os.environ.get("VENTIS_SECRETS_FILE", DEFAULT_SECRETS_FILE) + def resolve_env_file(config, base_dir=None): """ - Return the absolute path of the configured env file, or None when unset. + Return the absolute path of the env file to hand containers, or None. - Relative paths resolve against `base_dir` (default: the current working - directory), matching how `entrypoint` and `workflow_file` are resolved. + A file at `platform_secrets_file()` wins over `env_file`. A convention + beating an explicit setting only makes sense because the two describe + different machines: `env_file: .env` describes the user's own, and + carrying that config to a managed deployment does not make the `.env` + exist there. So it warns rather than raising. + + The platform must create that file unconditionally, empty when no secrets + are configured -- its presence means "managed", not "has secrets". Raises: ValueError: the file is configured but unusable. Deploy should fail here rather than start a fleet of agents with no API keys. """ + platform_path = platform_secrets_file() + if os.path.isfile(platform_path): + if not os.access(platform_path, os.R_OK): + raise ValueError(f"platform secrets file is not readable: {platform_path}") + if config.get("env_file"): + logger.warning( + "env_file %r is ignored in a managed deployment: secrets come " + "from the platform. Configure them on the platform instead.", + config["env_file"], + ) + return platform_path + + return _resolve_project_env_file(config, base_dir) + + +def _resolve_project_env_file(config, base_dir): + """ + Resolve the `env_file` a self-hosted deployment points at, or None. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + """ raw = config.get("env_file") if not raw: return None @@ -72,10 +115,11 @@ def env_file_args(controller, host, user, container_name, is_local): itself. Keep that body tight around `docker run` so the copy is never on the host longer than it has to be. - Yields an empty list when no `env_file` is configured. + Yields an empty list when no env file is configured, and when the one + configured is empty. """ env_file_path = getattr(controller, "env_file_path", None) - if not env_file_path: + if not env_file_path or _has_no_variables(env_file_path): yield [] return @@ -91,6 +135,18 @@ def env_file_args(controller, host, user, container_name, is_local): _remove_remote_copy(controller, remote_path, host, user) +def _has_no_variables(path): + """ + An empty platform file is routine, so skip it and spare a remote host a + copy pushed and deleted per container. A file that vanished since + resolution is not "empty" -- let it through so Docker reports it. + """ + try: + return os.path.getsize(path) == 0 + except OSError: + return False + + def _remove_remote_copy(controller, remote_path, host, user): """Delete a remote copy. Best effort -- never masks the caller's error.""" try: From 2423ff333de0ba5c12a8776d506f14632ae69b58 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 19:04:49 -0700 Subject: [PATCH 4/5] Adopt .car artifact layout --- README.md | 33 ++++++----- ventis/cli.py | 80 +++++++++++++++----------- ventis/controller/global_controller.py | 16 ++++-- 3 files changed, 75 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 81d944f..d9a3e52 100644 --- a/README.md +++ b/README.md @@ -39,26 +39,29 @@ cd my-app ``` This command creates a new directory `my-app` with the following structure: ``` -├── agents/ # Agent implementations and YAML definitions -│ ├── example_agent.py -│ └── example_agent.yaml -├── workflows/ # Workflow scripts (deployed as REST APIs) -│ └── example_workflow.py -├── config/ -│ ├── global_controller.yaml # Deployment configuration -│ └── policy.yaml # Access control rules -├── stubs/ # Generated agent stubs (auto-generated) -├── grpc_stubs/ # Generated gRPC stubs (auto-generated) -└── README.md # Readme for the project +├── .car/ # Canyon artifacts; source stays outside +│ ├── agents/ +│ │ ├── example_agent.py # Thin adapter +│ │ └── example_agent.yaml # Callable declaration +│ ├── workflow/ +│ │ └── example_workflow.py # HTTP workflow +│ ├── config/ +│ │ ├── global_controller.yaml +│ │ └── policy.yaml +│ ├── stubs/ # Generated by ventis build +│ └── grpc_stubs/ # Generated by ventis build +├── / # Unchanged application source +└── README.md ``` The Readme in the newly created project directory provides a quick overview of the project and how to use it. Including how to add new files etc. We provide some overview in next few steps. #### Step 2: Define Your Agents -Place your agent logic (`.py`) and definitions (`.yaml`) in the `agents/` directory. +Place Canyon adapters and declarations under `.car/agents/`; keep application +source in its existing location. -- **`agents/my_agent.yaml`**: Defines methods and schemas. -- **`agents/my_agent.py`**: Contains the actual Python implementation. +- **`.car/agents/my_agent.yaml`**: Defines methods and schemas. +- **`.car/agents/my_agent.py`**: Contains the thin Canyon adapter. We have provided an example of a finance agent and a market research agent in the `examples/` directory. To run the example, copy files into your newly created project directory from within the your my-app directory with the command - @@ -69,7 +72,7 @@ cp -r ../examples/* ./ ## Deployment Guide #### Step 1: Configure the Global Controller -Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +Edit `.car/config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. #### Step 1.1: Passing secrets to agents (optional) diff --git a/ventis/cli.py b/ventis/cli.py index 340f6dd..5b0a974 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -21,7 +21,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" +DEFAULT_CONFIG_PATH = ".car/config/global_controller.yaml" EC2_REQUIRED_CONFIG_KEYS = ( "ami_id", "subnet_id", @@ -53,6 +53,12 @@ def _load_config(config_path): return yaml.safe_load(f) +def _artifact_root(config_path): + """Return ```` for ``/config/``.""" + config_dir = os.path.dirname(os.path.abspath(config_path)) + return os.path.dirname(config_dir) + + def _normalize_requirements(agent_cfg): """Return an agent's `requirements` list, or [] if absent/null/malformed.""" requirements = agent_cfg.get("requirements") or [] @@ -175,12 +181,16 @@ def cmd_new_project(args): logger.error("Templates directory not found at %s", templates_dir) sys.exit(1) - # Copy the entire templates tree into the new project - shutil.copytree(templates_dir, project_dir) + # Canyon-owned files live under .car; keep the project README at the root. + artifact_root = os.path.join(project_dir, ".car") + shutil.copytree(templates_dir, artifact_root) + template_readme = os.path.join(artifact_root, "README.md") + if os.path.isfile(template_readme): + shutil.move(template_readme, os.path.join(project_dir, "README.md")) # Create empty output directories - os.makedirs(os.path.join(project_dir, "stubs"), exist_ok=True) - os.makedirs(os.path.join(project_dir, "grpc_stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "grpc_stubs"), exist_ok=True) logger.info("Created new Ventis project: %s", project_dir) logger.info("") @@ -199,7 +209,7 @@ def cmd_build(args): Generate stubs, compile gRPC protos, generate Docker contexts, and build Docker images. - Must be run from the project root (where config/ lives). + Must be run from the source project root (where .car/ lives). """ config_path = args.config if not os.path.isfile(config_path): @@ -209,13 +219,14 @@ def cmd_build(args): config = _load_config(config_path) agents = config.get("agents", []) project_dir = os.getcwd() + artifact_root = _artifact_root(config_path) package_dir = _get_package_dir() # -------------------------------------------------------------- # # Step 1: Discover agent YAML files and generate Python stubs # # -------------------------------------------------------------- # - agents_dir = os.path.join(project_dir, "agents") - stubs_dir = os.path.join(project_dir, "stubs") + agents_dir = os.path.join(artifact_root, "agents") + stubs_dir = os.path.join(artifact_root, "stubs") os.makedirs(stubs_dir, exist_ok=True) from ventis.stub_generator import ( @@ -229,8 +240,7 @@ def cmd_build(args): logger.warning("No agent YAML files found in %s", agents_dir) import yaml - - # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} for yaml_path in yaml_files: with open(yaml_path) as f: @@ -240,9 +250,9 @@ def cmd_build(args): entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} stub_entrypoints = { - f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] - for n, p in yaml_by_name.items() - if entrypoints_by_name.get(n) + f"{os.path.splitext(os.path.basename(path))[0]}.py": entrypoints_by_name[name] + for name, path in yaml_by_name.items() + if entrypoints_by_name.get(name) } stub_paths = [] @@ -256,7 +266,7 @@ def cmd_build(args): # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # - grpc_stubs_dir = os.path.join(project_dir, "grpc_stubs") + grpc_stubs_dir = os.path.join(artifact_root, "grpc_stubs") os.makedirs(grpc_stubs_dir, exist_ok=True) proto_dir = os.path.join(package_dir, "controller", "proto") @@ -294,12 +304,12 @@ def cmd_build(args): ) continue - workflow_path = os.path.join(project_dir, workflow_file) + workflow_path = os.path.join(artifact_root, workflow_file) if not os.path.isfile(workflow_path): logger.error("Workflow file not found: %s", workflow_path) continue - docker_context = os.path.join(project_dir, "docker_container", "Workflow") + docker_context = os.path.join(artifact_root, "docker_container", "Workflow") logger.info("Generating workflow Docker context for '%s'", agent_name) generate_workflow_docker( workflow_path, @@ -307,9 +317,9 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, stub_entrypoints=stub_entrypoints, - requirements=_normalize_requirements(agent_cfg), ) else: @@ -321,13 +331,14 @@ def cmd_build(args): ) continue - agent_file = os.path.join(project_dir, entrypoint) + agent_file = os.path.join(artifact_root, entrypoint) if not os.path.isfile(agent_file): logger.error("Agent file not found: %s", agent_file) continue # Find matching YAML by agent name matching_yaml = yaml_by_name.get(agent_name) + if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", @@ -335,7 +346,7 @@ def cmd_build(args): ) continue - docker_context = os.path.join(project_dir, "docker_container", agent_name) + docker_context = os.path.join(artifact_root, "docker_container", agent_name) logger.info("Generating Docker context for '%s'", agent_name) generate_docker( matching_yaml, @@ -343,9 +354,9 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, stub_entrypoints=stub_entrypoints, - requirements=_normalize_requirements(agent_cfg), ) bake_targets.append( @@ -362,7 +373,7 @@ def cmd_build(args): if not bake_targets: logger.info("No Docker images to build.") elif _docker_available() and _docker_available(("docker", "buildx", "version")): - docker_container_dir = os.path.join(project_dir, "docker_container") + docker_container_dir = os.path.join(artifact_root, "docker_container") os.makedirs(docker_container_dir, exist_ok=True) bake_file_path = os.path.join(docker_container_dir, "docker-bake.json") _write_bake_file(bake_targets, bake_file_path, _docker_platform()) @@ -409,23 +420,23 @@ def cmd_deploy(args): sys.exit(1) config = _load_config(config_path) - project_dir = os.getcwd() + artifact_root = _artifact_root(config_path) # Fail here rather than after a fleet of containers is already up without - # the API keys they need. + # the API keys they need. env_file remains relative to the source root. try: - resolve_env_file(config, base_dir=project_dir) + resolve_env_file(config, base_dir=os.getcwd()) except ValueError as e: logger.error("%s", e) sys.exit(1) - _ensure_grpc_stubs_importable(project_dir) + _ensure_grpc_stubs_importable(artifact_root) if any( agent.get("provider", "local").upper() == "EC2" for agent in config.get("agents", []) ): - _preflight_ec2_deploy(config, project_dir) + _preflight_ec2_deploy(config, artifact_root) from ventis.controller.global_controller import GlobalController @@ -466,13 +477,10 @@ def cmd_clean(args): """ Remove generated stubs, gRPC files, and Docker build contexts. """ - project_dir = os.getcwd() - - paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), - ] + config_path = getattr(args, "config", DEFAULT_CONFIG_PATH) + artifact_root = _artifact_root(config_path) + generated_names = ("stubs", "grpc_stubs", "docker_container") + paths_to_clean = [os.path.join(artifact_root, name) for name in generated_names] for path in paths_to_clean: if os.path.exists(path): @@ -538,6 +546,12 @@ def main(): "clean", help="Remove generated stubs, compiled protos, and Docker contexts", ) + clean.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"Select the artifact layout via its config (default: {DEFAULT_CONFIG_PATH})", + ) clean.set_defaults(func=cmd_clean) args = parser.parse_args() diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 0b30307..604cf72 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -28,8 +28,11 @@ from ventis.utils.redis_client import RedisClient from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS -# Add generated grpc_stubs from the local project to the path -sys.path.insert(0, os.path.abspath("grpc_stubs")) +# Support direct launch from the project root. The CLI may already have +# inserted the selected artifact path first. +grpc_stubs_path = os.path.abspath(".car/grpc_stubs") +if grpc_stubs_path not in sys.path: + sys.path.append(grpc_stubs_path) import local_controler_pb2 import local_controler_pb2_grpc import grpc @@ -783,9 +786,10 @@ def stop(self): if __name__ == "__main__": - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = os.path.join(script_dir, "..", "..") - default_config = os.path.join(project_root, "config", "global_controller.yaml") + project_root = os.getcwd() + default_config = os.path.join( + project_root, ".car", "config", "global_controller.yaml" + ) import argparse @@ -794,7 +798,7 @@ def stop(self): "-c", "--config", default=default_config, - help="Path to the YAML config file (default: config/global_controller.yaml)", + help="Path to the YAML config file (default: .car/config/global_controller.yaml)", ) args = parser.parse_args() From c079e2526992a2880c566779e40c6fa8b77780e9 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Fri, 28 Aug 2026 20:09:10 -0700 Subject: [PATCH 5/5] remove redundant fallback --- tests/test_cli.py | 10 +++++----- ventis/cli.py | 11 ++++------- ventis/controller/global_controller.py | 5 ----- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 406b95d..a05d592 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -40,6 +40,7 @@ def test_deploy_skips_ec2_preflight_for_local_config( with ( patch("ventis.cli.os.path.isfile", return_value=True), patch("ventis.cli._load_config", return_value=config), + patch("ventis.cli.resolve_env_file"), patch.dict( sys.modules, {"ventis.controller.global_controller": controller_module} ), @@ -71,6 +72,7 @@ def test_deploy_runs_ec2_preflight_for_ec2_config( with ( patch("ventis.cli.os.path.isfile", return_value=True), patch("ventis.cli._load_config", return_value=config), + patch("ventis.cli.resolve_env_file"), patch.dict( sys.modules, {"ventis.controller.global_controller": controller_module} ), @@ -78,12 +80,11 @@ def test_deploy_runs_ec2_preflight_for_ec2_config( cli.cmd_deploy(args) ensure_grpc.assert_called_once_with(os.getcwd()) - preflight.assert_called_once_with(config, os.getcwd()) + preflight.assert_called_once_with(config) controller.run.assert_called_once_with() - @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._require_docker_for_ec2") - def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc): + def test_preflight_does_not_require_ssh_fields(self, require_docker): config = { "ec2": { "ami_id": "ami-123", @@ -93,10 +94,9 @@ def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc } } - cli._preflight_ec2_deploy(config, os.getcwd()) + cli._preflight_ec2_deploy(config) require_docker.assert_called_once_with("deploy") - ensure_grpc.assert_called_once_with(os.getcwd()) class CliBuildTests(unittest.TestCase): diff --git a/ventis/cli.py b/ventis/cli.py index 5b0a974..56db794 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -150,7 +150,7 @@ def _ensure_grpc_stubs_importable(project_dir): ) from exc -def _preflight_ec2_deploy(config, project_dir): +def _preflight_ec2_deploy(config): ec2_cfg = config.get("ec2", {}) missing = [key for key in EC2_REQUIRED_CONFIG_KEYS if not ec2_cfg.get(key)] if missing: @@ -159,7 +159,6 @@ def _preflight_ec2_deploy(config, project_dir): ) _require_docker_for_ec2("deploy") - _ensure_grpc_stubs_importable(project_dir) # ------------------------------------------------------------------ # @@ -288,7 +287,7 @@ def cmd_build(args): ) # -------------------------------------------------------------- # - # Step 4: Generate Docker contexts # + # Step 3: Generate Docker contexts # # -------------------------------------------------------------- # bake_targets = [] for agent_cfg in agents: @@ -368,7 +367,7 @@ def cmd_build(args): ) # -------------------------------------------------------------- # - # Step 5: Build all Docker images # + # Step 4: Build all Docker images # # -------------------------------------------------------------- # if not bake_targets: logger.info("No Docker images to build.") @@ -436,7 +435,7 @@ def cmd_deploy(args): agent.get("provider", "local").upper() == "EC2" for agent in config.get("agents", []) ): - _preflight_ec2_deploy(config, artifact_root) + _preflight_ec2_deploy(config) from ventis.controller.global_controller import GlobalController @@ -486,8 +485,6 @@ def cmd_clean(args): if os.path.exists(path): logger.info("Cleaning %s...", path) if os.path.isdir(path): - import shutil - shutil.rmtree(path) else: os.remove(path) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 604cf72..b6f1a26 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -28,11 +28,6 @@ from ventis.utils.redis_client import RedisClient from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS -# Support direct launch from the project root. The CLI may already have -# inserted the selected artifact path first. -grpc_stubs_path = os.path.abspath(".car/grpc_stubs") -if grpc_stubs_path not in sys.path: - sys.path.append(grpc_stubs_path) import local_controler_pb2 import local_controler_pb2_grpc import grpc