[Improvement] Keep Canyon artifacts under .car

README.md

@@ -39,26 +39,29 @@ cd my-app
 ```
 This command creates a new directory `my-app` with the following structure:
 ```
-├── agents/               # Agent implementations and YAML definitions
-│   ├── example_agent.py
-│   └── example_agent.yaml
-├── workflows/            # Workflow scripts (deployed as REST APIs)
-│   └── example_workflow.py
-├── config/
-│   ├── global_controller.yaml   # Deployment configuration
-│   └── policy.yaml              # Access control rules
-├── stubs/                # Generated agent stubs (auto-generated)
-├── grpc_stubs/           # Generated gRPC stubs (auto-generated)
-└── README.md             # Readme for the project  
+├── .car/                         # Canyon artifacts; source stays outside
+│   ├── agents/
+│   │   ├── example_agent.py      # Thin adapter
+│   │   └── example_agent.yaml    # Callable declaration
+│   ├── workflow/
+│   │   └── example_workflow.py   # HTTP workflow
+│   ├── config/
+│   │   ├── global_controller.yaml
+│   │   └── policy.yaml
+│   ├── stubs/                    # Generated by ventis build
+│   └── grpc_stubs/               # Generated by ventis build
+├── <existing source>/            # Unchanged application source
+└── README.md
 ```
 The Readme in the newly created project directory provides a quick overview of the project and how to use it. Including how to add new files etc. We provide some overview in next few steps.
 
 
 #### Step 2: Define Your Agents
-Place your agent logic (`.py`) and definitions (`.yaml`) in the `agents/` directory.
+Place Canyon adapters and declarations under `.car/agents/`; keep application
+source in its existing location.
 
-- **`agents/my_agent.yaml`**: Defines methods and schemas.
-- **`agents/my_agent.py`**: Contains the actual Python implementation.
+- **`.car/agents/my_agent.yaml`**: Defines methods and schemas.
+- **`.car/agents/my_agent.py`**: Contains the thin Canyon adapter.
 
 We have provided an example of a finance agent and a market research agent in the `examples/` directory. To run the example, copy files into your newly created project directory from within the your my-app directory with the command - 
 
@@ -69,7 +72,7 @@ cp -r ../examples/* ./
 ## Deployment Guide
 
 #### Step 1: Configure the Global Controller
-Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default.
+Edit `.car/config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default.
 
 #### Step 1.1: Passing secrets to agents (optional)
 

tests/test_cli.py

@@ -40,6 +40,7 @@ def test_deploy_skips_ec2_preflight_for_local_config(
         with (
             patch("ventis.cli.os.path.isfile", return_value=True),
             patch("ventis.cli._load_config", return_value=config),
+            patch("ventis.cli.resolve_env_file"),
             patch.dict(
                 sys.modules, {"ventis.controller.global_controller": controller_module}
             ),
@@ -71,19 +72,19 @@ def test_deploy_runs_ec2_preflight_for_ec2_config(
         with (
             patch("ventis.cli.os.path.isfile", return_value=True),
             patch("ventis.cli._load_config", return_value=config),
+            patch("ventis.cli.resolve_env_file"),
             patch.dict(
                 sys.modules, {"ventis.controller.global_controller": controller_module}
             ),
         ):
             cli.cmd_deploy(args)
 
         ensure_grpc.assert_called_once_with(os.getcwd())
-        preflight.assert_called_once_with(config, os.getcwd())
+        preflight.assert_called_once_with(config)
         controller.run.assert_called_once_with()
 
-    @patch("ventis.cli._ensure_grpc_stubs_importable")
     @patch("ventis.cli._require_docker_for_ec2")
-    def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc):
+    def test_preflight_does_not_require_ssh_fields(self, require_docker):
         config = {
             "ec2": {
                 "ami_id": "ami-123",
@@ -93,10 +94,9 @@ def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc
             }
         }
 
-        cli._preflight_ec2_deploy(config, os.getcwd())
+        cli._preflight_ec2_deploy(config)
 
         require_docker.assert_called_once_with("deploy")
-        ensure_grpc.assert_called_once_with(os.getcwd())
 
 
 class CliBuildTests(unittest.TestCase):

ventis/cli.py

@@ -21,7 +21,7 @@
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("ventis")
 DEFAULT_DOCKER_PLATFORM = "linux/amd64"
-DEFAULT_CONFIG_PATH = "config/global_controller.yaml"
+DEFAULT_CONFIG_PATH = ".car/config/global_controller.yaml"
 EC2_REQUIRED_CONFIG_KEYS = (
     "ami_id",
     "subnet_id",
@@ -53,6 +53,12 @@ def _load_config(config_path):
         return yaml.safe_load(f)
 
 
+def _artifact_root(config_path):
+    """Return ``<artifact root>`` for ``<artifact root>/config/<config>``."""
+    config_dir = os.path.dirname(os.path.abspath(config_path))
+    return os.path.dirname(config_dir)
+
+
 def _normalize_requirements(agent_cfg):
     """Return an agent's `requirements` list, or [] if absent/null/malformed."""
     requirements = agent_cfg.get("requirements") or []
@@ -144,7 +150,7 @@ def _ensure_grpc_stubs_importable(project_dir):
         ) from exc
 
 
-def _preflight_ec2_deploy(config, project_dir):
+def _preflight_ec2_deploy(config):
     ec2_cfg = config.get("ec2", {})
     missing = [key for key in EC2_REQUIRED_CONFIG_KEYS if not ec2_cfg.get(key)]
     if missing:
@@ -153,7 +159,6 @@ def _preflight_ec2_deploy(config, project_dir):
         )
 
     _require_docker_for_ec2("deploy")
-    _ensure_grpc_stubs_importable(project_dir)
 
 
 # ------------------------------------------------------------------ #
@@ -175,12 +180,16 @@ def cmd_new_project(args):
         logger.error("Templates directory not found at %s", templates_dir)
         sys.exit(1)
 
-    # Copy the entire templates tree into the new project
-    shutil.copytree(templates_dir, project_dir)
+    # Canyon-owned files live under .car; keep the project README at the root.
+    artifact_root = os.path.join(project_dir, ".car")
+    shutil.copytree(templates_dir, artifact_root)
+    template_readme = os.path.join(artifact_root, "README.md")
+    if os.path.isfile(template_readme):
+        shutil.move(template_readme, os.path.join(project_dir, "README.md"))
 
     # Create empty output directories
-    os.makedirs(os.path.join(project_dir, "stubs"), exist_ok=True)
-    os.makedirs(os.path.join(project_dir, "grpc_stubs"), exist_ok=True)
+    os.makedirs(os.path.join(artifact_root, "stubs"), exist_ok=True)
+    os.makedirs(os.path.join(artifact_root, "grpc_stubs"), exist_ok=True)
 
     logger.info("Created new Ventis project: %s", project_dir)
     logger.info("")
@@ -199,7 +208,7 @@ def cmd_build(args):
     Generate stubs, compile gRPC protos, generate Docker contexts,
     and build Docker images.
 
-    Must be run from the project root (where config/ lives).
+    Must be run from the source project root (where .car/ lives).
     """
     config_path = args.config
     if not os.path.isfile(config_path):
@@ -209,13 +218,14 @@ def cmd_build(args):
     config = _load_config(config_path)
     agents = config.get("agents", [])
     project_dir = os.getcwd()
+    artifact_root = _artifact_root(config_path)
     package_dir = _get_package_dir()
 
     # -------------------------------------------------------------- #
     #  Step 1: Discover agent YAML files and generate Python stubs    #
     # -------------------------------------------------------------- #
-    agents_dir = os.path.join(project_dir, "agents")
-    stubs_dir = os.path.join(project_dir, "stubs")
+    agents_dir = os.path.join(artifact_root, "agents")
+    stubs_dir = os.path.join(artifact_root, "stubs")
     os.makedirs(stubs_dir, exist_ok=True)
 
     from ventis.stub_generator import (
@@ -229,8 +239,7 @@ def cmd_build(args):
         logger.warning("No agent YAML files found in %s", agents_dir)
 
     import yaml
-
-    # Looks up a config entry's YAML and to map stubs to entrypoints.
+
     yaml_by_name = {}
     for yaml_path in yaml_files:
         with open(yaml_path) as f:
@@ -240,9 +249,9 @@ def cmd_build(args):
 
     entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents}
     stub_entrypoints = {
-        f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n]
-        for n, p in yaml_by_name.items()
-        if entrypoints_by_name.get(n)
+        f"{os.path.splitext(os.path.basename(path))[0]}.py": entrypoints_by_name[name]
+        for name, path in yaml_by_name.items()
+        if entrypoints_by_name.get(name)
     }
 
     stub_paths = []
@@ -256,7 +265,7 @@ def cmd_build(args):
     # -------------------------------------------------------------- #
     #  Step 2: Compile gRPC protobuf stubs                            #
     # -------------------------------------------------------------- #
-    grpc_stubs_dir = os.path.join(project_dir, "grpc_stubs")
+    grpc_stubs_dir = os.path.join(artifact_root, "grpc_stubs")
     os.makedirs(grpc_stubs_dir, exist_ok=True)
 
     proto_dir = os.path.join(package_dir, "controller", "proto")
@@ -278,7 +287,7 @@ def cmd_build(args):
         )
 
     # -------------------------------------------------------------- #
-    #  Step 4: Generate Docker contexts                               #
+    #  Step 3: Generate Docker contexts                               #
     # -------------------------------------------------------------- #
     bake_targets = []
     for agent_cfg in agents:
@@ -294,22 +303,22 @@ def cmd_build(args):
                 )
                 continue
 
-            workflow_path = os.path.join(project_dir, workflow_file)
+            workflow_path = os.path.join(artifact_root, workflow_file)
             if not os.path.isfile(workflow_path):
                 logger.error("Workflow file not found: %s", workflow_path)
                 continue
 
-            docker_context = os.path.join(project_dir, "docker_container", "Workflow")
+            docker_context = os.path.join(artifact_root, "docker_container", "Workflow")
             logger.info("Generating workflow Docker context for '%s'", agent_name)
             generate_workflow_docker(
                 workflow_path,
                 stub_paths,
                 output_dir=docker_context,
                 grpc_stubs_dir=grpc_stubs_dir,
                 api_port=agent_cfg.get("api_port", 8080),
+                requirements=_normalize_requirements(agent_cfg),
                 project_dir=project_dir,
                 stub_entrypoints=stub_entrypoints,
-                requirements=_normalize_requirements(agent_cfg),
             )
 
         else:
@@ -321,31 +330,32 @@ def cmd_build(args):
                 )
                 continue
 
-            agent_file = os.path.join(project_dir, entrypoint)
+            agent_file = os.path.join(artifact_root, entrypoint)
             if not os.path.isfile(agent_file):
                 logger.error("Agent file not found: %s", agent_file)
                 continue
 
             # Find matching YAML by agent name
             matching_yaml = yaml_by_name.get(agent_name)
+
             if not matching_yaml:
                 logger.warning(
                     "No YAML definition found for agent '%s', skipping Docker",
                     agent_name,
                 )
                 continue
 
-            docker_context = os.path.join(project_dir, "docker_container", agent_name)
+            docker_context = os.path.join(artifact_root, "docker_container", agent_name)
             logger.info("Generating Docker context for '%s'", agent_name)
             generate_docker(
                 matching_yaml,
                 agent_file,
                 output_dir=docker_context,
                 grpc_stubs_dir=grpc_stubs_dir,
                 stub_files=stub_paths,
+                requirements=_normalize_requirements(agent_cfg),
                 project_dir=project_dir,
                 stub_entrypoints=stub_entrypoints,
-                requirements=_normalize_requirements(agent_cfg),
             )
 
         bake_targets.append(
@@ -357,12 +367,12 @@ def cmd_build(args):
         )
 
     # -------------------------------------------------------------- #
-    #  Step 5: Build all Docker images                                #
+    #  Step 4: Build all Docker images                                #
     # -------------------------------------------------------------- #
     if not bake_targets:
         logger.info("No Docker images to build.")
     elif _docker_available() and _docker_available(("docker", "buildx", "version")):
-        docker_container_dir = os.path.join(project_dir, "docker_container")
+        docker_container_dir = os.path.join(artifact_root, "docker_container")
         os.makedirs(docker_container_dir, exist_ok=True)
         bake_file_path = os.path.join(docker_container_dir, "docker-bake.json")
         _write_bake_file(bake_targets, bake_file_path, _docker_platform())
@@ -409,23 +419,23 @@ def cmd_deploy(args):
         sys.exit(1)
 
     config = _load_config(config_path)
-    project_dir = os.getcwd()
+    artifact_root = _artifact_root(config_path)
 
     # Fail here rather than after a fleet of containers is already up without
-    # the API keys they need.
+    # the API keys they need. env_file remains relative to the source root.
     try:
-        resolve_env_file(config, base_dir=project_dir)
+        resolve_env_file(config, base_dir=os.getcwd())
     except ValueError as e:
         logger.error("%s", e)
         sys.exit(1)
 
-    _ensure_grpc_stubs_importable(project_dir)
+    _ensure_grpc_stubs_importable(artifact_root)
 
     if any(
         agent.get("provider", "local").upper() == "EC2"
         for agent in config.get("agents", [])
     ):
-        _preflight_ec2_deploy(config, project_dir)
+        _preflight_ec2_deploy(config)
 
     from ventis.controller.global_controller import GlobalController
 
@@ -466,20 +476,15 @@ def cmd_clean(args):
     """
     Remove generated stubs, gRPC files, and Docker build contexts.
     """
-    project_dir = os.getcwd()
-
-    paths_to_clean = [
-        os.path.join(project_dir, "stubs"),
-        os.path.join(project_dir, "grpc_stubs"),
-        os.path.join(project_dir, "docker_container"),
-    ]
+    config_path = getattr(args, "config", DEFAULT_CONFIG_PATH)
+    artifact_root = _artifact_root(config_path)
+    generated_names = ("stubs", "grpc_stubs", "docker_container")
+    paths_to_clean = [os.path.join(artifact_root, name) for name in generated_names]
 
     for path in paths_to_clean:
         if os.path.exists(path):
             logger.info("Cleaning %s...", path)
             if os.path.isdir(path):
-                import shutil
-
                 shutil.rmtree(path)
             else:
                 os.remove(path)
@@ -538,6 +543,12 @@ def main():
         "clean",
         help="Remove generated stubs, compiled protos, and Docker contexts",
     )
+    clean.add_argument(
+        "-c",
+        "--config",
+        default=DEFAULT_CONFIG_PATH,
+        help=f"Select the artifact layout via its config (default: {DEFAULT_CONFIG_PATH})",
+    )
     clean.set_defaults(func=cmd_clean)
 
     args = parser.parse_args()

ventis/controller/global_controller.py

@@ -28,8 +28,6 @@
 from ventis.utils.redis_client import RedisClient
 from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS
 
-# Add generated grpc_stubs from the local project to the path
-sys.path.insert(0, os.path.abspath("grpc_stubs"))
 import local_controler_pb2
 import local_controler_pb2_grpc
 import grpc
@@ -783,9 +781,10 @@ def stop(self):
 
 
 if __name__ == "__main__":
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    project_root = os.path.join(script_dir, "..", "..")
-    default_config = os.path.join(project_root, "config", "global_controller.yaml")
+    project_root = os.getcwd()
+    default_config = os.path.join(
+        project_root, ".car", "config", "global_controller.yaml"
+    )
 
     import argparse
 
@@ -794,7 +793,7 @@ def stop(self):
         "-c",
         "--config",
         default=default_config,
-        help="Path to the YAML config file (default: config/global_controller.yaml)",
+        help="Path to the YAML config file (default: .car/config/global_controller.yaml)",
     )
     args = parser.parse_args()