From 4d28f95d8bb56f70fb2507642278975948ad448f Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 17:26:35 -0700 Subject: [PATCH 1/5] includes all the files when ventis build --- ventis/cli.py | 2 ++ ventis/stub_generator.py | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 3aceb18..9ffc149 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -274,6 +274,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + project_dir=project_dir, ) else: @@ -316,6 +317,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + project_dir=project_dir, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..a571ccc 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -263,8 +263,36 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# Directories ventis build itself generates inside a project -- never swept. +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} + + +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + swept = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + for fname in files: + if fname.endswith(".py"): + abs_src = os.path.join(root, fname) + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) + return swept + + +def _stub_destination(stub_file, project_dir): + """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" + basename = os.path.basename(stub_file) + return os.path.join("agents", basename) if project_dir else basename + + def generate_docker( - yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None + yaml_path, + agent_file, + output_dir=None, + grpc_stubs_dir=None, + stub_files=None, + project_dir=None, ): """ Generate a minimal Docker build context for an agent. @@ -278,6 +306,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -301,8 +330,13 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements) + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + # Copy general agent files - files_to_copy = [ + files_to_copy += [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -322,13 +356,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs + # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), os.path.basename(stub_file)) + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -339,7 +373,9 @@ def generate_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") @@ -377,7 +413,12 @@ def generate_docker( def generate_workflow_docker( - workflow_file, stub_files, output_dir=None, grpc_stubs_dir=None, api_port=8080 + workflow_file, + stub_files, + output_dir=None, + grpc_stubs_dir=None, + api_port=8080, + project_dir=None, ): """ Generate a Docker build context for a workflow. @@ -391,6 +432,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -417,7 +459,12 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - files_to_copy = [ + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + + files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -437,9 +484,11 @@ def generate_workflow_docker( ], ] - # Copy stub files + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + files_to_copy.append( + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -449,7 +498,9 @@ def generate_workflow_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") From 23e3928d01bc5996fd1cd4e086d45212c283de0e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 18:23:51 -0700 Subject: [PATCH 2/5] fixed some bugs --- ventis/cli.py | 19 ++++++++++++++ ventis/stub_generator.py | 54 +++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 9ffc149..cb9ee0a 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -220,6 +220,23 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) + # Map each stub's basename to its agent's declared entrypoint, so a stub + # overwrites the exact real file it replaces instead of guessing its path. + stub_entrypoints = {} + for agent_cfg in agents: + entrypoint = agent_cfg.get("entrypoint") + if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: + continue + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_cfg["name"]: + base_name = os.path.splitext(os.path.basename(yaml_path))[0] + stub_entrypoints[f"{base_name}.py"] = entrypoint + break + # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -275,6 +292,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) else: @@ -318,6 +336,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a571ccc..a56080d 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,7 +271,12 @@ def _sweep_py_files(project_dir): """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and not (root == project_dir and d in _GENERATED_DIRS) + ] for fname in files: if fname.endswith(".py"): abs_src = os.path.join(root, fname) @@ -280,10 +285,15 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, project_dir): - """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" +def _stub_destination(stub_file, stub_entrypoints): + """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" basename = os.path.basename(stub_file) - return os.path.join("agents", basename) if project_dir else basename + entrypoint = stub_entrypoints.get(basename) + if entrypoint: + normalized = entrypoint.replace("\\", "/") + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return entrypoint + return basename def generate_docker( @@ -293,6 +303,7 @@ def generate_docker( grpc_stubs_dir=None, stub_files=None, project_dir=None, + stub_entrypoints=None, ): """ Generate a minimal Docker build context for an agent. @@ -301,12 +312,13 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -360,7 +372,10 @@ def generate_docker( if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -419,6 +434,7 @@ def generate_workflow_docker( grpc_stubs_dir=None, api_port=8080, project_dir=None, + stub_entrypoints=None, ): """ Generate a Docker build context for a workflow. @@ -428,11 +444,12 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -487,7 +504,10 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) # Copy gRPC generated stubs if they exist From 95240ca941167a11c6da6cc791ca4a2fb13c7ebe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 19:17:55 -0700 Subject: [PATCH 3/5] ventis build: sweep project .py files into Docker build contexts generate_docker()/generate_workflow_docker() now recursively sweep every .py file under the project directory into the build context, preserving directory structure, so helper files that aren't declared as an agent entrypoint still make it into the image. Generated dirs (docker_container/, stubs/, grpc_stubs/) are excluded at the project root only, not at every depth. Stub files are placed at their agent's declared entrypoint path (mapped from global_controller.yaml) instead of a hardcoded guess, so a stub overwrites the exact real file it replaces. Guards against absolute and '..'-containing entrypoints, symlinked sources, and symlinked-destination escapes, with warnings on unsafe or unmapped stubs. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 45 +++++++++++++++--------------------- ventis/stub_generator.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index cb9ee0a..4c9badf 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -212,6 +212,23 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) + import yaml + + # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} + for yaml_path in yaml_files: + with open(yaml_path) as f: + name = yaml.safe_load(f).get("agent", {}).get("name") + if name: + yaml_by_name[name] = yaml_path + + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -220,23 +237,6 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) - # Map each stub's basename to its agent's declared entrypoint, so a stub - # overwrites the exact real file it replaces instead of guessing its path. - stub_entrypoints = {} - for agent_cfg in agents: - entrypoint = agent_cfg.get("entrypoint") - if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: - continue - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_cfg["name"]: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - stub_entrypoints[f"{base_name}.py"] = entrypoint - break - # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -310,16 +310,7 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = None - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_name: - matching_yaml = yaml_path - break - + matching_yaml = yaml_by_name.get(agent_name) if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a56080d..9ea6c99 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -278,24 +278,43 @@ def _sweep_py_files(project_dir): and not (root == project_dir and d in _GENERATED_DIRS) ] for fname in files: - if fname.endswith(".py"): - abs_src = os.path.join(root, fname) + abs_src = os.path.join(root, fname) + if fname.endswith(".py") and not os.path.islink(abs_src): rel_dst = os.path.relpath(abs_src, project_dir) swept.append((abs_src, rel_dst)) return swept def _stub_destination(stub_file, stub_entrypoints): - """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") if not normalized.startswith("/") and ".." not in normalized.split("/"): - return entrypoint + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + elif stub_entrypoints: + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") return basename +def _copy_files(output_dir, files_to_copy): + """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" + real_output_dir = os.path.realpath(output_dir) + for src, dst in files_to_copy: + if not os.path.isfile(src): + print(f" Warning: source file not found, skipping: {src}") + continue + dest_path = os.path.join(output_dir, dst) + real_dest = os.path.realpath(dest_path) + if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: + print(f" Warning: destination escapes build context, skipping: {dst}") + continue + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) + + def generate_docker( yaml_path, agent_file, @@ -386,13 +405,7 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # Copy the YAML definition too shutil.copy2( @@ -477,9 +490,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -500,7 +511,7 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( @@ -516,13 +527,7 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading From 69d5405c24485ff51db960ad7843e496d91354ce Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:06:31 -0700 Subject: [PATCH 4/5] Fix missing os import in metrics_agent.py Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json From 8692b0078ff21f2843b0875192f9d12980541c50 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 27 Aug 2026 17:30:53 -0700 Subject: [PATCH 5/5] Fixed Nicks issue --- examples/portfolio/workflow/portfolio_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..73e94d2 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,7 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() + intent = intent_agent.parse(query=query) # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below.