From ab7f1cb18acd4e1bb64d22533fe80671c856b14e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Sat, 29 Aug 2026 17:52:47 -0700 Subject: [PATCH] Compress EC2 docker image transfer with zstd; drop unused ipdb/ipython docker save previously piped a raw uncompressed tar over SSH for every EC2 agent bootstrap. Pipe it through zstd -T0 on the sender and zstd -d on the receiver instead, cutting transfer size/time. Also drops ipdb/ipython from BASE_AGENT_REQUIREMENTS/requirements.txt -- known-unused interactive-debugger tooling baked into every agent image. Co-Authored-By: Claude Sonnet 5 --- requirements.txt | 2 -- ventis/controller/cloud_provider_logic/EC2/_runtime.py | 5 +++-- ventis/stub_generator.py | 4 +--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index b0dd97e..d4f39f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..71109f1 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -238,11 +238,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 54a46b9..08c8273 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"]