From 67304842330e9331605e4fd7f7936904d9ca5b61 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Sun, 25 Oct 2026 00:52:49 -0700 Subject: [PATCH 1/2] [autoscaler] Strip login-shell profile output from captured SSH command output The autoscaler runs remote commands through a login shell (bash --login -i over ssh -tt). Profile scripts frequently print a MOTD, quota warnings or other banners, which end up in the captured stdout of commands whose output the autoscaler parses, e.g. the docker inspect JSON used during node setup. Bracket captured commands with start/end markers and return only the output between them, preserving the command's exit code. Commands that do not capture output are unchanged. Fixes #62045 Co-Authored-By: Claude Fable 5.1 Signed-off-by: lonexreb --- .../ray/autoscaler/_private/command_runner.py | 34 ++++++++++++++- python/ray/tests/test_command_runner.py | 43 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/python/ray/autoscaler/_private/command_runner.py b/python/ray/autoscaler/_private/command_runner.py index 7d27ac2bc9c3..cf638f11b18a 100644 --- a/python/ray/autoscaler/_private/command_runner.py +++ b/python/ray/autoscaler/_private/command_runner.py @@ -105,6 +105,34 @@ def _with_environment_variables(cmd: str, environment_variables: Dict[str, objec return all_vars + cmd +_OUTPUT_START_MARKER = "__RAY_COMMAND_OUTPUT_START__" +_OUTPUT_END_MARKER = "__RAY_COMMAND_OUTPUT_END__" + + +def _with_output_markers(cmd: str) -> str: + """Bracket the command's stdout with markers. + + Login shells run profile scripts that may print a MOTD, quota warnings or + similar banners before (and after) the command runs. Callers that parse the + captured output, e.g. ``docker inspect`` JSON, break on that extra text. + The markers let :func:`_strip_output_markers` recover just the command's + own output while preserving its exit code. + """ + return ( + f"echo {_OUTPUT_START_MARKER}; ({cmd}); __ray_status=$?; " + f"echo; echo {_OUTPUT_END_MARKER}; exit $__ray_status" + ) + + +def _strip_output_markers(output: bytes) -> bytes: + """Return the part of ``output`` between the markers, if both are present.""" + start = output.find(_OUTPUT_START_MARKER.encode()) + end = output.rfind(_OUTPUT_END_MARKER.encode()) + if start == -1 or end == -1 or end < start: + return output + return output[start + len(_OUTPUT_START_MARKER) : end].strip(b"\r\n") + + def _with_interactive(cmd): force_interactive = ( f"source ~/.bashrc; " @@ -304,7 +332,9 @@ def _run_helper( use_login_shells=is_using_login_shells(), ) else: - return self.process_runner.check_output(final_cmd) + return _strip_output_markers( + self.process_runner.check_output(final_cmd) + ) except subprocess.CalledProcessError as e: joined_cmd = " ".join(final_cmd) if not is_using_login_shells(): @@ -387,6 +417,8 @@ def run( if cmd: if environment_variables: cmd = _with_environment_variables(cmd, environment_variables) + if with_output: + cmd = _with_output_markers(cmd) if is_using_login_shells(): final_cmd += _with_interactive(cmd) else: diff --git a/python/ray/tests/test_command_runner.py b/python/ray/tests/test_command_runner.py index b3f94b271beb..39d0cde47e90 100644 --- a/python/ray/tests/test_command_runner.py +++ b/python/ray/tests/test_command_runner.py @@ -6,6 +6,9 @@ import pytest from ray.autoscaler._private.command_runner import ( + _OUTPUT_END_MARKER, + _OUTPUT_START_MARKER, + _strip_output_markers, DockerCommandRunner, SSHCommandRunner, _with_environment_variables, @@ -218,6 +221,46 @@ def fail_if_ip_is_requested(_): assert f"ray@{node_id}:/remote/source" in download_command +def test_ssh_command_runner_strips_profile_output(): + """Regression test for https://github.com/ray-project/ray/issues/62045. + + Login shells may print a MOTD or other banners around the command's own + output. Captured output must contain only what the command printed. + """ + process_runner = MockProcessRunner() + provider = MockProvider() + provider.create_node({}, {}, 1) + cmd_runner = SSHCommandRunner( + log_prefix="prefix", + node_id="0", + provider=provider, + auth_config=auth_config, + cluster_name="cluster", + process_runner=process_runner, + use_internal_ip=False, + ) + process_runner.respond_to_call( + "echo hello", + [ + "Welcome to the node, disk quota is 95% full\r\n" + f"{_OUTPUT_START_MARKER}\r\nhello\r\n\r\n{_OUTPUT_END_MARKER}\r\n" + "logout banner\r\n" + ], + ) + + assert cmd_runner.run("echo hello", with_output=True) == b"hello" + process_runner.assert_has_call("1.2.3.4", pattern=_OUTPUT_START_MARKER) + process_runner.assert_has_call("1.2.3.4", pattern="exit \\$__ray_status") + + +@pytest.mark.parametrize( + "output", + [b"plain output with no markers", b"only " + _OUTPUT_START_MARKER.encode()], +) +def test_strip_output_markers_passthrough(output): + assert _strip_output_markers(output) == output + + def test_docker_command_runner(): process_runner = MockProcessRunner() provider = MockProvider() From 3a42e3fdf22fd083492385c78298358b5f3964a9 Mon Sep 17 00:00:00 2001 From: lonexreb Date: Wed, 28 Oct 2026 01:00:39 -0700 Subject: [PATCH 2/2] Sort imports in test_command_runner Co-Authored-By: Claude Opus 5.5 Signed-off-by: lonexreb --- python/ray/tests/test_command_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_command_runner.py b/python/ray/tests/test_command_runner.py index 39d0cde47e90..c2a6ea37081c 100644 --- a/python/ray/tests/test_command_runner.py +++ b/python/ray/tests/test_command_runner.py @@ -8,9 +8,9 @@ from ray.autoscaler._private.command_runner import ( _OUTPUT_END_MARKER, _OUTPUT_START_MARKER, - _strip_output_markers, DockerCommandRunner, SSHCommandRunner, + _strip_output_markers, _with_environment_variables, ) from ray.autoscaler.command_runner import CommandRunnerInterface