From 7d8a2acfda8ec2f2a67f96df7ab75cd4a5062ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A1=82=E9=A9=AC?= Date: Wed, 16 Sep 2026 16:01:54 +0800 Subject: [PATCH 1/4] feat(skill): support StartChat SSE reconnect --- scripts/a2a/e2e/README.md | 72 ++ scripts/a2a/e2e/README.zh-CN.md | 56 + .../e2e/reconnect/aliyun_cli_mcp_server.py | 435 +++++++ scripts/a2a/e2e/reconnect/fake_aliyun_cli.py | 149 +++ .../e2e/reconnect/run_qoder_mcp_reconnect.py | 610 ++++++++++ skills/alicloud-ros-agent/SKILL.md | 13 +- skills/alicloud-ros-agent/SKILL.md.template | 13 +- .../scripts/_ros_agent_core.py | 206 +++- .../scripts/_ros_agent_projection.py | 57 +- .../scripts/_ros_agent_runtime.py | 635 ++++++++-- .../alicloud-ros-agent/scripts/ros_agent.py | 12 + tests/a2a_e2e/test_qoder_mcp_reconnect.py | 347 ++++++ tests/skill_bridge/start_chat_relay.py | 147 ++- .../test_alicloud_ros_agent_bridge.py | 1061 ++++++++++++++++- tests/skill_bridge/test_start_chat_relay.py | 189 ++- 15 files changed, 3806 insertions(+), 196 deletions(-) create mode 100755 scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py create mode 100755 scripts/a2a/e2e/reconnect/fake_aliyun_cli.py create mode 100755 scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py create mode 100644 tests/a2a_e2e/test_qoder_mcp_reconnect.py diff --git a/scripts/a2a/e2e/README.md b/scripts/a2a/e2e/README.md index 124e5c41f..bf8e3030e 100644 --- a/scripts/a2a/e2e/README.md +++ b/scripts/a2a/e2e/README.md @@ -107,6 +107,78 @@ shared serial checkpoint evidence, no Sub Pipeline checkpoint, native StartChat usage, exact resource cleanup, and retention of the pre-existing VPC inventory. +## Real Qoder/MCP StartChat reconnect matrix + +`reconnect/run_qoder_mcp_reconnect.py` drives the release-gated reconnect path: + +```text +Qoder Work -> installed Skill -> fake aliyun CLI -> stdio MCP + -> real aliyun CLI (--profile test-guima) -> ros-pre.aliyuncs.com +``` + +The runner backs up both Qoder Skill locations, installs the repository Skill, +sets `aliyun_cli`/`remote` and the pre-production endpoint in its `config.json`, +and patches only those temporary installed copies so their default `aliyun` +path is the fake CLI. It restores the complete previous installations in +`finally`. The fake CLI calls the MCP tool `alibabacloud___callcli`; the MCP +server validates the ROS-only command shape, forces the real CLI Profile, and +applies a 120-second timeout to each real CLI invocation. + +The first complete event is carried in the same MCP CallTool progress channel. +The fake CLI writes the invocation-scoped bootstrap envelope and does not let +the MCP server continue until the bridge has acknowledged the event after +projection and cursor persistence. A timed-out ordinary MCP result is mapped +to stable `ExecutorTimeout`; the bridge therefore reconnects only when that +committed SessionId/full-cursor anchor exists. The timeout message itself is +not parsed as recovery state. + +Run every scenario in a fresh directory: + +```bash +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-normal \ + --scenario normal + +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-first-timeout \ + --scenario first-call-timeout + +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-later-timeout \ + --scenario reconnect-call-timeout +``` + +`normal` uses a short no-tool response. Both timeout scenarios ask iac-code to +run `echo xxx1`, sleep 75 seconds, `echo xxx2`, sleep 75 seconds, then +`echo xxx3`. `first-call-timeout` lets invocation 1 reach the real 120-second +MCP timeout. `reconnect-call-timeout` closes invocation 1 immediately after +its bootstrap ack, lets invocation 2 reach the 120-second timeout, and expects +invocation 3 to continue from the newest committed cursor. The assertions +require exactly one Query, no pre-generated SessionId, no Query on reconnect, +one Session/stream identity, non-decreasing full-cursor sequence, and the +expected final output. + +The preflight is read-only and refuses to start Qoder or create `--run-dir` +unless ROS CLI plugin `0.9.1` or newer is installed, the `test-guima` Profile +exists, and the repository Python environment provides MCP. Reconnect accepts +no Query and carries StreamOptions in the global `--body` JSON rather than a +plugin-specific `--stream-options` flag. An older plugin is reported as a +readiness blocker instead of producing a misleading partial E2E run. +`mcp-scenario-state.json`, `qoder-turns.jsonl`, and `result.json` contain bounded +evidence; Query, SessionId, and full cursor values are represented only by +hashes, booleans, and cursor sequence numbers. + +The no-cloud protocol regression uses a fixture in place of the real CLI while +retaining the real stdio MCP client/server, progress notification, ack, and +timeout behavior: + +```bash +uv run pytest -q tests/a2a_e2e/test_qoder_mcp_reconnect.py +``` + The controlled Sub-Pipeline fixture uses a real `PipelineRunner` with two real `AgentLoop` candidates. One candidate parks at an actual permission Future while the other completes naturally; after the configured hard timeout, the parent diff --git a/scripts/a2a/e2e/README.zh-CN.md b/scripts/a2a/e2e/README.zh-CN.md index 4ec4da4df..fe32ea834 100644 --- a/scripts/a2a/e2e/README.zh-CN.md +++ b/scripts/a2a/e2e/README.zh-CN.md @@ -124,6 +124,62 @@ Normal 和 Pipeline 都要分别运行 grace、挂起后恢复和进程重启变 不会替用户批准。最终断言要求：真实非只读权限、Normal/顶层 Pipeline 本地与共享 checkpoint、Sub Pipeline 无 checkpoint、确实经过原生 StartChat、精确清理本次资源，以及全部原有 VPC 仍存在。 +## 真实 Qoder/MCP StartChat 断线重连矩阵 + +`reconnect/run_qoder_mcp_reconnect.py` 驱动发布闸门对应的真实链路： + +```text +Qoder Work -> 安装后的 Skill -> 假 aliyun CLI -> stdio MCP + -> 真 aliyun CLI（--profile test-guima）-> ros-pre.aliyuncs.com +``` + +Runner 会备份两个 Qoder Skill 目录，安装仓库内 Skill，在 `config.json` 中固定 +`aliyun_cli`/`remote` 与 ROS 预发 endpoint，并且只修改这两份临时安装副本，把默认 `aliyun` 路径指向 +假 CLI；`finally` 中完整恢复原安装。假 CLI 调用 MCP 工具 `alibabacloud___callcli`；MCP 服务只接受 +ROS StartChat/StopChat 命令，为真 CLI 强制使用 `test-guima` Profile，并对每次真 CLI 调用执行 120 秒超时。 + +首个完整事件通过同一次 MCP CallTool 的 progress 通道传递。假 CLI 输出 invocation-scoped bootstrap +envelope 后，必须等 bridge 在完成投影和 cursor 持久化后写入 ack，MCP 服务才继续。普通 MCP 结果超时后， +假 CLI 把它统一映射成稳定的 `ExecutorTimeout`；bridge 只有已经提交 SessionId/full cursor 锚点时才重连， +不会解析超时报错文本来猜恢复状态。 + +每个场景必须使用新的目录： + +```bash +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-normal \ + --scenario normal + +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-first-timeout \ + --scenario first-call-timeout + +uv run python scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py \ + --allow-real-cloud \ + --run-dir /tmp/iac-reconnect-later-timeout \ + --scenario reconnect-call-timeout +``` + +`normal` 使用一个不调用工具的短回复。两个超时场景都要求 iac-code 依次执行 `echo xxx1`、sleep 75 秒、 +`echo xxx2`、sleep 75 秒、`echo xxx3`。`first-call-timeout` 让第 1 次调用真实到达 MCP 的 120 秒超时； +`reconnect-call-timeout` 在第 1 次调用的 bootstrap ack 后立即断开，让第 2 次调用真实到达 120 秒超时， +并要求第 3 次调用从最近提交的 cursor 继续。最终断言包括：Query 恰好一次、首次调用不预生成 SessionId、 +Reconnect 不带 Query、Session/stream identity 一致、完整 cursor 序号不回退，以及最终输出完整。 + +预检是只读的：只有真 ROS CLI plugin 版本不低于 `0.9.1`、`test-guima` Profile 存在，并且仓库 Python +环境已安装 MCP，Runner 才会启动 Qoder 和创建 `--run-dir`。Reconnect 允许不带 Query，并通过全局 +`--body` JSON 传递 StreamOptions，不再要求 plugin 暴露专用 `--stream-options` flag。旧版本 plugin 会 +明确报告 readiness blocker，不会跑出一条误导性的半链路。`mcp-scenario-state.json`、`qoder-turns.jsonl` +和 `result.json` 保存有界证据；Query、SessionId 和完整 cursor 只保留 hash、布尔值和 cursor 序号。 + +无真实云回归会用 fixture 代替真 CLI，但仍经过真实 stdio MCP client/server、progress、ack 和 timeout： + +```bash +uv run pytest -q tests/a2a_e2e/test_qoder_mcp_reconnect.py +``` + 受控 Sub Pipeline fixture 使用真实 `PipelineRunner` 和两个真实 `AgentLoop` candidate：一个 candidate 停在真实权限 Future，另一个自然完成；到达配置的硬超时后，父 Pipeline 聚合两个 conclusion、进入 candidate 选择并自然完成。fixture 同时安装生产 A2A 备份 hook，证明 Sub 权限本身不会触发权限关键备份。验收运行使用 diff --git a/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py b/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py new file mode 100755 index 000000000..ed66fd655 --- /dev/null +++ b/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""stdio MCP server that runs the real aliyun CLI for reconnect E2E tests.""" + +from __future__ import annotations + +import asyncio +import codecs +import hashlib +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +from mcp.server.fastmcp import Context, FastMCP + +TOOL_NAME = "alibabacloud___callcli" +MAX_STDOUT_BYTES = (6 * 64 * 1024 * 1024) + (1024 * 1024) +MAX_STDERR_BYTES = 64 * 1024 +EXPECTED_ENDPOINT = "ros-pre.aliyuncs.com" +SUPPORTED_SCENARIOS = {"normal", "first-call-timeout", "reconnect-call-timeout"} + + +class _JSONValueStream: + """Incrementally decode concatenated JSON values or a top-level array.""" + + def __init__(self) -> None: + self._buffer = "" + self._array: bool | None = None + self._array_complete = False + self._decoder = json.JSONDecoder() + + def feed(self, text: str, *, final: bool = False) -> list[Any]: + self._buffer += text + values: list[Any] = [] + while True: + self._buffer = self._buffer.lstrip() + if not self._buffer: + break + if self._array is None: + self._array = self._buffer.startswith("[") + if self._array: + self._buffer = self._buffer[1:] + continue + if self._array: + self._buffer = self._buffer.lstrip() + if self._buffer.startswith("]"): + self._buffer = self._buffer[1:] + self._array_complete = True + break + if self._buffer.startswith(","): + self._buffer = self._buffer[1:] + continue + try: + value, end = self._decoder.raw_decode(self._buffer) + except ValueError: + break + values.append(value) + self._buffer = self._buffer[end:] + if self._array_complete: + break + if final: + if self._array and not self._array_complete: + raise ValueError("real aliyun CLI returned an incomplete JSON array") + if self._buffer.strip(): + raise ValueError("real aliyun CLI returned incomplete JSON") + return values + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def _load_state(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {"schemaVersion": 1, "requests": []} + if not isinstance(value, dict) or not isinstance(value.get("requests"), list): + raise RuntimeError("E2E scenario state is invalid") + return value + + +def _mutate_state(path: Path, mutation) -> Any: + state = _load_state(path) + result = mutation(state) + _atomic_json(path, state) + return result + + +def _start_request(path: Path, argv: list[str]) -> tuple[int, dict[str, Any]]: + initial = argv[:2] == ["ros", "start-chat"] + stop = argv[:2] == ["ros", "stop-chat"] + if not initial and not stop: + raise ValueError("the E2E MCP server accepts only ros start-chat and ros stop-chat") + + endpoint = _option_value(argv, "--endpoint") + if endpoint != EXPECTED_ENDPOINT: + raise ValueError("the E2E MCP server requires endpoint {}".format(EXPECTED_ENDPOINT)) + if "--profile" in argv: + raise ValueError("the remote bridge must not pass a local aliyun CLI Profile") + if "--stream-options" in argv: + raise ValueError("reconnect StreamOptions must be sent in --body") + + def add(state: dict[str, Any]) -> tuple[int, dict[str, Any]]: + requests = state["requests"] + invocation = 0 + if initial: + invocation = 1 + sum(item.get("operation") == "start-chat" for item in requests if isinstance(item, dict)) + query = _option_value(argv, "--query") + session_id = _option_value(argv, "--session-id") + body = _body_object(argv) if initial else {} + stream_keys = {"StreamOptions.Action", "StreamOptions.Cursor"} + if set(body) - stream_keys or not all(isinstance(body.get(key), str) for key in set(body) & stream_keys): + raise ValueError("the reconnect StreamOptions body is invalid") + stream = { + key.removeprefix("StreamOptions."): value + for key, value in body.items() + if key in stream_keys + } + cursor = stream.get("Cursor") + cursor_identity = None + cursor_sequence = None + if cursor: + identity, separator, sequence = cursor.rpartition(".") + if separator and identity and sequence.isdigit(): + cursor_identity = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + cursor_sequence = int(sequence) + record: dict[str, Any] = { + "operation": "start-chat" if initial else "stop-chat", + "invocation": invocation, + "hasQuery": query is not None, + "queryHash": hashlib.sha256(query.encode("utf-8")).hexdigest() if query is not None else None, + "hasSessionId": session_id is not None, + "sessionHash": ( + hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:16] if session_id is not None else None + ), + "hasCursor": cursor is not None, + "streamOptionsInBody": bool(set(body) & stream_keys), + "cursorIdentityHash": cursor_identity, + "cursorSequence": cursor_sequence, + "action": stream.get("Action"), + "endpointVerified": endpoint == EXPECTED_ENDPOINT, + "profileApplied": False, + "bootstrapDelivered": False, + "bootstrapAcked": False, + "outcome": "running", + } + requests.append(record) + return len(requests) - 1, record + + return _mutate_state(path, add) + + +def _update_request(path: Path, index: int, **updates: Any) -> None: + def update(state: dict[str, Any]) -> None: + request = state["requests"][index] + if not isinstance(request, dict): + raise RuntimeError("E2E request state is invalid") + request.update(updates) + + _mutate_state(path, update) + + +def _valid_start_shape(request: dict[str, Any]) -> bool: + invocation = request.get("invocation") + if invocation == 1: + return ( + request.get("hasQuery") is True + and request.get("hasSessionId") is False + and request.get("hasCursor") is False + and request.get("streamOptionsInBody") is False + and request.get("action") is None + ) + return ( + isinstance(invocation, int) + and invocation > 1 + and request.get("hasQuery") is False + and request.get("hasSessionId") is True + and request.get("hasCursor") is True + and request.get("streamOptionsInBody") is True + and request.get("action") == "Reconnect" + and isinstance(request.get("cursorIdentityHash"), str) + and isinstance(request.get("cursorSequence"), int) + ) + + +def _option_value(argv: list[str], option: str) -> str | None: + try: + index = argv.index(option) + except ValueError: + return None + if index + 1 >= len(argv) or argv[index + 1].startswith("--"): + return None + return argv[index + 1] + + +def _body_object(argv: list[str]) -> dict[str, Any]: + if argv.count("--body") > 1: + raise ValueError("the aliyun CLI invocation must contain at most one --body") + raw = _option_value(argv, "--body") + if raw is None: + return {} + try: + value = json.loads(raw) + except ValueError as exc: + raise ValueError("the aliyun CLI --body value is not valid JSON") from exc + if not isinstance(value, dict): + raise ValueError("the aliyun CLI --body value must be a JSON object") + return value + + +def _find_first(value: Any, *keys: str) -> Any: + if isinstance(value, dict): + for key in keys: + if value.get(key) not in (None, ""): + return value[key] + for child in value.values(): + found = _find_first(child, *keys) + if found not in (None, ""): + return found + elif isinstance(value, list): + for child in value: + found = _find_first(child, *keys) + if found not in (None, ""): + return found + return None + + +def _envelopes(value: Any): + if isinstance(value, list): + for item in value: + yield from _envelopes(item) + elif isinstance(value, dict) and isinstance(value.get("id"), str) and isinstance(value.get("data"), dict): + yield value + + +def _error_result(code: str, message: str) -> str: + return json.dumps({"ok": False, "code": code, "message": message}, ensure_ascii=False, separators=(",", ":")) + + +async def _terminate(process: asyncio.subprocess.Process) -> None: + if process.returncode is not None: + return + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=5) + except asyncio.TimeoutError: + process.kill() + await process.wait() + + +async def _call_real_cli(argv: list[str], ctx: Context) -> str: + real_aliyun = os.environ["IAC_CODE_E2E_REAL_ALIYUN"] + profile = os.environ.get("IAC_CODE_E2E_CLI_IDENTITY", "test-guima") + timeout_seconds = float(os.environ.get("IAC_CODE_E2E_MCP_TIMEOUT_SECONDS", "120")) + scenario = os.environ.get("IAC_CODE_E2E_SCENARIO", "normal") + state_path = Path(os.environ["IAC_CODE_E2E_SCENARIO_STATE"]).expanduser().resolve() + if scenario not in SUPPORTED_SCENARIOS: + return _error_result("InvalidE2EScenario", "the E2E reconnect scenario is invalid") + if timeout_seconds <= 0: + return _error_result("InvalidE2ETimeout", "the E2E MCP timeout must be positive") + + try: + state_index, request = _start_request(state_path, argv) + except (RuntimeError, ValueError) as exc: + return _error_result("InvalidCLIInvocation", str(exc)) + if request["operation"] == "start-chat" and not _valid_start_shape(request): + _update_request(state_path, state_index, outcome="invalid-invocation") + return _error_result( + "InvalidCLIInvocation", + "the initial or reconnect StartChat command shape violated the E2E contract", + ) + + command = [real_aliyun, *argv, "--profile", profile] + started = time.monotonic() + try: + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except OSError as exc: + _update_request( + state_path, + state_index, + outcome="real-cli-start-error", + elapsedSeconds=round(time.monotonic() - started, 3), + ) + return _error_result("RealAliyunCLIStartFailed", str(exc)[:1000]) + _update_request(state_path, state_index, profileApplied=True) + assert process.stdout is not None and process.stderr is not None + stdout = bytearray() + stderr = bytearray() + parser = _JSONValueStream() + utf8_decoder = codecs.getincrementaldecoder("utf-8")("replace") + bootstrap_sent = False + force_disconnect = asyncio.Event() + + async def read_stdout() -> None: + nonlocal bootstrap_sent + while True: + chunk = await process.stdout.read(64 * 1024) + if not chunk: + break + stdout.extend(chunk) + if len(stdout) > MAX_STDOUT_BYTES: + raise RuntimeError("real aliyun CLI output exceeded the E2E wrapper limit") + if bootstrap_sent or request["operation"] != "start-chat": + continue + for value in parser.feed(utf8_decoder.decode(chunk)): + for envelope in _envelopes(value): + bootstrap_sent = True + session_id = _find_first(envelope["data"], "contextId", "context_id", "SessionId") + progress = {"kind": "bootstrap", "event": envelope} + if isinstance(session_id, str) and session_id: + progress["sessionId"] = session_id + _update_request(state_path, state_index, bootstrapDelivered=True) + await ctx.report_progress( + 1, + 1, + json.dumps(progress, ensure_ascii=False, separators=(",", ":")), + ) + _update_request(state_path, state_index, bootstrapAcked=True) + if scenario == "reconnect-call-timeout" and request["invocation"] == 1: + force_disconnect.set() + break + if bootstrap_sent: + break + if not bootstrap_sent: + parser.feed(utf8_decoder.decode(b"", final=True), final=True) + + async def read_stderr() -> None: + while True: + chunk = await process.stderr.read(16 * 1024) + if not chunk: + break + remaining = MAX_STDERR_BYTES - len(stderr) + if remaining > 0: + stderr.extend(chunk[:remaining]) + + stdout_task = asyncio.create_task(read_stdout()) + stderr_task = asyncio.create_task(read_stderr()) + process_task = asyncio.create_task(process.wait()) + disconnect_task = asyncio.create_task(force_disconnect.wait()) + outcome = "failed" + try: + done, _pending = await asyncio.wait( + {process_task, disconnect_task}, + timeout=timeout_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + if disconnect_task in done and force_disconnect.is_set(): + await _terminate(process) + outcome = "connection-reset" + return _error_result( + "ConnectionReset", + "E2E injected a transport disconnect after the first committed bootstrap event.", + ) + if process_task not in done: + await _terminate(process) + outcome = "timeout" + return _error_result( + "ExecutorTimeout", + ( + "Aliyun MCP CORE CallTool tool.mcp.aliyun.core.{}: " + "timeout waiting for SSE response after {:.3f}s" + ).format( + TOOL_NAME, + timeout_seconds, + ), + ) + await asyncio.gather(stdout_task, stderr_task) + return_code = process_task.result() + if return_code != 0: + outcome = "real-cli-error" + detail = stderr.decode("utf-8", "replace").strip()[:3000] + _update_request( + state_path, + state_index, + realCLIReturnCode=return_code, + realCLIError=detail or "the real aliyun CLI failed", + ) + return _error_result("RealAliyunCLIFailed", detail or "the real aliyun CLI failed") + outcome = "success" + return json.dumps( + { + "ok": True, + "returnCode": 0, + "stdout": stdout.decode("utf-8", "replace"), + }, + ensure_ascii=False, + separators=(",", ":"), + ) + except (OSError, RuntimeError, ValueError) as exc: + await _terminate(process) + outcome = "mcp-error" + return _error_result("MCPExecutorFailed", str(exc)[:1000]) + finally: + disconnect_task.cancel() + if not stdout_task.done(): + stdout_task.cancel() + if not stderr_task.done(): + stderr_task.cancel() + await asyncio.gather(stdout_task, stderr_task, disconnect_task, return_exceptions=True) + _update_request( + state_path, + state_index, + outcome=outcome, + elapsedSeconds=round(time.monotonic() - started, 3), + ) + + +server = FastMCP("aliyun-cli-reconnect-e2e", log_level="ERROR") + + +@server.tool(name=TOOL_NAME) +async def alibabacloud_callcli(argv: list[str], ctx: Context) -> str: + """Run one validated ROS CLI operation with the E2E Profile and timeout.""" + + return await _call_real_cli(argv, ctx) + + +def main() -> int: + server.run(transport="stdio") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py b/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py new file mode 100755 index 000000000..686526c0b --- /dev/null +++ b/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Fake aliyun executable that routes validated CLI calls through stdio MCP.""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from datetime import timedelta +from pathlib import Path +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +TOOL_NAME = "alibabacloud___callcli" +BOOTSTRAP_CAPABILITY = "startchat-reconnect-bootstrap-v1" + + +def _write_error(code: str, message: str) -> int: + print(json.dumps({"code": code, "message": message}, ensure_ascii=False, separators=(",", ":")), file=sys.stderr) + return 1 + + +def _load_ack(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, ValueError): + return None + return value if isinstance(value, dict) else None + + +async def _wait_for_ack(path: Path, invocation_id: str, event_id: str, timeout: float) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + ack = _load_ack(path) + if ( + isinstance(ack, dict) + and ack.get("invocationId") == invocation_id + and ack.get("eventId") == event_id + and ack.get("committed") is True + ): + return + await asyncio.sleep(0.02) + raise TimeoutError("the ROS Agent bridge did not commit the MCP bootstrap event") + + +def _result_text(result: Any) -> str: + pieces = [] + for item in getattr(result, "content", []): + if getattr(item, "type", None) == "text" and isinstance(getattr(item, "text", None), str): + pieces.append(item.text) + return "".join(pieces) + + +async def _call_mcp(argv: list[str]) -> dict[str, Any]: + server_path = Path(os.environ["IAC_CODE_E2E_MCP_SERVER"]).expanduser().resolve() + python = os.environ.get("IAC_CODE_E2E_PYTHON") or sys.executable + mcp_timeout = float(os.environ.get("IAC_CODE_E2E_MCP_TIMEOUT_SECONDS", "120")) + ack_timeout = float(os.environ.get("IAC_CODE_E2E_BOOTSTRAP_ACK_TIMEOUT_SECONDS", "15")) + invocation_id = os.environ.get("ALICLOUD_ROS_AGENT_INVOCATION_ID", "") + ack_file = os.environ.get("ALICLOUD_ROS_AGENT_BOOTSTRAP_ACK_FILE", "") + protocol = os.environ.get("ALICLOUD_ROS_AGENT_BOOTSTRAP_PROTOCOL", "") + bootstrap_forwarded = False + + async def progress_callback(_progress: float, _total: float | None, message: str | None) -> None: + nonlocal bootstrap_forwarded + if bootstrap_forwarded or not message: + return + try: + progress = json.loads(message) + except ValueError: + return + if not isinstance(progress, dict): + return + event = progress.get("event") + event_id = event.get("id") if isinstance(event, dict) else None + event_data = event.get("data") if isinstance(event, dict) else None + if ( + progress.get("kind") != "bootstrap" + or not isinstance(event_id, str) + or not event_id + or not isinstance(event_data, dict) + ): + return + if protocol != BOOTSTRAP_CAPABILITY or not invocation_id or not ack_file: + raise RuntimeError("the bridge did not provide the bootstrap/ack contract") + envelope: dict[str, Any] = {"invocationId": invocation_id, "event": event} + session_id = progress.get("sessionId") + if isinstance(session_id, str) and session_id: + envelope["sessionId"] = session_id + print(json.dumps(envelope, ensure_ascii=False, separators=(",", ":")), flush=True) + await _wait_for_ack(Path(ack_file), invocation_id, event_id, ack_timeout) + bootstrap_forwarded = True + + log_path = Path(os.environ["IAC_CODE_E2E_MCP_STDERR_LOG"]).expanduser().resolve() + log_path.parent.mkdir(parents=True, exist_ok=True) + parameters = StdioServerParameters( + command=python, + args=[str(server_path)], + env=dict(os.environ), + cwd=str(server_path.parent), + ) + with log_path.open("a", encoding="utf-8") as errlog: + async with stdio_client(parameters, errlog=errlog) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool( + TOOL_NAME, + {"argv": argv}, + read_timeout_seconds=timedelta(seconds=mcp_timeout + ack_timeout + 15), + progress_callback=progress_callback, + ) + text = _result_text(result) + try: + value = json.loads(text) + except ValueError as exc: + raise RuntimeError("the E2E MCP server returned invalid JSON") from exc + if not isinstance(value, dict): + raise RuntimeError("the E2E MCP server returned an invalid result") + return value + + +async def _main(argv: list[str]) -> int: + if argv[:2] not in (["ros", "start-chat"], ["ros", "stop-chat"]): + return _write_error("UnsupportedCommand", "the fake aliyun CLI accepts only ROS StartChat and StopChat") + try: + result = await _call_mcp(argv) + except TimeoutError as exc: + return _write_error("BootstrapAckTimeout", str(exc)) + except Exception as exc: + return _write_error("MCPCallFailed", str(exc)[:1000]) + if result.get("ok") is not True: + return _write_error(str(result.get("code") or "MCPCallFailed"), str(result.get("message") or "MCP call failed")) + stdout = result.get("stdout") + if not isinstance(stdout, str): + return _write_error("MCPCallFailed", "the E2E MCP result did not contain CLI stdout") + sys.stdout.write(stdout) + sys.stdout.flush() + return int(result.get("returnCode") or 0) + + +def main() -> int: + return asyncio.run(_main(sys.argv[1:])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py b/scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py new file mode 100755 index 000000000..1cd22107b --- /dev/null +++ b/scripts/a2a/e2e/reconnect/run_qoder_mcp_reconnect.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +"""Credential-gated Qoder Work -> fake CLI -> MCP -> real CLI -> ROS pre E2E.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ENDPOINT = "ros-pre.aliyuncs.com" +PROFILE = "test-guima" +MIN_ROS_PLUGIN_VERSION = (0, 9, 1) +BOOTSTRAP_CAPABILITY = "startchat-reconnect-bootstrap-v1" +MCP_TIMEOUT_SECONDS = 120.0 +TERMINAL_STATES = {"turn-completed", "completed"} +FAILURE_STATES = {"failed", "canceled", "rejected"} +SCENARIOS = ("normal", "first-call-timeout", "reconnect-call-timeout") +FORWARDED_ENV = ( + "IAC_CODE_E2E_MCP_SERVER", + "IAC_CODE_E2E_REAL_ALIYUN", + "IAC_CODE_E2E_CLI_IDENTITY", + "IAC_CODE_E2E_MCP_TIMEOUT_SECONDS", + "IAC_CODE_E2E_SCENARIO", + "IAC_CODE_E2E_SCENARIO_STATE", + "IAC_CODE_E2E_MCP_STDERR_LOG", + "IAC_CODE_E2E_PYTHON", + "IAC_CODE_E2E_BOOTSTRAP_ACK_TIMEOUT_SECONDS", +) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--allow-real-cloud", action="store_true") + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--scenario", choices=SCENARIOS, default="normal") + parser.add_argument("--long-sleep-seconds", type=int, default=75) + parser.add_argument("--max-qoder-turns", type=int, default=12) + parser.add_argument("--qoder-turn-timeout", type=float, default=300.0) + parser.add_argument( + "--qoder-cli", + type=Path, + default=Path("/Applications/QoderWork.app/Contents/Resources/bin/qodercli"), + ) + parser.add_argument("--qoder-config-dir", type=Path, default=Path("~/.qoderwork")) + parser.add_argument("--real-aliyun", type=Path) + parser.add_argument("--skill-root", type=Path, action="append", default=None) + args = parser.parse_args() + if args.skill_root is None: + args.skill_root = [Path("~/.qoder/skills"), Path("~/.qoderwork/skills")] + return args + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def _plugin_version(value: Any) -> tuple[int, int, int] | None: + if not isinstance(value, str): + return None + match = re.fullmatch(r"([0-9]+)\.([0-9]+)\.([0-9]+)", value) + if match is None: + return None + return tuple(int(part) for part in match.groups()) + + +def _append_jsonl(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(value, ensure_ascii=False, separators=(",", ":")) + "\n") + + +def _manager_root(run_id: str) -> Path: + return Path(tempfile.gettempdir()).resolve() / "iac-code-reconnect-e2e" / run_id + + +@dataclass(frozen=True) +class _SkillBackup: + destination: Path + backup_root: Path + existed: bool + + +def _remove_path(path: Path) -> None: + immutable = getattr(stat, "UF_IMMUTABLE", 0) + chflags = getattr(os, "chflags", None) + if immutable and chflags is not None and (path.exists() or path.is_symlink()): + targets = [path] + if path.is_dir() and not path.is_symlink(): + targets.extend(path.rglob("*")) + for target in targets: + try: + flags = target.lstat().st_flags + except FileNotFoundError: + continue + if flags & immutable: + chflags(target, flags & ~immutable, follow_symlinks=False) + if path.is_symlink() or path.is_file(): + path.unlink(missing_ok=True) + elif path.is_dir(): + shutil.rmtree(path) + + +def _restore_skills(backups: list[_SkillBackup]) -> None: + errors: list[tuple[_SkillBackup, OSError]] = [] + for backup in reversed(backups): + restored = False + try: + _remove_path(backup.destination) + saved = backup.backup_root / "skill" + if backup.existed: + shutil.copytree(saved, backup.destination, symlinks=True) + restored = True + except OSError as exc: + errors.append((backup, exc)) + finally: + if restored: + _remove_path(backup.backup_root) + if errors: + failed, cause = errors[0] + raise RuntimeError( + "failed to restore Qoder Skill {}; backup retained at {}".format( + failed.destination, + failed.backup_root, + ) + ) from cause + + +def _install_patched_skill(repo_root: Path, roots: list[Path], fake_cli: Path) -> list[_SkillBackup]: + source = repo_root / "skills" / "alicloud-ros-agent" + backups: list[_SkillBackup] = [] + try: + destinations = [] + for raw_root in roots: + destination = raw_root.expanduser().resolve() / "alicloud-ros-agent" + existed = destination.exists() or destination.is_symlink() + if existed and (destination.is_symlink() or not destination.is_dir()): + raise RuntimeError("Qoder Skill destination must be a directory: {}".format(destination)) + backup_root = Path(tempfile.mkdtemp(prefix="iac-code-reconnect-skill-backup-")) + if existed: + shutil.copytree(destination, backup_root / "skill", symlinks=True) + backups.append(_SkillBackup(destination, backup_root, existed)) + destinations.append(destination) + + default_expression = json.dumps(str(fake_cli.resolve())) + needle = 'add_argument("--aliyun-path", default="aliyun")' + replacement = 'add_argument("--aliyun-path", default={})'.format(default_expression) + config = { + "transport": "aliyun_cli", + "aliyunCLIExecutionMode": "remote", + "endpoint": ENDPOINT, + "allowedAgentModes": ["normal"], + "enableThinking": True, + "managerIdleSeconds": 3, + "aliyunCLIForwardEnv": list(FORWARDED_ENV), + } + for destination in destinations: + _remove_path(destination) + shutil.copytree(source, destination, symlinks=True) + runtime = destination / "scripts" / "_ros_agent_runtime.py" + text = runtime.read_text(encoding="utf-8") + if text.count(needle) != 2: + raise RuntimeError("the installed Skill no longer has the two expected aliyun-path defaults") + runtime.write_text(text.replace(needle, replacement), encoding="utf-8") + _write_json(destination / "config.json", config) + except BaseException: + _restore_skills(backups) + raise + return backups + + +def _plugin_manifest() -> dict[str, Any]: + root = Path(os.environ.get("ALIBABA_CLOUD_CLI_PLUGIN_DIR", "~/.aliyun/plugins")).expanduser().resolve() + path = root / "manifest.json" + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RuntimeError("the aliyun CLI plugin manifest is unavailable or invalid") from exc + plugins = value.get("plugins") if isinstance(value, dict) else None + plugin = plugins.get("aliyun-cli-ros") if isinstance(plugins, dict) else None + if not isinstance(plugin, dict): + raise RuntimeError("the ROS aliyun CLI plugin is not installed") + return plugin + + +def _preflight(args: argparse.Namespace, repo_root: Path) -> tuple[Path, dict[str, Any]]: + if not args.allow_real_cloud: + raise SystemExit("Refusing to run Qoder/LLM/real-cloud E2E without --allow-real-cloud") + if args.long_sleep_seconds < 61: + raise SystemExit("--long-sleep-seconds must be at least 61 so the two sleeps exceed the 120-second MCP limit") + if args.max_qoder_turns <= 0 or args.qoder_turn_timeout <= 0: + raise SystemExit("Qoder turn limits must be positive") + + qoder_cli = args.qoder_cli.expanduser().resolve() + if not qoder_cli.is_file(): + raise RuntimeError("Qoder Work CLI is unavailable") + qoder_config = args.qoder_config_dir.expanduser().resolve() + if not qoder_config.is_dir(): + raise RuntimeError("Qoder Work config directory is unavailable") + real_aliyun = args.real_aliyun.expanduser().resolve() if args.real_aliyun else None + if real_aliyun is None: + discovered = shutil.which("aliyun") + real_aliyun = Path(discovered).resolve() if discovered else None + if real_aliyun is None or not real_aliyun.is_file(): + raise RuntimeError("the real aliyun CLI is unavailable") + + profile_check = subprocess.run( + [str(real_aliyun), "configure", "get", "--profile", PROFILE], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=15, + ) + if profile_check.returncode != 0: + raise RuntimeError("the required aliyun CLI Profile {} is unavailable".format(PROFILE)) + + manifest = _plugin_manifest() + version = manifest.get("version") + parsed_version = _plugin_version(version) + if parsed_version is None or parsed_version < MIN_ROS_PLUGIN_VERSION: + raise RuntimeError( + "ROS CLI plugin {} is not reconnect-ready: version 0.9.1 or newer is required".format( + version if isinstance(version, str) else "unknown" + ) + ) + + dependency_check = subprocess.run( + [sys.executable, "-c", "import mcp"], + cwd=repo_root, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + ) + if dependency_check.returncode != 0: + raise RuntimeError("the repository Python environment does not provide the mcp package") + return real_aliyun, { + "qoderAvailable": True, + "profileAvailable": True, + "rosPluginVersion": version, + "rosPluginReconnectReady": True, + "rosPluginStreamOptionsBody": True, + } + + +def _task_prompt(scenario: str, sleep_seconds: int, marker: str) -> str: + if scenario == "normal": + return ( + "这是 StartChat 断线重连 E2E 的正常场景。不要调用任何工具，只回复唯一文本 {}，不要添加其他内容。" + ).format(marker) + return ( + "这是 StartChat 断线重连 E2E 的长任务场景。只使用 bash 工具执行下面这一条命令，并等待它完成；" + "不要调用云 API，不要请求确认，不要改变命令：\n" + "echo xxx1; sleep {sleep}; echo xxx2; sleep {sleep}; echo xxx3\n" + "命令完成后，回复包含 xxx1、xxx2、xxx3 的简短结果。" + ).format(sleep=sleep_seconds) + + +def _jobs(state_root: Path) -> list[tuple[Path, dict[str, Any]]]: + results = [] + for path in sorted((state_root / "jobs").glob("*/job.json")): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(value, dict): + results.append((path, value)) + return results + + +def _bridge_subcommands(block: dict[str, Any]) -> tuple[str, list[str]]: + tool_input = block.get("input") + if not isinstance(tool_input, dict): + return "", [] + command = tool_input.get("command") + if not isinstance(command, str): + return "", [] + subcommands = re.findall(r"ros_agent\.py[\"']?\s+(check|start|follow)\b", command) + return command, subcommands + + +def _qoder_turn( + *, + args: argparse.Namespace, + env: dict[str, str], + workspace: Path, + state_root: Path, + session_id: str, + prompt: str, + turn: int, + run_dir: Path, +) -> dict[str, Any]: + driver_policy = ( + "You are driving one bounded alicloud-ros-agent reconnect E2E. Use the installed Skill. Prefix every bridge " + "command with ALICLOUD_ROS_AGENT_STATE_DIR={state}. Run check exactly once before managed start. Run managed " + "start exactly once with --prompt-file task-prompt.txt --mode normal; do not pass --endpoint, --profile, " + "--aliyun-path, or --follow. Once a job exists, never start another job or resend its prompt; only follow " + "that job from the newest public integer cursor until it reaches a boundary. Do not execute the task " + "locally and do not replace the remote Skill." + ).format(state=state_root) + command = [ + str(args.qoder_cli.expanduser().resolve()), + "-p", + "--output-format", + "stream-json", + "--config-dir", + str(args.qoder_config_dir.expanduser().resolve()), + "--dangerously-skip-permissions", + "--append-system-prompt", + driver_policy, + "--cwd", + str(workspace), + ] + if turn: + command.extend(["--resume", session_id]) + else: + command.extend(["--session-id", session_id]) + command.append(prompt) + started = time.monotonic() + completed = subprocess.run( + command, + cwd=workspace, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=args.qoder_turn_timeout, + ) + bridge_commands = [] + bridge_subcommands = [] + result_codes: set[str] = set() + for line in completed.stdout.splitlines(): + try: + item = json.loads(line) + except ValueError: + continue + message = item.get("message") if isinstance(item, dict) else None + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + serialized = json.dumps(block, ensure_ascii=False) + if block.get("type") == "tool_use": + command, subcommands = _bridge_subcommands(block) + if subcommands: + bridge_commands.append(command) + bridge_subcommands.extend(subcommands) + if block.get("type") == "tool_result": + result_codes.update(re.findall(r'"code"\s*:\s*"([A-Za-z0-9_.-]{1,80})"', serialized)) + evidence = { + "turn": turn, + "returnCode": completed.returncode, + "elapsedSeconds": round(time.monotonic() - started, 3), + "stdoutBytes": len(completed.stdout.encode("utf-8")), + "stderrBytes": len(completed.stderr.encode("utf-8")), + "bridgeCommandCount": len(bridge_subcommands), + "bridgeCheck": bridge_subcommands.count("check"), + "bridgeStart": bridge_subcommands.count("start"), + "bridgeFollow": bridge_subcommands.count("follow"), + "passedAliyunPath": any("--aliyun-path" in value for value in bridge_commands), + "resultCodes": sorted(result_codes), + } + _append_jsonl(run_dir / "qoder-turns.jsonl", evidence) + if completed.returncode != 0: + raise RuntimeError("Qoder turn {} failed; see qoder-turns.jsonl".format(turn)) + if evidence["passedAliyunPath"]: + raise RuntimeError("Qoder bypassed the temporary installed-Skill fake CLI default") + return evidence + + +def _scenario_checks(scenario: str, state: dict[str, Any], final_text: str, marker: str) -> dict[str, bool]: + requests = [item for item in state.get("requests", []) if isinstance(item, dict)] + starts = [item for item in requests if item.get("operation") == "start-chat"] + initial = starts[:1] + reconnects = starts[1:] + query_requests = [item for item in starts if item.get("hasQuery") is True] + session_hashes = {item.get("sessionHash") for item in reconnects if item.get("sessionHash")} + cursor_identities = {item.get("cursorIdentityHash") for item in reconnects if item.get("cursorIdentityHash")} + cursor_sequences = [item.get("cursorSequence") for item in reconnects] + timeout_invocations = [item.get("invocation") for item in starts if item.get("outcome") == "timeout"] + common = { + "one initial Query was sent": len(query_requests) == 1 and bool(initial) and initial[0].get("hasQuery") is True, + "initial request did not pre-generate SessionId or cursor": bool(initial) + and initial[0].get("hasSessionId") is False + and initial[0].get("hasCursor") is False + and initial[0].get("streamOptionsInBody") is False, + "all reconnects omitted Query": all(item.get("hasQuery") is False for item in reconnects), + "all reconnects used SessionId, Reconnect, and full cursor": all( + item.get("hasSessionId") is True + and item.get("hasCursor") is True + and item.get("streamOptionsInBody") is True + and item.get("action") == "Reconnect" + and isinstance(item.get("cursorSequence"), int) + for item in reconnects + ), + "reconnect stayed in one Session and stream": len(session_hashes) <= 1 and len(cursor_identities) <= 1, + "reconnect cursor never moved backwards": cursor_sequences == sorted(cursor_sequences), + "MCP applied test-guima and ros-pre to every call": bool(requests) + and all(item.get("profileApplied") is True and item.get("endpointVerified") is True for item in requests), + "last StartChat invocation completed": bool(starts) and starts[-1].get("outcome") == "success", + "expected task output completed": ( + marker in final_text + if scenario == "normal" + else all(value in final_text for value in ("xxx1", "xxx2", "xxx3")) + ), + } + if scenario == "normal": + common.update( + { + "normal path had no forced timeout or disconnect": bool(initial) + and initial[0].get("outcome") == "success" + and not reconnects + and not timeout_invocations, + } + ) + elif scenario == "first-call-timeout": + common.update( + { + "first call timed out only after bootstrap ack": bool(initial) + and initial[0].get("outcome") == "timeout" + and initial[0].get("bootstrapAcked") is True, + "first-call timeout reconnected": bool(reconnects), + "only the first call timed out": timeout_invocations == [1], + } + ) + else: + common.update( + { + "first call disconnected only after bootstrap ack": bool(initial) + and initial[0].get("outcome") == "connection-reset" + and initial[0].get("bootstrapAcked") is True, + "the non-initial call timed out": len(starts) >= 3 and starts[1].get("outcome") == "timeout", + "only the second call timed out": timeout_invocations == [2], + "later timeout continued from the latest anchor": len(starts) >= 3 + and starts[2].get("hasQuery") is False + and isinstance(starts[1].get("cursorSequence"), int) + and isinstance(starts[2].get("cursorSequence"), int) + and starts[2].get("cursorSequence") >= starts[1].get("cursorSequence"), + } + ) + return common + + +def _cancel_nonterminal_job(repo_root: Path, state_root: Path, env: dict[str, str]) -> None: + jobs = _jobs(state_root) + if len(jobs) != 1 or jobs[0][1].get("state") in TERMINAL_STATES | FAILURE_STATES: + return + manager_path = state_root / "manager.json" + try: + manager = json.loads(manager_path.read_text(encoding="utf-8")) + script = Path(manager["scriptPath"]).resolve() + job_id = str(jobs[0][1]["jobId"]) + except (OSError, ValueError, KeyError, TypeError): + return + if script.name != "ros_agent.py" or "alicloud-ros-agent" not in script.parts: + return + subprocess.run( + [sys.executable, str(script), "cancel", "--job-id", job_id], + cwd=repo_root, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=90, + ) + + +def run(args: argparse.Namespace) -> dict[str, Any]: + repo_root = Path(__file__).resolve().parents[4] + real_aliyun, preflight = _preflight(args, repo_root) + run_dir = args.run_dir.expanduser().resolve() + run_dir.mkdir(parents=True, exist_ok=False) + run_id = "reconnect-{}-{}".format(args.scenario, uuid.uuid4().hex[:8]) + manager_root = _manager_root(run_id) + workspace = manager_root / "qoder-workspace" + state_root = manager_root / "ros-agent-state" + workspace.mkdir(parents=True) + state_root.mkdir() + marker = "RECONNECT_E2E_{}".format(uuid.uuid4().hex[:12]) + (workspace / "task-prompt.txt").write_text( + _task_prompt(args.scenario, args.long_sleep_seconds, marker), + encoding="utf-8", + ) + fake_cli_source = repo_root / "scripts" / "a2a" / "e2e" / "reconnect" / "fake_aliyun_cli.py" + fake_cli = run_dir / "fake-aliyun" + fake_text = fake_cli_source.read_text(encoding="utf-8") + fake_cli.write_text("#!{}\n{}".format(sys.executable, fake_text.split("\n", 1)[1]), encoding="utf-8") + fake_cli.chmod(0o700) + mcp_server = repo_root / "scripts" / "a2a" / "e2e" / "reconnect" / "aliyun_cli_mcp_server.py" + scenario_state = run_dir / "mcp-scenario-state.json" + env = os.environ.copy() + env.update( + { + "ALICLOUD_ROS_AGENT_STATE_DIR": str(state_root), + "ALICLOUD_ROS_AGENT_EXECUTOR_VERSION": "qoder-mcp-e2e-v1", + "ALICLOUD_ROS_AGENT_EXECUTOR_CAPABILITIES": BOOTSTRAP_CAPABILITY, + "IAC_CODE_E2E_MCP_SERVER": str(mcp_server), + "IAC_CODE_E2E_REAL_ALIYUN": str(real_aliyun), + "IAC_CODE_E2E_CLI_IDENTITY": PROFILE, + "IAC_CODE_E2E_MCP_TIMEOUT_SECONDS": str(MCP_TIMEOUT_SECONDS), + "IAC_CODE_E2E_SCENARIO": args.scenario, + "IAC_CODE_E2E_SCENARIO_STATE": str(scenario_state), + "IAC_CODE_E2E_MCP_STDERR_LOG": str(run_dir / "mcp-stderr.log"), + "IAC_CODE_E2E_PYTHON": sys.executable, + "IAC_CODE_E2E_BOOTSTRAP_ACK_TIMEOUT_SECONDS": "15", + "PYTHONUTF8": "1", + } + ) + backups: list[_SkillBackup] = [] + job: dict[str, Any] | None = None + qoder_session = str(uuid.uuid4()) + try: + backups = _install_patched_skill(repo_root, args.skill_root, fake_cli) + next_prompt = "按系统指令开始本次测试：只执行一次 alicloud-ros-agent readiness check。" + for turn in range(args.max_qoder_turns): + evidence = _qoder_turn( + args=args, + env=env, + workspace=workspace, + state_root=state_root, + session_id=qoder_session, + prompt=next_prompt, + turn=turn, + run_dir=run_dir, + ) + jobs = _jobs(state_root) + if not jobs: + if evidence["bridgeCheck"] != 1: + raise RuntimeError("Qoder did not perform the one required readiness check") + next_prompt = ( + "readiness check 已完成。现在只执行一次 managed start：使用 task-prompt.txt、--mode normal，" + "不要传 --follow，也不要重复 check。" + ) + continue + if len(jobs) != 1: + raise RuntimeError("expected exactly one ROS Agent job") + job = jobs[0][1] + state = str(job.get("state") or "") + if state in FAILURE_STATES: + raise RuntimeError("ROS Agent job ended in {}".format(state)) + if state == "input-required": + raise RuntimeError("the reconnect E2E task unexpectedly requested input") + if state in TERMINAL_STATES: + break + next_prompt = "只对当前 job 从最新整数 cursor 调用一次 follow；不要 start、continue 或重发任务。" + else: + raise TimeoutError("Qoder turn limit reached before the ROS Agent job completed") + + assert job is not None + mcp_state = json.loads(scenario_state.read_text(encoding="utf-8")) + checks = _scenario_checks(args.scenario, mcp_state, str(job.get("finalText") or ""), marker) + qoder_turns = [ + json.loads(line) + for line in (run_dir / "qoder-turns.jsonl").read_text(encoding="utf-8").splitlines() + if line + ] + checks["Qoder used the patched installed Skill"] = all( + item.get("passedAliyunPath") is False for item in qoder_turns + ) + checks["Qoder ran check and start exactly once"] = ( + sum(int(item.get("bridgeCheck") or 0) for item in qoder_turns) == 1 + and sum(int(item.get("bridgeStart") or 0) for item in qoder_turns) == 1 + ) + result = { + "schemaVersion": 1, + "runId": run_id, + "scenario": args.scenario, + "endpoint": ENDPOINT, + "profile": PROFILE, + "mcpTimeoutSeconds": MCP_TIMEOUT_SECONDS, + "preflight": preflight, + "checks": checks, + "passed": all(checks.values()), + } + _write_json(run_dir / "result.json", result) + return result + finally: + try: + _cancel_nonterminal_job(repo_root, state_root, env) + except (OSError, subprocess.SubprocessError): + pass + _restore_skills(backups) + time.sleep(3.5) + shutil.rmtree(manager_root, ignore_errors=True) + + +def main() -> int: + result = run(_parse_args()) + print(json.dumps(result, ensure_ascii=False, separators=(",", ":"))) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/alicloud-ros-agent/SKILL.md b/skills/alicloud-ros-agent/SKILL.md index 854ebccd2..dd4e07b01 100644 --- a/skills/alicloud-ros-agent/SKILL.md +++ b/skills/alicloud-ros-agent/SKILL.md @@ -33,7 +33,7 @@ The selected credential must be allowed to call `ros:StartChat`. Explicit cancel python3 /ros_agent.py check ``` -The bounded JSON result includes the effective `transport`, `aliyunCLIExecutionMode`, endpoint, Agent modes, Thinking policy, configured Profile policy, effective region when locally available, and only non-secret credential metadata. `cli` and `version` are null when the code transport does not need the CLI. In unpinned code mode, `mode: DefaultCredentialChain` means the Credentials SDK resolved the identity without bridge-level credential parsing. In local CLI mode, `rosPluginReady`, `pluginAutoInstallEnabled`, and `pluginInstallRequired` describe plugin readiness. If and only if `pluginInstallRequired` is true, visibly report that the required ROS CLI plugin is being installed, run exactly `aliyun plugin install --name ros`, and then rerun `check`; never add a version, package URL, mirror, or source override. If the plugin is absent but CLI automatic plugin installation is enabled, `pluginInstallRequired` is false and the first `start-chat` invocation may install it. In remote CLI mode, `check` deliberately does not run CLI management commands or inspect local Profiles/plugins; it reports only the configured forwarded environment names and which names are currently present, never their values. +The bounded JSON result includes the effective `transport`, `aliyunCLIExecutionMode`, endpoint, Agent modes, Thinking policy, configured Profile policy, effective region when locally available, and only non-secret credential metadata. `cli` and `version` are null when the code transport does not need the CLI. `startChatReconnectReady` is the release gate for this workflow; if it is false, report the returned `startChatReconnectBlockers` and stop before `start`. In unpinned code mode, `mode: DefaultCredentialChain` means the Credentials SDK resolved the identity without bridge-level credential parsing. In local CLI mode, `rosPluginReady`, `pluginAutoInstallEnabled`, and `pluginInstallRequired` describe plugin readiness. If and only if `pluginInstallRequired` is true, visibly report that the required reconnect-capable ROS CLI plugin is being installed, run exactly `aliyun plugin install --name ros`, and then rerun `check`; never add a version, package URL, mirror, or source override. This explicit install-and-recheck step is required when the reconnect capability is missing even if CLI automatic plugin installation is enabled. In remote CLI mode, `check` deliberately does not run CLI management commands or inspect local Profiles/plugins; it reports only non-secret executor version/capability metadata and configured forwarded environment names, never their values. Use the check result as the sole readiness source. Except for the one local-mode plugin install command directed by `pluginInstallRequired`, never run `aliyun configure`, `aliyun plugin`, or other discovery/management commands, enumerate profiles, or read Alibaba Cloud CLI configuration files yourself. The check deliberately excludes credential values and does not prove that a token is still accepted by ROS; the StartChat response is authoritative for authentication and authorization failures. @@ -75,11 +75,12 @@ Unknown fields, invalid values, and duplicate modes fail closed. Never edit `con 2. Start a normal managed job from the target workspace. The bridge uses its process working directory only for local prompt-file isolation; it never sends a workspace or `cwd` field to StartChat: ```text - python3 /ros_agent.py start --prompt-file --mode normal --follow + python3 /ros_agent.py start --prompt-file --mode normal + python3 /ros_agent.py follow --job-id --cursor 0 --wait-seconds 60 ``` Pass `--region-id` only when the user explicitly supplied a region. Otherwise the bridge uses the first supported region environment variable, then an explicitly pinned Profile region, then `cn-hangzhou`; do not query CLI configuration to fill it. Use `--mode pipeline` only when the user explicitly wants the candidate-architecture, cost-comparison, confirmation, and deployment Pipeline. Thinking is installation policy from `config.json`, not an Agent choice. Forward underspecified infrastructure requirements to ROS Agent as written so its own `ask_user_question` can gather them. -3. Preserve the returned `jobId` and newest `cursor`. A temporary authenticated loopback manager owns the job, and a detached worker keeps the selected StartChat transport open after the outer tool call returns. In the default code transport, each SSE event is projected as it arrives. `--follow` returns at every step start, step completion/failure, input boundary, completed turn, terminal state, or its bounded wait window so the user can see the Pipeline progressing. A result can contain multiple ordered `userUpdates` when events were already queued, and can also contain `inputRequired` or a terminal result; present all updates first, then handle that result without an extra drain-only `follow`. +3. Preserve the returned `jobId` and newest `cursor`. Submit `start`, `continue`, and `respond` first, then call `follow`; the legacy `--follow` option remains compatible but is not the recommended Skill flow. A temporary authenticated loopback manager owns the job, and a detached worker keeps the selected StartChat transport open after the outer tool call returns. The worker automatically reconnects the same StartChat stream from the last committed server event ID after a retryable transport failure. Keep the same `jobId`; never start a replacement job or resend the user message to recover a stream. In the default code transport, each SSE event is projected as it arrives. `follow` returns at every step start, step completion/failure, input boundary, completed turn, terminal state, or its bounded wait window so the user can see the Pipeline progressing. A result can contain multiple ordered `userUpdates` when events were already queued, and can also contain `inputRequired` or a terminal result; present all updates first, then handle that result without an extra drain-only `follow`. 4. When the result has `boundaryReached: true`, present every `userUpdates` string to the user, then immediately follow from the returned cursor: ```text @@ -90,7 +91,7 @@ Unknown fields, invalid values, and duplicate modes fail closed. Never edit `con 5. For every natural-language follow-up, answer to `ask_user_question`, or `candidate_selection`, write a new prompt file and continue the same job: ```text - python3 /ros_agent.py continue --job-id --prompt-file --follow + python3 /ros_agent.py continue --job-id --prompt-file ``` Do not invent a `SessionId`; the job binds the remote session, mode, endpoint, region, Profile, and workspace. When a completed Pipeline returns `normalHandoffReady: true` or `conversationMode: normal`, its next user message is a Normal chat turn reached through this same `continue` command and `jobId`; the bridge intentionally keeps the StartChat mode while the remote A2A context performs the handoff. Never replace that handoff with `start --mode normal`. Do not start a new job merely to continue the same task. @@ -165,13 +166,13 @@ The event classes have different execution behavior: Do not answer a permission with natural language or create a permission JSON file. The managed job already owns the exact correlation identifiers. When exactly one permission is waiting, call `respond` with only the job and the user's decision: ```text -python3 /ros_agent.py respond --job-id --decision --follow +python3 /ros_agent.py respond --job-id --decision ``` If multiple `pendingPermissions` are waiting, keep each returned `permissionRef` associated with the action shown to the user and include only the selected short reference: ```text -python3 /ros_agent.py respond --job-id --permission-ref --decision --follow +python3 /ros_agent.py respond --job-id --permission-ref --decision ``` Never type, copy, reconstruct, transform, or save `requestTaskId`, `contextId`, `inputId`, or `toolUseId`. Do not use a shell or another script to extract `inputRequired`; `respond` resolves those fields atomically from the current job. Without `--permission-ref`, it fails closed if more than one permission is waiting. A supplied reference must match exactly one still-pending permission. diff --git a/skills/alicloud-ros-agent/SKILL.md.template b/skills/alicloud-ros-agent/SKILL.md.template index d8642c77b..ef859a63a 100644 --- a/skills/alicloud-ros-agent/SKILL.md.template +++ b/skills/alicloud-ros-agent/SKILL.md.template @@ -33,7 +33,7 @@ The selected credential must be allowed to call `ros:StartChat`. Explicit cancel python3 /ros_agent.py check ``` -The bounded JSON result includes the effective `transport`, `aliyunCLIExecutionMode`, endpoint, Agent modes, Thinking policy, configured Profile policy, effective region when locally available, and only non-secret credential metadata. `cli` and `version` are null when the code transport does not need the CLI. In unpinned code mode, `mode: DefaultCredentialChain` means the Credentials SDK resolved the identity without bridge-level credential parsing. In local CLI mode, `rosPluginReady`, `pluginAutoInstallEnabled`, and `pluginInstallRequired` describe plugin readiness. If and only if `pluginInstallRequired` is true, visibly report that the required ROS CLI plugin is being installed, run exactly `aliyun plugin install --name ros`, and then rerun `check`; never add a version, package URL, mirror, or source override. If the plugin is absent but CLI automatic plugin installation is enabled, `pluginInstallRequired` is false and the first `start-chat` invocation may install it. In remote CLI mode, `check` deliberately does not run CLI management commands or inspect local Profiles/plugins; it reports only the configured forwarded environment names and which names are currently present, never their values. +The bounded JSON result includes the effective `transport`, `aliyunCLIExecutionMode`, endpoint, Agent modes, Thinking policy, configured Profile policy, effective region when locally available, and only non-secret credential metadata. `cli` and `version` are null when the code transport does not need the CLI. `startChatReconnectReady` is the release gate for this workflow; if it is false, report the returned `startChatReconnectBlockers` and stop before `start`. In unpinned code mode, `mode: DefaultCredentialChain` means the Credentials SDK resolved the identity without bridge-level credential parsing. In local CLI mode, `rosPluginReady`, `pluginAutoInstallEnabled`, and `pluginInstallRequired` describe plugin readiness. If and only if `pluginInstallRequired` is true, visibly report that the required reconnect-capable ROS CLI plugin is being installed, run exactly `aliyun plugin install --name ros`, and then rerun `check`; never add a version, package URL, mirror, or source override. This explicit install-and-recheck step is required when the reconnect capability is missing even if CLI automatic plugin installation is enabled. In remote CLI mode, `check` deliberately does not run CLI management commands or inspect local Profiles/plugins; it reports only non-secret executor version/capability metadata and configured forwarded environment names, never their values. Use the check result as the sole readiness source. Except for the one local-mode plugin install command directed by `pluginInstallRequired`, never run `aliyun configure`, `aliyun plugin`, or other discovery/management commands, enumerate profiles, or read Alibaba Cloud CLI configuration files yourself. The check deliberately excludes credential values and does not prove that a token is still accepted by ROS; the StartChat response is authoritative for authentication and authorization failures. @@ -75,11 +75,12 @@ Unknown fields, invalid values, and duplicate modes fail closed. Never edit `con 2. Start a normal managed job from the target workspace. The bridge uses its process working directory only for local prompt-file isolation; it never sends a workspace or `cwd` field to StartChat: ```text - python3 /ros_agent.py start --prompt-file --mode normal --follow + python3 /ros_agent.py start --prompt-file --mode normal + python3 /ros_agent.py follow --job-id --cursor 0 --wait-seconds 60 ``` Pass `--region-id` only when the user explicitly supplied a region. Otherwise the bridge uses the first supported region environment variable, then an explicitly pinned Profile region, then `cn-hangzhou`; do not query CLI configuration to fill it. Use `--mode pipeline` only when the user explicitly wants the candidate-architecture, cost-comparison, confirmation, and deployment Pipeline. Thinking is installation policy from `config.json`, not an Agent choice. Forward underspecified infrastructure requirements to ROS Agent as written so its own `ask_user_question` can gather them. -3. Preserve the returned `jobId` and newest `cursor`. A temporary authenticated loopback manager owns the job, and a detached worker keeps the selected StartChat transport open after the outer tool call returns. In the default code transport, each SSE event is projected as it arrives. `--follow` returns at every step start, step completion/failure, input boundary, completed turn, terminal state, or its bounded wait window so the user can see the Pipeline progressing. A result can contain multiple ordered `userUpdates` when events were already queued, and can also contain `inputRequired` or a terminal result; present all updates first, then handle that result without an extra drain-only `follow`. +3. Preserve the returned `jobId` and newest `cursor`. Submit `start`, `continue`, and `respond` first, then call `follow`; the legacy `--follow` option remains compatible but is not the recommended Skill flow. A temporary authenticated loopback manager owns the job, and a detached worker keeps the selected StartChat transport open after the outer tool call returns. The worker automatically reconnects the same StartChat stream from the last committed server event ID after a retryable transport failure. Keep the same `jobId`; never start a replacement job or resend the user message to recover a stream. In the default code transport, each SSE event is projected as it arrives. `follow` returns at every step start, step completion/failure, input boundary, completed turn, terminal state, or its bounded wait window so the user can see the Pipeline progressing. A result can contain multiple ordered `userUpdates` when events were already queued, and can also contain `inputRequired` or a terminal result; present all updates first, then handle that result without an extra drain-only `follow`. 4. When the result has `boundaryReached: true`, present every `userUpdates` string to the user, then immediately follow from the returned cursor: ```text @@ -90,7 +91,7 @@ Unknown fields, invalid values, and duplicate modes fail closed. Never edit `con 5. For every natural-language follow-up, answer to `ask_user_question`, or `candidate_selection`, write a new prompt file and continue the same job: ```text - python3 /ros_agent.py continue --job-id --prompt-file --follow + python3 /ros_agent.py continue --job-id --prompt-file ``` Do not invent a `SessionId`; the job binds the remote session, mode, endpoint, region, Profile, and workspace. When a completed Pipeline returns `normalHandoffReady: true` or `conversationMode: normal`, its next user message is a Normal chat turn reached through this same `continue` command and `jobId`; the bridge intentionally keeps the StartChat mode while the remote A2A context performs the handoff. Never replace that handoff with `start --mode normal`. Do not start a new job merely to continue the same task. @@ -165,13 +166,13 @@ The event classes have different execution behavior: Do not answer a permission with natural language or create a permission JSON file. The managed job already owns the exact correlation identifiers. When exactly one permission is waiting, call `respond` with only the job and the user's decision: ```text -python3 /ros_agent.py respond --job-id --decision --follow +python3 /ros_agent.py respond --job-id --decision ``` If multiple `pendingPermissions` are waiting, keep each returned `permissionRef` associated with the action shown to the user and include only the selected short reference: ```text -python3 /ros_agent.py respond --job-id --permission-ref --decision --follow +python3 /ros_agent.py respond --job-id --permission-ref --decision ``` Never type, copy, reconstruct, transform, or save `requestTaskId`, `contextId`, `inputId`, or `toolUseId`. Do not use a shell or another script to extract `inputRequired`; `respond` resolves those fields atomically from the current job. Without `--permission-ref`, it fails closed if more than one permission is waiting. A supplied reference must match exactly one still-pending permission. diff --git a/skills/alicloud-ros-agent/scripts/_ros_agent_core.py b/skills/alicloud-ros-agent/scripts/_ros_agent_core.py index 47dfaab90..12f7b9531 100644 --- a/skills/alicloud-ros-agent/scripts/_ros_agent_core.py +++ b/skills/alicloud-ros-agent/scripts/_ros_agent_core.py @@ -682,6 +682,15 @@ def build_start_chat_parameters( return parameters +def build_reconnect_start_chat_parameters(session_id: str, cursor: str) -> Dict[str, str]: + return { + "AgentVersion": "V2", + "SessionId": session_id, + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": cursor, + } + + def build_command( args: argparse.Namespace, prompt: str, @@ -743,6 +752,51 @@ def build_command( return command +def build_reconnect_command(args: argparse.Namespace, session_id: str, cursor: str) -> List[str]: + endpoint_kind = _endpoint_kind(args.endpoint or "") + execution_mode = getattr(args, "aliyun_cli_execution_mode", DEFAULT_ALIYUN_CLI_EXECUTION_MODE) + if execution_mode == "remote" and endpoint_kind != "aliyun": + raise BridgeError("invalid_input", "Remote aliyun CLI execution requires a public aliyuncs.com endpoint.") + if execution_mode == "remote" and args.profile: + raise BridgeError("invalid_input", "Remote aliyun CLI execution does not accept a local Profile.") + command = [ + resolve_aliyun(args.aliyun_path), + "ros", + "start-chat", + "--endpoint", + args.endpoint, + "--connect-timeout", + str(args.connect_timeout), + "--read-timeout", + str(args.read_timeout), + "--user-agent", + USER_AGENT, + "--yes", + ] + if endpoint_kind == "loopback": + command.extend(["--secure", "--skip-secure-verify"]) + if args.profile: + command.extend(["--profile", args.profile]) + if args.region_id: + command.extend(["--region", args.region_id]) + command.extend( + [ + "--agent-version", + "V2", + "--session-id", + session_id, + "--body", + _json_bytes( + { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": cursor, + } + ).decode("utf-8"), + ] + ) + return command + + def build_stop_command(job: Dict[str, Any], session_id: str) -> List[str]: endpoint = str(job.get("endpoint") or "") endpoint_kind = _endpoint_kind(endpoint) @@ -834,6 +888,15 @@ def _read_cli_configuration() -> Dict[str, Any]: return value +def _ros_plugin_supports_reconnect(version: Any) -> bool: + if not isinstance(version, str): + return False + match = re.fullmatch(r"([0-9]+)\.([0-9]+)\.([0-9]+)", version) + if match is None: + return False + return tuple(int(part) for part in match.groups()) >= MIN_ROS_PLUGIN_VERSION + + def _local_ros_plugin_status() -> Dict[str, Any]: configured_root = os.environ.get("ALIBABA_CLOUD_CLI_PLUGINS_DIR") root = ( @@ -877,6 +940,7 @@ def _local_ros_plugin_status() -> Dict[str, Any]: version = plugin.get("version") if isinstance(version, str) and version: result["version"] = sanitize_text(version, 80) + result["reconnectReady"] = result["ready"] and _ros_plugin_supports_reconnect(version) return result @@ -1082,7 +1146,8 @@ def _open_code_request( detail = value.get("Message", value.get("message")) if isinstance(code, str) or isinstance(detail, str): message = "{}: {}".format(code or "{}Failed".format(operation), detail or "Request failed") - raise BridgeError(error_code, sanitize_text(message, 2000), response.status_code >= 500) + retryable = response.status_code == 429 or response.status_code >= 500 + raise BridgeError(error_code, sanitize_text(message, 2000), retryable) return wrapped @@ -1093,20 +1158,41 @@ def _response_text_lines(response: Any) -> Iterator[str]: yield raw_line.decode("utf-8", "replace") -def iter_sse_payloads(lines: Iterable[str]) -> Iterator[Tuple[Optional[Dict[str, Any]], str]]: +def parse_stream_event_id(value: Any) -> Tuple[str, int]: + if not isinstance(value, str) or not value: + raise BridgeError("invalid_stream_event_id", "StartChat returned an invalid SSE event ID.") + identity, separator, raw_sequence = value.rpartition(".") + if ( + not separator + or not identity.startswith("v1.") + or len(identity) <= 3 + or not raw_sequence.isdigit() + ): + raise BridgeError("invalid_stream_event_id", "StartChat returned an invalid SSE event ID.") + try: + sequence = int(raw_sequence) + except ValueError as exc: + raise BridgeError("invalid_stream_event_id", "StartChat returned an invalid SSE event ID.") from exc + if sequence <= 0: + raise BridgeError("invalid_stream_event_id", "StartChat returned an invalid SSE event ID.") + return identity, sequence + + +def iter_sse_payloads(lines: Iterable[str]) -> Iterator[Dict[str, Any]]: data_lines = [] # type: List[str] raw_lines = [] # type: List[str] + event_id = None # type: Optional[str] event_bytes = 0 - def decode(data: List[str], raw: List[str]) -> Tuple[Optional[Dict[str, Any]], str]: + def decode(data: List[str], raw: List[str], current_event_id: Optional[str]) -> Dict[str, Any]: payload_text = "\n".join(data).strip() if data else "\n".join(raw).strip() if len(payload_text.encode("utf-8")) > MAX_SSE_EVENT_BYTES: - raise BridgeError("stream_failed", "A StartChat SSE event exceeded the bridge limit.") + raise BridgeError("stream_event_too_large", "A StartChat SSE event exceeded the bridge limit.") try: value = json.loads(payload_text) except ValueError: - return None, payload_text - return (value if isinstance(value, dict) else None), payload_text + value = None + return {"id": current_event_id, "payload": value if isinstance(value, dict) else None, "raw": payload_text} for raw_line in lines: event_bytes += len(raw_line.encode("utf-8")) @@ -1115,19 +1201,22 @@ def decode(data: List[str], raw: List[str]) -> Tuple[Optional[Dict[str, Any]], s line = raw_line.rstrip("\r\n") if not line: if data_lines or raw_lines: - yield decode(data_lines, raw_lines) + yield decode(data_lines, raw_lines, event_id) data_lines = [] raw_lines = [] + event_id = None event_bytes = 0 continue if line.startswith(":"): continue if line.startswith("data:"): data_lines.append(line[5:].lstrip()) + elif line.startswith("id:"): + event_id = line[3:].lstrip() elif not data_lines: raw_lines.append(line) if data_lines or raw_lines: - yield decode(data_lines, raw_lines) + yield decode(data_lines, raw_lines, event_id) def _cli_plugin_payload(payload: Dict[str, Any]) -> Dict[str, Any]: @@ -1135,24 +1224,67 @@ def _cli_plugin_payload(payload: Dict[str, Any]) -> Dict[str, Any]: return data if isinstance(data, dict) else payload -def iter_cli_plugin_payloads(lines: Iterable[str]) -> Iterator[Tuple[Optional[Dict[str, Any]], str]]: +def _is_cli_failure_envelope(value: Any) -> bool: + if not isinstance(value, dict) or any(key in value for key in ("id", "data", "event")): + return False + candidates = [value] + nested = value.get("error") + if isinstance(nested, dict): + candidates.append(nested) + return any( + isinstance(candidate.get("statusCode", candidate.get("httpStatus")), int) + or isinstance(candidate.get("code", candidate.get("Code")), str) + for candidate in candidates + ) + + +def iter_cli_plugin_payloads(lines: Iterable[str]) -> Iterator[Dict[str, Any]]: decoder = json.JSONDecoder() buffer = "" + raw_bytes = 0 + decoded_bytes = 0 - def projected(value: Any, raw: str) -> Iterator[Tuple[Optional[Dict[str, Any]], str]]: + def projected(value: Any, raw: str) -> Iterator[Dict[str, Any]]: + nonlocal decoded_bytes if isinstance(value, list): for item in value: item_raw = json.dumps(item, ensure_ascii=False, separators=(",", ":")) - yield (_cli_plugin_payload(item) if isinstance(item, dict) else None), item_raw + yield from projected(item, item_raw) + return + if not isinstance(value, dict): + yield {"id": None, "payload": None, "raw": raw} return - yield (_cli_plugin_payload(value) if isinstance(value, dict) else None), raw + if _is_cli_failure_envelope(value): + yield {"id": None, "payload": None, "raw": raw, "cliFailure": True} + return + bootstrap = value if isinstance(value.get("event"), dict) and "invocationId" in value else None + envelope = value.get("event") if isinstance(bootstrap, dict) else value + assert isinstance(envelope, dict) + payload = _cli_plugin_payload(envelope) + payload_bytes = len(_json_bytes(payload)) + if payload_bytes > MAX_SSE_EVENT_BYTES: + raise BridgeError("stream_event_too_large", "A StartChat CLI event exceeded the bridge limit.") + if bootstrap is None: + decoded_bytes += payload_bytes + if decoded_bytes > MAX_SERVER_REPLAY_BYTES: + raise BridgeError("server_replay_too_large", "StartChat replay exceeded the server replay limit.") + yield { + "id": envelope.get("id"), + "payload": payload, + "raw": raw, + "bootstrap": bootstrap, + } for raw_line in lines: - if len(raw_line.encode("utf-8")) > MAX_SSE_LINE_BYTES: - raise BridgeError("stream_failed", "A StartChat CLI output line exceeded the bridge limit.") + line_bytes = len(raw_line.encode("utf-8")) + if line_bytes > MAX_CLI_BATCH_BYTES: + raise BridgeError("cli_output_too_large", "A StartChat CLI output line exceeded the batch limit.") + raw_bytes += line_bytes + if raw_bytes > MAX_CLI_BATCH_BYTES: + raise BridgeError("cli_output_too_large", "StartChat CLI output exceeded the batch limit.") buffer += raw_line - if len(buffer.encode("utf-8")) > MAX_SSE_EVENT_BYTES: - raise BridgeError("stream_failed", "Buffered StartChat CLI output exceeded the bridge limit.") + if len(buffer.encode("utf-8")) > MAX_CLI_BATCH_BYTES: + raise BridgeError("cli_output_too_large", "Buffered StartChat CLI output exceeded the batch limit.") while buffer.strip(): leading = len(buffer) - len(buffer.lstrip()) try: @@ -1168,11 +1300,49 @@ def projected(value: Any, raw: str) -> Iterator[Tuple[Optional[Dict[str, Any]], break if remainder[remainder_end:].strip(): break - yield None, first_line.strip() + yield {"id": None, "payload": None, "raw": first_line.strip()} buffer = remainder continue raw = buffer[leading:end] yield from projected(value, raw) buffer = buffer[end:] if buffer.strip(): - yield None, buffer.strip() + yield {"id": None, "payload": None, "raw": buffer.strip()} + + +def is_retryable_cli_failure(return_code: int, *diagnostics: str) -> bool: + if return_code == 0: + return False + stable_timeout_codes = { + "executortimeout", + "readtimeout", + "connectiontimeout", + "connectionreset", + } + + def values(item: Any) -> Iterator[Tuple[Optional[int], Optional[str]]]: + if isinstance(item, dict): + raw_status = item.get("statusCode", item.get("httpStatus")) + status = raw_status if isinstance(raw_status, int) else None + raw_code = item.get("code", item.get("Code")) + code = raw_code.lower() if isinstance(raw_code, str) else None + yield status, code + for nested in item.values(): + yield from values(nested) + elif isinstance(item, list): + for nested in item: + yield from values(nested) + + for text in diagnostics: + if not text: + continue + try: + value = json.loads(text) + except ValueError: + continue + for status, code in values(value): + if status == 429 or (isinstance(status, int) and 500 <= status <= 599): + return True + if code in stable_timeout_codes: + return True + return False diff --git a/skills/alicloud-ros-agent/scripts/_ros_agent_projection.py b/skills/alicloud-ros-agent/scripts/_ros_agent_projection.py index e4ad351f1..6bed477c2 100644 --- a/skills/alicloud-ros-agent/scripts/_ros_agent_projection.py +++ b/skills/alicloud-ros-agent/scripts/_ros_agent_projection.py @@ -1127,7 +1127,7 @@ def _project_managed_stream_event( return projection -def _append_projection(job_id: str, projection: Dict[str, Any]) -> None: +def _append_projection(job_id: str, projection: Dict[str, Any], stream_event_id: Optional[str] = None) -> bool: root, job_path, spool = _job_paths(job_id) _secure_directory(root) projection = _bound_projection(projection) @@ -1135,11 +1135,17 @@ def _append_projection(job_id: str, projection: Dict[str, Any]) -> None: job = _load_state_json(job_path) request_seq = projection.get("requestSeq") if isinstance(request_seq, int) and request_seq != job.get("activeRequestSeq"): - return + return False worker_role = projection.get("workerRole") worker_token = projection.get("workerToken") if worker_role == "sideband" and worker_token != job.get("sidebandWorkerToken"): - return + return False + if job.get("stopRequestedAt") is not None: + session_id = projection.get("sessionId") + if isinstance(session_id, str) and session_id and not isinstance(job.get("sessionId"), str): + job["sessionId"] = session_id + _atomic_json(job_path, job) + return False projection_error = projection.get("error") if ( worker_role != "sideband" @@ -1314,18 +1320,38 @@ def _append_projection(job_id: str, projection: Dict[str, Any]) -> None: wire_projection.pop("workerRole", None) wire_projection.pop("workerToken", None) meaningful = wire_projection.get("type") != "status" or identity_changed + already_spooled = False + if meaningful and isinstance(stream_event_id, str): + wire_projection["streamEventId"] = stream_event_id + already_spooled = any( + value.get("streamEventId") == stream_event_id + and value.get("requestSeq") == request_seq + for value in _read_spool(spool) + ) if meaningful: data = _json_bytes(wire_projection) + b"\n" current_size = spool.stat().st_size if spool.exists() else 0 - if current_size + len(data) > MAX_SPOOL_BYTES: + if not already_spooled and current_size + len(data) > MAX_SPOOL_BYTES: raise BridgeError("stream_failed", "The bounded ROS Agent event spool is full.") - with spool.open("ab") as handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - if os.name != "nt": - os.chmod(str(spool), 0o600) + if not already_spooled: + with spool.open("ab") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + if os.name != "nt": + os.chmod(str(spool), 0o600) + if isinstance(stream_event_id, str): + identity, sequence = parse_stream_event_id(stream_event_id) + if worker_role == "sideband": + job["sidebandStreamCursor"] = stream_event_id + job["sidebandStreamIdentity"] = identity + job["sidebandStreamSequence"] = sequence + else: + job["streamCursor"] = stream_event_id + job["streamIdentity"] = identity + job["streamSequence"] = sequence _atomic_json(job_path, job) + return True def _finish_job( @@ -1340,6 +1366,8 @@ def _finish_job( job = _load_state_json(job_path) if job.get("activeRequestSeq") != request_seq: return False + if job.get("stopRequestedAt") is not None: + return False if expected_worker_pid is not None: current_worker_pid = job.get("workerPid") worker_matches = ( @@ -1482,6 +1510,8 @@ def _finish_sideband_job( job = _load_state_json(job_path) if job.get("activeRequestSeq") != request_seq or job.get("sidebandWorkerToken") != worker_token: return + if job.get("stopRequestedAt") is not None: + return for key in ("sessionId", "taskId", "iacCodeSessionId", "requestId", "wireState"): value = result.get(key) if isinstance(value, str) and value: @@ -1860,8 +1890,9 @@ def _job_result( seen.add(signature) milestones.append(milestone) job_state = str(job.get("state") or "unknown") - has_result_gate = job_state in TERMINAL_STATES | {"turn-completed", "failed"} or isinstance( - job.get("inputRequired"), dict + has_result_gate = ( + job_state in TERMINAL_STATES | {"turn-completed", "failed", "canceling", "not-active"} + or isinstance(job.get("inputRequired"), dict) ) state = job_state if has_result_gate else ("working" if boundary_reached else job_state) result = { @@ -1952,7 +1983,7 @@ def _follow_ready_result(job_id: str, start_cursor: int) -> Tuple[Optional[Dict[ state = job.get("state") if ( has_step_boundary - or state in TERMINAL_STATES | {"turn-completed", "failed"} + or state in TERMINAL_STATES | {"turn-completed", "failed", "canceling", "not-active"} or isinstance(job.get("inputRequired"), dict) or isinstance(job.get("sidebandError"), dict) ): diff --git a/skills/alicloud-ros-agent/scripts/_ros_agent_runtime.py b/skills/alicloud-ros-agent/scripts/_ros_agent_runtime.py index 1112853c0..0a2df1078 100644 --- a/skills/alicloud-ros-agent/scripts/_ros_agent_runtime.py +++ b/skills/alicloud-ros-agent/scripts/_ros_agent_runtime.py @@ -21,96 +21,308 @@ def _consume_start_chat( *, summary_mode: Optional[str] = None, on_payload: Optional[Any] = None, + worker_role: str = "primary", + permission_response: Optional[Dict[str, Any]] = None, + can_reconnect: Optional[Any] = None, ) -> Dict[str, Any]: summary = StreamSummary(args.session_id, mode=summary_mode or args.mode) diagnostics = [] # type: List[str] + stream_cursor = None # type: Optional[str] + stream_identity = None # type: Optional[str] + stream_sequence = 0 + pending_applied_stream_event_id = None # type: Optional[str] + reconnect_attempt = 0 + + def boundary_is_immediate() -> bool: + if summary.state in TERMINAL_STATES: + return False + if summary.input_required is not None and not summary.input_required_from_pending: + return True + if summary.assistant_final or (summary.mode == "normal" and summary.state == "input-required"): + return True + return ( + worker_role == "sideband" + and isinstance(permission_response, dict) + and isinstance(summary.permission_ack, dict) + and _permission_response_is_acknowledged(permission_response, summary.permission_ack) + ) - if getattr(args, "transport", "aliyun_cli") == "code": - response = _open_code_request( - "StartChat", - build_start_chat_parameters(args, prompt, client_context, attachments), - str(args.endpoint), - args.profile, - args.region_id, - args.aliyun_path, - int(args.connect_timeout), - int(args.read_timeout), - credential_source=getattr(args, "credential_source", None), + def process_event(event: Dict[str, Any], invocation_id: Optional[str] = None) -> bool: + nonlocal stream_cursor, stream_identity, stream_sequence, pending_applied_stream_event_id + bootstrap = event.get("bootstrap") + bootstrap_session_id = None # type: Optional[str] + if isinstance(bootstrap, dict): + if bootstrap.get("invocationId") != invocation_id: + return True + nested = bootstrap.get("event") + if not isinstance(nested, dict) or not isinstance(nested.get("data"), dict): + return True + candidate = bootstrap.get("sessionId") + if isinstance(candidate, str) and candidate: + bootstrap_session_id = candidate + + payload = event.get("payload") + raw = event.get("raw") + if not isinstance(payload, dict): + summary.malformed_event_count += 1 + if isinstance(raw, str) and raw: + diagnostics.append(raw) + return True + payload_session_id = _find_first(payload, "contextId", "context_id", "SessionId") + if ( + isinstance(payload_session_id, str) + and payload_session_id + and bootstrap_session_id is not None + and payload_session_id != bootstrap_session_id + ): + raise BridgeError("stream_session_mismatch", "StartChat bootstrap returned a different SessionId.") + observed_session_id = ( + payload_session_id + if isinstance(payload_session_id, str) and payload_session_id + else bootstrap_session_id ) - try: - content_type = str(response.headers.get("Content-Type", "")).lower() - if "text/event-stream" not in content_type: - raw = response.read(MAX_DIAGNOSTIC_BYTES + 1) - detail = sanitize_text(raw.decode("utf-8", "replace"), 2000) + if ( + isinstance(observed_session_id, str) + and observed_session_id + and summary.session_id + and observed_session_id != summary.session_id + ): + raise BridgeError("stream_session_mismatch", "StartChat returned a different SessionId.") + if can_reconnect is not None and can_reconnect() is False: + if isinstance(observed_session_id, str) and observed_session_id: + summary.session_id = observed_session_id + if on_payload is not None: + on_payload(None, payload, summary, False) + return False + event_id = event.get("id") + heartbeat = str(payload.get("object", "")).lower() in {"heartbeat", "keepalive"} + if heartbeat: + summary.apply(payload) + return True + if event_id is None: + raise BridgeError("missing_stream_event_id", "StartChat returned a business event without an SSE ID.") + identity, sequence = parse_stream_event_id(event_id) + if stream_identity is not None and identity != stream_identity: + raise BridgeError( + "stream_event_identity_mismatch", + "StartChat returned an event from a different stream during reconnect.", + ) + if stream_identity is None: + stream_identity = identity + if sequence <= stream_sequence: + return True + + already_applied = pending_applied_stream_event_id == event_id + if not already_applied: + if not (isinstance(payload_session_id, str) and payload_session_id) and bootstrap_session_id is not None: + summary.session_id = bootstrap_session_id + summary.apply(payload) + if not isinstance(summary.session_id, str) or not summary.session_id: raise BridgeError( - "stream_failed", - detail or "Alibaba Cloud ROS StartChat did not return an SSE stream.", - True, + "missing_stream_session_id", + "StartChat did not provide the SessionId required for reconnect.", ) - for payload, raw in iter_sse_payloads(_response_text_lines(response)): - if payload is None: - summary.malformed_event_count += 1 - if raw: - diagnostics.append(raw) - continue - summary.apply(payload) - if on_payload is not None: - on_payload(payload, summary) - except BridgeError: - raise - except Exception as exc: - raise BridgeError( - "stream_failed", - "Alibaba Cloud ROS StartChat stream ended unexpectedly.", - True, - ) from exc - finally: - response.close() + pending_applied_stream_event_id = event_id + + if on_payload is not None and on_payload(event_id, payload, summary, already_applied) is False: + return False + stream_cursor = event_id + stream_sequence = sequence + pending_applied_stream_event_id = None + return True + + def result() -> Dict[str, Any]: return summary.to_result(0, "\n".join(diagnostics)) - command = build_command(args, prompt, client_context, attachments) - with tempfile.TemporaryFile(mode="w+b") as stderr_file: - try: - process = subprocess.Popen( - command, - cwd=str(workspace), - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=stderr_file, - text=True, - encoding="utf-8", - errors="replace", + while True: + reconnecting = stream_cursor is not None + if reconnecting and can_reconnect is not None and can_reconnect() is False: + return {"ok": True, "state": "worker-stopped", "_workerStopped": True} + immediate = False + transport_error = None # type: Optional[BridgeError] + cli_return_code = 0 + cli_stderr = "" + cli_failure_seen = False + + if getattr(args, "transport", "aliyun_cli") == "code": + parameters = ( + build_reconnect_start_chat_parameters(str(summary.session_id), stream_cursor) + if reconnecting + else build_start_chat_parameters(args, prompt, client_context, attachments) ) - except OSError as exc: - raise BridgeError("cli_start_failed", "Alibaba Cloud CLI could not be started.", True) from exc - assert process.stdout is not None - try: - for payload, raw in iter_cli_plugin_payloads(process.stdout): - if payload is None: - summary.malformed_event_count += 1 - if raw: - diagnostics.append(raw) - else: - summary.apply(payload) - if on_payload is not None: - on_payload(payload, summary) - return_code = process.wait() - except KeyboardInterrupt as exc: - _stop_process(process) + response = None + try: + response = _open_code_request( + "StartChat", + parameters, + str(args.endpoint), + args.profile, + args.region_id, + args.aliyun_path, + int(args.connect_timeout), + int(args.read_timeout), + credential_source=getattr(args, "credential_source", None), + ) + content_type = str(response.headers.get("Content-Type", "")).lower() + if "text/event-stream" not in content_type: + raw = response.read(MAX_DIAGNOSTIC_BYTES + 1) + detail = sanitize_text(raw.decode("utf-8", "replace"), 2000) + raise BridgeError( + "stream_failed", + detail or "Alibaba Cloud ROS StartChat did not return an SSE stream.", + True, + ) + for event in iter_sse_payloads(_response_text_lines(response)): + if not process_event(event): + return {"ok": True, "state": "worker-stopped", "_workerStopped": True} + if boundary_is_immediate(): + immediate = True + break + except KeyboardInterrupt as exc: + raise BridgeError( + "interrupted", + "StartChat was interrupted locally; remote cancellation is not confirmed.", + ) from exc + except BridgeError as exc: + transport_error = exc + except Exception as exc: + transport_error = BridgeError( + "stream_failed", "Alibaba Cloud ROS StartChat stream ended unexpectedly.", True + ) + transport_error.__cause__ = exc + finally: + if response is not None: + response.close() + else: + command = ( + build_reconnect_command(args, str(summary.session_id), stream_cursor) + if reconnecting + else build_command(args, prompt, client_context, attachments) + ) + invocation_id = ( + uuid.uuid4().hex if getattr(args, "aliyun_cli_execution_mode", "local") == "remote" else None + ) + with tempfile.TemporaryDirectory() as bootstrap_root, tempfile.TemporaryFile(mode="w+b") as stderr_file: + process_environment = None # type: Optional[Dict[str, str]] + ack_path = None # type: Optional[pathlib.Path] + if invocation_id is not None: + ack_path = pathlib.Path(bootstrap_root) / "bootstrap-ack.json" + process_environment = dict(os.environ) + process_environment[REMOTE_BOOTSTRAP_INVOCATION_ENV] = invocation_id + process_environment[REMOTE_BOOTSTRAP_ACK_FILE_ENV] = str(ack_path) + process_environment[REMOTE_BOOTSTRAP_PROTOCOL_ENV] = REMOTE_BOOTSTRAP_CAPABILITY + try: + process = subprocess.Popen( + command, + cwd=str(workspace), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=stderr_file, + text=True, + encoding="utf-8", + errors="replace", + env=process_environment, + ) + except OSError as exc: + process = None + transport_error = BridgeError("cli_start_failed", "Alibaba Cloud CLI could not be started.", True) + transport_error.__cause__ = exc + if process is not None: + assert process.stdout is not None + try: + for event in iter_cli_plugin_payloads(process.stdout): + if event.get("cliFailure") is True: + cli_failure_seen = True + raw = event.get("raw") + if isinstance(raw, str) and raw: + diagnostics.append(raw) + continue + bootstrap = event.get("bootstrap") + if isinstance(bootstrap, dict) and ( + bootstrap.get("invocationId") != invocation_id + or not isinstance(bootstrap.get("event"), dict) + or not isinstance(bootstrap["event"].get("id"), str) + or not isinstance(bootstrap["event"].get("data"), dict) + ): + continue + if not process_event(event, invocation_id): + _stop_process(process) + return {"ok": True, "state": "worker-stopped", "_workerStopped": True} + if ( + isinstance(bootstrap, dict) + and ack_path is not None + and isinstance(event.get("id"), str) + ): + _atomic_json( + ack_path, + { + "invocationId": invocation_id, + "eventId": event["id"], + "committed": True, + }, + ) + if boundary_is_immediate(): + immediate = True + _stop_process(process) + break + cli_return_code = process.wait() + except KeyboardInterrupt as exc: + _stop_process(process) + raise BridgeError( + "interrupted", + "StartChat was interrupted locally; remote cancellation is not confirmed.", + ) from exc + except BridgeError as exc: + _stop_process(process) + transport_error = exc + except BaseException as exc: + _stop_process(process) + transport_error = BridgeError( + "stream_failed", "Alibaba Cloud CLI StartChat output ended unexpectedly.", True + ) + transport_error.__cause__ = exc + finally: + process.stdout.close() + stderr_file.seek(0) + cli_stderr = stderr_file.read(MAX_DIAGNOSTIC_BYTES).decode("utf-8", "replace") + + if immediate: + return result() + if transport_error is None and cli_return_code == 0 and cli_failure_seen: raise BridgeError( - "interrupted", - "StartChat was interrupted locally; remote cancellation is not confirmed.", - ) from exc - except BaseException: - _stop_process(process) - raise - finally: - process.stdout.close() - stderr_file.seek(0) - stderr_text = stderr_file.read(MAX_DIAGNOSTIC_BYTES).decode("utf-8", "replace") - if diagnostics and not stderr_text: - stderr_text = "\n".join(diagnostics) - return summary.to_result(return_code, stderr_text) + "aliyun_cli_failed", + sanitize_text("\n".join(diagnostics[-8:]), 3000) or "Alibaba Cloud CLI StartChat failed.", + ) + if transport_error is None and cli_return_code != 0: + diagnostic_text = "\n".join(diagnostics[-8:]) + if is_retryable_cli_failure(cli_return_code, cli_stderr, diagnostic_text): + transport_error = BridgeError( + "aliyun_cli_failed", + sanitize_text(cli_stderr, 3000) or "Alibaba Cloud CLI StartChat was interrupted.", + True, + ) + else: + return summary.to_result(cli_return_code, cli_stderr or diagnostic_text) + if transport_error is not None: + if not transport_error.retryable: + raise transport_error + if stream_cursor is None or not isinstance(summary.session_id, str) or not summary.session_id: + raise BridgeError( + "missing_recovery_anchor", + "StartChat was interrupted before a complete recovery anchor was committed.", + ) from transport_error + elif summary.state in TERMINAL_STATES: + return result() + elif stream_cursor is None or not isinstance(summary.session_id, str) or not summary.session_id: + raise BridgeError( + "missing_recovery_anchor", + "StartChat ended before a complete recovery anchor was committed.", + ) + + if can_reconnect is not None and can_reconnect() is False: + return {"ok": True, "state": "worker-stopped", "_workerStopped": True} + reconnect_attempt += 1 + time.sleep(min(0.1 * (2 ** min(reconnect_attempt - 1, 6)), MAX_RECONNECT_BACKOFF_SECONDS)) def run_chat(args: argparse.Namespace) -> Dict[str, Any]: @@ -239,6 +451,7 @@ def _request_from_job(job: Dict[str, Any], prompt: str) -> Dict[str, Any]: "aliyunPath": job.get("aliyunPath", "aliyun"), "clientContext": None, "attachments": [], + "streamCursor": None, } @@ -371,6 +584,8 @@ def _continue_job_local(payload: Dict[str, Any]) -> Dict[str, Any]: job["conversationMode"] = "normal" job.pop("pipelineResult", None) job["activeRequestSeq"] = int(job.get("activeRequestSeq") or 0) + 1 + for key in ("streamCursor", "streamIdentity", "streamSequence"): + job.pop(key, None) job["state"] = "submitted" job.pop("inputRequired", None) job.pop("error", None) @@ -525,10 +740,14 @@ def _respond_job_local(payload: Dict[str, Any]) -> Dict[str, Any]: job["sidebandResponseInputId"] = response.get("inputId") job["sidebandResponse"] = pending job["state"] = "working" + for key in ("sidebandStreamCursor", "sidebandStreamIdentity", "sidebandStreamSequence"): + job.pop(key, None) else: if primary_worker_alive: raise BridgeError("job_busy", "The current StartChat request is still running.", True) job["activeRequestSeq"] = int(job.get("activeRequestSeq") or 0) + 1 + for key in ("streamCursor", "streamIdentity", "streamSequence"): + job.pop(key, None) job["state"] = "submitted" job["permissionResponseInput"] = pending job["lastPermissionResponse"] = response @@ -634,32 +853,168 @@ def _run_stop_chat(job: Dict[str, Any], session_id: str) -> Dict[str, Any]: return result +def _claim_stop_dispatch(job_id: str, session_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + root, job_path, _spool = _job_paths(job_id) + with StateLock(root / ".job.lock"): + job = _load_state_json(job_path) + if job.get("stopRequestedAt") is None: + return None + if isinstance(session_id, str) and session_id: + existing = job.get("sessionId") + if isinstance(existing, str) and existing and existing != session_id: + return None + job["sessionId"] = session_id + effective_session_id = job.get("sessionId") + if ( + not isinstance(effective_session_id, str) + or not effective_session_id + or job.get("stopDispatchStartedAt") is not None + ): + if isinstance(session_id, str) and session_id: + _atomic_json(job_path, job) + return None + job["stopDispatchStartedAt"] = int(time.time()) + _atomic_json(job_path, job) + return dict(job) + + +def _settle_stop_dispatch( + job_id: str, + stopped: Optional[Dict[str, Any]] = None, + error: Optional[BridgeError] = None, +) -> Dict[str, Any]: + root, job_path, _spool = _job_paths(job_id) + with StateLock(root / ".job.lock"): + job = _load_state_json(job_path) + job.setdefault("stopRequestedAt", int(time.time())) + stop_status = stopped.get("status") if isinstance(stopped, dict) else None + if error is not None or stop_status == "Failed" or stop_status not in {"Stopped", "Stopping", "NoActiveStream"}: + job["stopStatus"] = "Failed" + job["state"] = "failed" + job["error"] = { + "code": "stop_chat_failed", + "message": sanitize_text(error.message, 2000) + if isinstance(error, BridgeError) + else "Alibaba Cloud ROS could not stop the active chat.", + "retryable": True, + } + else: + job["stopStatus"] = stop_status + job["state"] = { + "Stopped": "canceled", + "Stopping": "canceling", + "NoActiveStream": "not-active", + }[stop_status] + job.pop("inputRequired", None) + job.pop("pendingPermissions", None) + if isinstance(stopped, dict) and isinstance(stopped.get("requestId"), str): + job["stopRequestId"] = stopped["requestId"] + _atomic_json(job_path, job) + return job + + +def _dispatch_claimed_stop(job_id: str, claimed_job: Dict[str, Any]) -> Dict[str, Any]: + session_id = claimed_job.get("sessionId") + if not isinstance(session_id, str) or not session_id: + return _settle_stop_dispatch( + job_id, + error=BridgeError("stop_chat_failed", "Alibaba Cloud ROS StopChat has no SessionId.", True), + ) + stop_job = dict(claimed_job) + try: + stop_job["_transientEnvironment"] = _capture_remote_cli_environment( + stop_job.get("aliyunCLIForwardEnv", []) + ) + stopped = _run_stop_chat(stop_job, session_id) + except BridgeError as exc: + return _settle_stop_dispatch(job_id, error=exc) + except BaseException as exc: + error = BridgeError("stop_chat_failed", "Alibaba Cloud ROS StopChat failed.", True) + error.__cause__ = exc + return _settle_stop_dispatch(job_id, error=error) + return _settle_stop_dispatch(job_id, stopped=stopped) + + +def _record_stopped_worker_exit( + job_id: str, + request_seq: int, + worker_role: Any, + worker_token: Any, + worker_pid: int, +) -> None: + root, job_path, _spool = _job_paths(job_id) + with StateLock(root / ".job.lock"): + job = _load_state_json(job_path) + if job.get("activeRequestSeq") != request_seq: + return + if worker_role == "sideband": + if job.get("sidebandWorkerToken") != worker_token: + return + if job.get("sidebandWorkerPid") == worker_pid: + job.pop("sidebandWorkerPid", None) + job.pop("sidebandWorkerToken", None) + elif job.get("workerPid") == worker_pid: + job.pop("workerPid", None) + _atomic_json(job_path, job) + + def _cancel_job_local(payload: Dict[str, Any]) -> Dict[str, Any]: job_id = str(payload.get("jobId") or "") root, job_path, spool = _job_paths(job_id) + with StateLock(root / ".job.lock"): + job = _load_state_json(job_path) + job.setdefault("stopRequestedAt", int(time.time())) + _atomic_json(job_path, job) + deadline = time.monotonic() + STOP_SESSION_WAIT_SECONDS + claimed_job = _claim_stop_dispatch(job_id) while True: job = _load_state_json(job_path) - session_id = job.get("sessionId") - if isinstance(session_id, str) and session_id: + if claimed_job is not None or job.get("stopStatus") is not None: + break + claimed_job = _claim_stop_dispatch(job_id) + if claimed_job is not None: break if time.monotonic() >= deadline: - raise BridgeError("job_not_ready", "The ROS Agent job has not received a SessionId yet.", True) + with StateLock(root / ".job.lock"): + latest = _load_state_json(job_path) + if latest.get("stopStatus") is None and latest.get("state") not in TERMINAL_STATES | {"failed"}: + latest["state"] = "canceling" + _atomic_json(job_path, latest) + return { + "ok": True, + "jobId": job_id, + "state": "canceling", + "mode": latest.get("mode"), + "preferredLanguage": latest.get("preferredLanguage", "en"), + "cursor": len(_read_spool(spool)), + "turn": int(latest.get("turn") or 1), + "presentationRequired": True, + } time.sleep(0.1) - stop_job = dict(job) - stop_job["_transientEnvironment"] = _remote_cli_environment_from_payload(job, payload) - stopped = _run_stop_chat(stop_job, session_id) - stop_status = stopped["status"] - with StateLock(root / ".job.lock"): - latest = _load_state_json(job_path) - latest["stopStatus"] = stop_status - latest["stopRequestedAt"] = int(time.time()) - if stop_status == "Stopped": - latest["state"] = "canceled" - latest.pop("inputRequired", None) - latest.pop("pendingPermissions", None) - _atomic_json(job_path, latest) + if claimed_job is not None: + claimed_job["_transientEnvironment"] = _remote_cli_environment_from_payload(claimed_job, payload) + try: + stopped = _run_stop_chat(claimed_job, str(claimed_job["sessionId"])) + except BridgeError as exc: + latest = _settle_stop_dispatch(job_id, error=exc) + except BaseException as exc: + error = BridgeError("stop_chat_failed", "Alibaba Cloud ROS StopChat failed.", True) + error.__cause__ = exc + latest = _settle_stop_dispatch(job_id, error=error) + else: + latest = _settle_stop_dispatch(job_id, stopped=stopped) + else: + while time.monotonic() < deadline: + latest = _load_state_json(job_path) + if latest.get("stopStatus") is not None: + break + time.sleep(0.1) + else: + latest = _load_state_json(job_path) + + stop_status = str(latest.get("stopStatus") or "Stopping") state_by_status = { "Stopped": "canceled", "Stopping": "canceling", @@ -675,13 +1030,15 @@ def _cancel_job_local(payload: Dict[str, Any]) -> Dict[str, Any]: "preferredLanguage": latest.get("preferredLanguage", "en"), "cursor": len(_read_spool(spool)), "turn": int(latest.get("turn") or 1), - "sessionId": session_id, "presentationRequired": True, } # type: Dict[str, Any] + session_id = latest.get("sessionId") + if isinstance(session_id, str) and session_id: + result["sessionId"] = session_id if latest.get("conversationMode") in SUPPORTED_AGENT_MODES: result["conversationMode"] = latest["conversationMode"] - if isinstance(stopped.get("requestId"), str): - result["requestId"] = stopped["requestId"] + if isinstance(latest.get("stopRequestId"), str): + result["requestId"] = latest["stopRequestId"] if stop_status == "Failed": result["error"] = { "code": "stop_chat_failed", @@ -738,8 +1095,23 @@ def fail_worker(error: BridgeError) -> None: attachments = request.get("attachments") if isinstance(request.get("attachments"), list) else [] summary_mode = request.get("summaryMode") if request.get("summaryMode") in SUPPORTED_AGENT_MODES else args.mode - def project(payload: Dict[str, Any], summary: StreamSummary) -> None: - _append_projection( + def worker_can_continue() -> bool: + _current_root, current_job_path, _current_spool = _job_paths(job_id) + with StateLock(_current_root / ".job.lock"): + current = _load_state_json(current_job_path) + if current.get("activeRequestSeq") != request_seq or current.get("stopRequestedAt") is not None: + return False + if worker_role == "sideband" and current.get("sidebandWorkerToken") != worker_token: + return False + return True + + def project( + stream_event_id: Optional[str], + payload: Dict[str, Any], + summary: StreamSummary, + _already_applied: bool, + ) -> bool: + return _append_projection( job_id, _project_managed_stream_event( payload, @@ -749,6 +1121,7 @@ def project(payload: Dict[str, Any], summary: StreamSummary) -> None: str(worker_role or "primary"), worker_token, ), + stream_event_id, ) try: @@ -760,18 +1133,50 @@ def project(payload: Dict[str, Any], summary: StreamSummary) -> None: attachments, summary_mode=summary_mode, on_payload=project, + worker_role=str(worker_role or "primary"), + permission_response=( + request.get("permissionResponse") + if isinstance(request.get("permissionResponse"), dict) + else None + ), + can_reconnect=worker_can_continue, ) except BaseException as exc: error = exc if isinstance(exc, BridgeError) else BridgeError("stream_failed", str(exc), True) + if not worker_can_continue(): + claimed_job = _claim_stop_dispatch(job_id) + if claimed_job is not None: + _dispatch_claimed_stop(job_id, claimed_job) + _record_stopped_worker_exit(job_id, request_seq, worker_role, worker_token, worker_pid) + latest = _load_state_json(_job_paths(job_id)[1]) + return 1 if latest.get("state") == "failed" else 0 fail_worker(error) return 1 + if result.get("_workerStopped") is True: + claimed_job = _claim_stop_dispatch(job_id) + if claimed_job is not None: + _dispatch_claimed_stop(job_id, claimed_job) + _record_stopped_worker_exit(job_id, request_seq, worker_role, worker_token, worker_pid) + latest = _load_state_json(_job_paths(job_id)[1]) + return 1 if latest.get("state") == "failed" else 0 + permission_response = request.get("permissionResponse") if isinstance(permission_response, dict): result["permissionResponse"] = permission_response if worker_role == "sideband" and isinstance(worker_token, str): _finish_sideband_job(job_id, request_seq, worker_token, result, worker_pid) + if not worker_can_continue(): + claimed_job = _claim_stop_dispatch(job_id) + if claimed_job is not None: + _dispatch_claimed_stop(job_id, claimed_job) + _record_stopped_worker_exit(job_id, request_seq, worker_role, worker_token, worker_pid) else: - _finish_job(job_id, request_seq, result, worker_pid) + finished = _finish_job(job_id, request_seq, result, worker_pid) + if not finished: + claimed_job = _claim_stop_dispatch(job_id) + if claimed_job is not None: + _dispatch_claimed_stop(job_id, claimed_job) + _record_stopped_worker_exit(job_id, request_seq, worker_role, worker_token, worker_pid) return 0 if result.get("ok") is True else 1 @@ -1148,11 +1553,26 @@ def run_check(args: argparse.Namespace) -> Dict[str, Any]: plugin_status = None # type: Optional[Dict[str, Any]] plugin_auto_install = None # type: Optional[bool] + reconnect_ready = args.transport == "code" + reconnect_blockers = [] # type: List[str] if args.transport == "aliyun_cli" and cli_execution_mode == "remote": resolve_aliyun(args.aliyun_path) current_profile = {"configured": True, "mode": "RemoteSandbox"} cli = "aliyun" version = None + executor_version = sanitize_text(os.environ.get(REMOTE_EXECUTOR_VERSION_ENV, ""), 120) + raw_capabilities = os.environ.get(REMOTE_EXECUTOR_CAPABILITIES_ENV, "") + executor_capabilities = { + value.strip() for value in raw_capabilities.split(",") if value.strip() + } + reconnect_ready = bool( + executor_version + and REMOTE_BOOTSTRAP_CAPABILITY in executor_capabilities + ) + if not executor_version: + reconnect_blockers.append("remote_executor_version_unavailable") + if REMOTE_BOOTSTRAP_CAPABILITY not in executor_capabilities: + reconnect_blockers.append("remote_bootstrap_capability_unavailable") elif args.transport == "code" and not args.profile_pinned: assert sdk is not None region_id = _environment_region() or "cn-hangzhou" @@ -1202,6 +1622,9 @@ def run_check(args: argparse.Namespace) -> Dict[str, Any]: version = sanitize_text((version_result.stdout or b"").decode("utf-8", "replace"), 200) plugin_status = _local_ros_plugin_status() plugin_auto_install = bool(selected.get("autoPluginInstall")) + reconnect_ready = plugin_status.get("reconnectReady") is True + if not reconnect_ready: + reconnect_blockers.append("ros_cli_plugin_reconnect_version_unavailable") result = { "ok": True, @@ -1215,18 +1638,22 @@ def run_check(args: argparse.Namespace) -> Dict[str, Any]: "enableThinking": args.enable_thinking, "aliyunCLIProfile": args.aliyun_cli_profile, "currentProfile": current_profile, + "startChatReconnectReady": reconnect_ready, } # type: Dict[str, Any] + if reconnect_blockers: + result["startChatReconnectBlockers"] = reconnect_blockers if args.transport == "aliyun_cli" and cli_execution_mode == "remote": result["aliyunCLIForwardEnv"] = args.aliyun_cli_forward_env result["aliyunCLIForwardEnvPresent"] = [ name for name in args.aliyun_cli_forward_env if os.environ.get(name) is not None ] + if executor_version: + result["remoteExecutorVersion"] = executor_version + result["remoteExecutorCapabilities"] = sorted(executor_capabilities) if plugin_status is not None: result["rosPluginReady"] = plugin_status["ready"] result["pluginAutoInstallEnabled"] = plugin_auto_install - result["pluginInstallRequired"] = bool(plugin_status["installed"] and not plugin_status["ready"]) or bool( - not plugin_status["installed"] and not plugin_auto_install - ) + result["pluginInstallRequired"] = plugin_status.get("reconnectReady") is not True if plugin_status.get("version"): result["rosPluginVersion"] = plugin_status["version"] return result diff --git a/skills/alicloud-ros-agent/scripts/ros_agent.py b/skills/alicloud-ros-agent/scripts/ros_agent.py index 576626d01..d960648d4 100644 --- a/skills/alicloud-ros-agent/scripts/ros_agent.py +++ b/skills/alicloud-ros-agent/scripts/ros_agent.py @@ -32,6 +32,10 @@ MAX_PLUGIN_MANIFEST_BYTES = 2 * 1024 * 1024 MAX_SSE_LINE_BYTES = 16 * 1024 * 1024 MAX_SSE_EVENT_BYTES = 16 * 1024 * 1024 +MAX_SERVER_REPLAY_BYTES = 64 * 1024 * 1024 +# A JSON wrapper can expand a server event through escaping. This is a parser +# bound, not a claim about the as-yet unpublished remote executor contract. +MAX_CLI_BATCH_BYTES = (6 * MAX_SERVER_REPLAY_BYTES) + (1024 * 1024) MAX_FINAL_TEXT_BYTES = 10 * 1024 MAX_DIAGNOSTIC_BYTES = 64 * 1024 MAX_RESULT_BYTES = 32 * 1024 @@ -48,6 +52,7 @@ MANAGER_START_TIMEOUT_SECONDS = 10.0 STOP_SESSION_WAIT_SECONDS = 10.0 STOP_REQUEST_TIMEOUT_SECONDS = 60.0 +MAX_RECONNECT_BACKOFF_SECONDS = 5.0 MANAGER_IDLE_SECONDS = 60 MAX_MANAGER_IDLE_SECONDS = 24 * 60 * 60 MANAGER_SCHEMA_VERSION = 3 @@ -69,6 +74,13 @@ SKILL_NAME = "alicloud-ros-agent" USER_AGENT_TEMPLATE = "AlibabaCloud-Agent-Skills/alicloud-ros-agent" REQUIREMENTS_FILE = "requirements-code.txt" +REMOTE_EXECUTOR_VERSION_ENV = "ALICLOUD_ROS_AGENT_EXECUTOR_VERSION" +REMOTE_EXECUTOR_CAPABILITIES_ENV = "ALICLOUD_ROS_AGENT_EXECUTOR_CAPABILITIES" +MIN_ROS_PLUGIN_VERSION = (0, 9, 1) +REMOTE_BOOTSTRAP_CAPABILITY = "startchat-reconnect-bootstrap-v1" +REMOTE_BOOTSTRAP_INVOCATION_ENV = "ALICLOUD_ROS_AGENT_INVOCATION_ID" +REMOTE_BOOTSTRAP_ACK_FILE_ENV = "ALICLOUD_ROS_AGENT_BOOTSTRAP_ACK_FILE" +REMOTE_BOOTSTRAP_PROTOCOL_ENV = "ALICLOUD_ROS_AGENT_BOOTSTRAP_PROTOCOL" def _skill_user_agent() -> str: diff --git a/tests/a2a_e2e/test_qoder_mcp_reconnect.py b/tests/a2a_e2e/test_qoder_mcp_reconnect.py new file mode 100644 index 000000000..12de6fdd2 --- /dev/null +++ b/tests/a2a_e2e/test_qoder_mcp_reconnect.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import stat +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER_PATH = REPO_ROOT / "scripts" / "a2a" / "e2e" / "reconnect" / "run_qoder_mcp_reconnect.py" +FAKE_CLI_PATH = REPO_ROOT / "scripts" / "a2a" / "e2e" / "reconnect" / "fake_aliyun_cli.py" +MCP_SERVER_PATH = REPO_ROOT / "scripts" / "a2a" / "e2e" / "reconnect" / "aliyun_cli_mcp_server.py" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _runner(): + return _load("qoder_mcp_reconnect_runner", RUNNER_PATH) + + +def _mcp_server(): + return _load("qoder_mcp_reconnect_server", MCP_SERVER_PATH) + + +def test_runner_requires_explicit_real_cloud_opt_in(tmp_path) -> None: + runner = _runner() + + with pytest.raises(SystemExit, match="--allow-real-cloud"): + runner._preflight(SimpleNamespace(allow_real_cloud=False), tmp_path) + + +@pytest.mark.parametrize( + ("value", "expected"), + [("0.9.0", (0, 9, 0)), ("0.9.1", (0, 9, 1)), ("0.10.0", (0, 10, 0)), ("latest", None)], +) +def test_runner_parses_stable_ros_plugin_versions(value, expected) -> None: + assert _runner()._plugin_version(value) == expected + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("python /tmp/alicloud-ros-agent/scripts/ros_agent.py check", ["check"]), + ("ALICLOUD_ROS_AGENT_STATE_DIR=/tmp/state python '/tmp/ros_agent.py' start --mode normal", ["start"]), + ("python /tmp/ros_agent.py follow --job-id fixture", ["follow"]), + ( + "python /tmp/ros_agent.py start --mode normal; python /tmp/ros_agent.py follow --job-id fixture", + ["start", "follow"], + ), + ("rg -n ros_agent.py /tmp/alicloud-ros-agent/SKILL.md", []), + ("sed -n '1,80p' /tmp/alicloud-ros-agent/scripts/ros_agent.py", []), + ], +) +def test_runner_counts_only_actual_bridge_invocations(command, expected) -> None: + _command, subcommands = _runner()._bridge_subcommands( + {"type": "tool_use", "name": "Bash", "input": {"command": command}} + ) + + assert subcommands == expected + + +def test_installed_skill_is_temporarily_patched_to_remote_fake_cli(tmp_path) -> None: + runner = _runner() + skill_root = tmp_path / "qoder-skills" + destination = skill_root / "alicloud-ros-agent" + destination.mkdir(parents=True) + (destination / "old.txt").write_text("old", encoding="utf-8") + fake_cli = tmp_path / "fake-aliyun" + fake_cli.write_text("#!/bin/sh\n", encoding="utf-8") + + backups = runner._install_patched_skill(REPO_ROOT, [skill_root], fake_cli) + + config = json.loads((destination / "config.json").read_text(encoding="utf-8")) + runtime = (destination / "scripts" / "_ros_agent_runtime.py").read_text(encoding="utf-8") + assert config["transport"] == "aliyun_cli" + assert config["aliyunCLIExecutionMode"] == "remote" + assert config["endpoint"] == "ros-pre.aliyuncs.com" + assert config["allowedAgentModes"] == ["normal"] + assert config["aliyunCLIForwardEnv"] == list(runner.FORWARDED_ENV) + assert runtime.count(str(fake_cli.resolve())) == 2 + + runner._restore_skills(backups) + + assert (destination / "old.txt").read_text(encoding="utf-8") == "old" + assert not (destination / "config.json").exists() + + +@pytest.mark.skipif( + not hasattr(os, "chflags") or not hasattr(stat, "UF_IMMUTABLE"), + reason="user immutable flags are not supported", +) +def test_installed_skill_restores_user_immutable_files(tmp_path) -> None: + runner = _runner() + skill_root = tmp_path / "qoder-skills" + destination = skill_root / "alicloud-ros-agent" + destination.mkdir(parents=True) + config = destination / "config.json" + config.write_text('{"original":true}\n', encoding="utf-8") + original_flags = config.stat().st_flags + os.chflags(config, original_flags | stat.UF_IMMUTABLE, follow_symlinks=False) + fake_cli = tmp_path / "fake-aliyun" + fake_cli.write_text("#!/bin/sh\n", encoding="utf-8") + + try: + backups = runner._install_patched_skill(REPO_ROOT, [skill_root], fake_cli) + assert json.loads(config.read_text(encoding="utf-8"))["transport"] == "aliyun_cli" + + runner._restore_skills(backups) + + assert json.loads(config.read_text(encoding="utf-8")) == {"original": True} + assert config.stat().st_flags & stat.UF_IMMUTABLE + finally: + if config.exists(): + os.chflags(config, config.stat().st_flags & ~stat.UF_IMMUTABLE, follow_symlinks=False) + + +def _request( + invocation: int, + *, + outcome: str, + query: bool, + session: bool, + cursor_sequence: int | None, + bootstrap_acked: bool = True, +) -> dict[str, object]: + return { + "operation": "start-chat", + "invocation": invocation, + "hasQuery": query, + "hasSessionId": session, + "sessionHash": "session" if session else None, + "hasCursor": cursor_sequence is not None, + "streamOptionsInBody": cursor_sequence is not None, + "cursorIdentityHash": "stream" if cursor_sequence is not None else None, + "cursorSequence": cursor_sequence, + "action": "Reconnect" if cursor_sequence is not None else None, + "endpointVerified": True, + "profileApplied": True, + "bootstrapAcked": bootstrap_acked, + "outcome": outcome, + } + + +@pytest.mark.parametrize( + ("scenario", "requests", "final_text"), + [ + ("normal", [_request(1, outcome="success", query=True, session=False, cursor_sequence=None)], "marker"), + ( + "first-call-timeout", + [ + _request(1, outcome="timeout", query=True, session=False, cursor_sequence=None), + _request(2, outcome="success", query=False, session=True, cursor_sequence=1), + ], + "xxx1 xxx2 xxx3", + ), + ( + "reconnect-call-timeout", + [ + _request(1, outcome="connection-reset", query=True, session=False, cursor_sequence=None), + _request(2, outcome="timeout", query=False, session=True, cursor_sequence=1), + _request(3, outcome="success", query=False, session=True, cursor_sequence=2), + ], + "xxx1 xxx2 xxx3", + ), + ], +) +def test_scenario_checks_cover_normal_first_timeout_and_reconnect_timeout( + scenario, requests, final_text +) -> None: + runner = _runner() + + checks = runner._scenario_checks(scenario, {"requests": requests}, final_text, "marker") + + assert checks + assert all(checks.values()) + + +def test_mcp_server_requires_reconnect_stream_options_in_body(tmp_path) -> None: + server = _mcp_server() + state_path = tmp_path / "state.json" + server._start_request( + state_path, + [ + "ros", + "start-chat", + "--endpoint", + "ros-pre.aliyuncs.com", + "--query", + "fixture", + "--agent-version", + "V2", + ], + ) + body = json.dumps( + {"StreamOptions.Action": "Reconnect", "StreamOptions.Cursor": "v1.stream.3"}, + separators=(",", ":"), + ) + _index, request = server._start_request( + state_path, + [ + "ros", + "start-chat", + "--endpoint", + "ros-pre.aliyuncs.com", + "--agent-version", + "V2", + "--session-id", + "session-1", + "--body", + body, + ], + ) + + assert request["streamOptionsInBody"] is True + assert request["action"] == "Reconnect" + assert request["cursorSequence"] == 3 + assert server._valid_start_shape(request) is True + + with pytest.raises(ValueError, match="must be sent in --body"): + server._start_request( + tmp_path / "legacy.json", + [ + "ros", + "start-chat", + "--endpoint", + "ros-pre.aliyuncs.com", + "--session-id", + "session-1", + "--stream-options", + "Action=Reconnect", + "Cursor=v1.stream.3", + ], + ) + + +def _real_cli_fixture(tmp_path: Path, *, sleep_seconds: float) -> Path: + path = tmp_path / "real-aliyun-fixture.py" + path.write_text( + "#!/usr/bin/env python3\n" + "import json, sys, time\n" + "assert sys.argv[1:3] == ['ros', 'start-chat']\n" + "assert sys.argv[sys.argv.index('--endpoint') + 1] == 'ros-pre.aliyuncs.com'\n" + "assert sys.argv[sys.argv.index('--profile') + 1] == 'test-guima'\n" + "print(json.dumps({'id':'v1.fixture.1','data':{'contextId':'session-fixture','value':'one'}}), flush=True)\n" + f"time.sleep({sleep_seconds!r})\n" + + "print(json.dumps({'id':'v1.fixture.2','data':{'contextId':'session-fixture','value':'two'}}), flush=True)\n", + encoding="utf-8", + ) + path.chmod(0o700) + return path + + +def _run_fake_cli(tmp_path: Path, *, timeout_seconds: float, fixture_sleep: float): + ack = tmp_path / "ack.json" + state = tmp_path / "state.json" + real_cli = _real_cli_fixture(tmp_path, sleep_seconds=fixture_sleep) + env = os.environ.copy() + env.update( + { + "IAC_CODE_E2E_MCP_SERVER": str(MCP_SERVER_PATH), + "IAC_CODE_E2E_REAL_ALIYUN": str(real_cli), + "IAC_CODE_E2E_CLI_IDENTITY": "test-guima", + "IAC_CODE_E2E_MCP_TIMEOUT_SECONDS": str(timeout_seconds), + "IAC_CODE_E2E_SCENARIO": "first-call-timeout" if fixture_sleep > timeout_seconds else "normal", + "IAC_CODE_E2E_SCENARIO_STATE": str(state), + "IAC_CODE_E2E_MCP_STDERR_LOG": str(tmp_path / "mcp.log"), + "IAC_CODE_E2E_PYTHON": sys.executable, + "IAC_CODE_E2E_BOOTSTRAP_ACK_TIMEOUT_SECONDS": "5", + "ALICLOUD_ROS_AGENT_INVOCATION_ID": "invocation-fixture", + "ALICLOUD_ROS_AGENT_BOOTSTRAP_ACK_FILE": str(ack), + "ALICLOUD_ROS_AGENT_BOOTSTRAP_PROTOCOL": "startchat-reconnect-bootstrap-v1", + } + ) + process = subprocess.Popen( + [ + sys.executable, + str(FAKE_CLI_PATH), + "ros", + "start-chat", + "--endpoint", + "ros-pre.aliyuncs.com", + "--query", + "fixture", + "--agent-version", + "V2", + ], + cwd=REPO_ROOT, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + assert process.stdout is not None + with ThreadPoolExecutor(max_workers=1) as pool: + bootstrap_line = pool.submit(process.stdout.readline).result(timeout=15) + bootstrap = json.loads(bootstrap_line) + ack.write_text( + json.dumps( + { + "invocationId": bootstrap["invocationId"], + "eventId": bootstrap["event"]["id"], + "committed": True, + } + ), + encoding="utf-8", + ) + stdout, stderr = process.communicate(timeout=20) + return process.returncode, bootstrap_line + stdout, stderr, json.loads(state.read_text(encoding="utf-8")) + + +def test_fake_cli_uses_real_stdio_mcp_progress_and_ack_before_returning_cli_output(tmp_path) -> None: + return_code, stdout, stderr, state = _run_fake_cli(tmp_path, timeout_seconds=5, fixture_sleep=0) + + values = [json.loads(line) for line in stdout.splitlines()] + assert return_code == 0 + assert stderr == "" + assert values[0]["invocationId"] == "invocation-fixture" + assert values[0]["event"]["id"] == "v1.fixture.1" + assert [value["id"] for value in values[1:]] == ["v1.fixture.1", "v1.fixture.2"] + assert state["requests"][0]["bootstrapAcked"] is True + assert state["requests"][0]["outcome"] == "success" + + +def test_mcp_timeout_discards_ordinary_result_but_preserves_committed_bootstrap(tmp_path) -> None: + return_code, stdout, stderr, state = _run_fake_cli(tmp_path, timeout_seconds=1, fixture_sleep=3) + + assert return_code == 1 + assert len(stdout.splitlines()) == 1 + assert json.loads(stdout)["event"]["id"] == "v1.fixture.1" + assert json.loads(stderr)["code"] == "ExecutorTimeout" + assert state["requests"][0]["bootstrapAcked"] is True + assert state["requests"][0]["outcome"] == "timeout" diff --git a/tests/skill_bridge/start_chat_relay.py b/tests/skill_bridge/start_chat_relay.py index 08471af37..6e8ed8e6b 100644 --- a/tests/skill_bridge/start_chat_relay.py +++ b/tests/skill_bridge/start_chat_relay.py @@ -35,6 +35,8 @@ "EnableThinking", "RegionId", "ClientContext", + "StreamOptions.Action", + "StreamOptions.Cursor", } ) STOP_CHAT_PARAMETERS = frozenset({"SessionId", "AgentVersion"}) @@ -89,10 +91,39 @@ def _single_value_parameters(path: str, body: bytes) -> dict[str, str]: return {key: values[0] for key, values in combined.items()} +def _stream_options_body_parameters(body: bytes) -> dict[str, str]: + try: + value = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise StartChatRequestError("InvalidParameter", "StartChat JSON body is invalid.") from exc + allowed = {"StreamOptions.Action", "StreamOptions.Cursor"} + if not isinstance(value, dict) or not value or set(value) - allowed: + raise StartChatRequestError("InvalidParameter.StreamOptions", "StreamOptions body is invalid.") + parameters: dict[str, str] = {} + for key in sorted(allowed): + item = value.get(key) + if item is not None: + if not isinstance(item, str): + raise StartChatRequestError( + "InvalidParameter.{}".format(key), + "{} must be a string.".format(key), + ) + parameters[key] = item + return parameters + + def parse_start_chat_request(path: str, body: bytes, headers: Any) -> dict[str, str]: """Validate the exact OpenAPI request surface and return business parameters.""" - parameters = _single_value_parameters(path, body) + if body.lstrip().startswith(b"{"): + parameters = _single_value_parameters(path, b"") + body_parameters = _stream_options_body_parameters(body) + repeated = sorted(set(parameters) & set(body_parameters)) + if repeated: + raise StartChatRequestError("InvalidParameter", "Repeated parameter: {}".format(repeated[0])) + parameters.update(body_parameters) + else: + parameters = _single_value_parameters(path, body) action = parameters.get("Action") or headers.get("x-acs-action") if action != "StartChat": raise StartChatRequestError("InvalidAction", "Action must be StartChat.") @@ -105,9 +136,29 @@ def parse_start_chat_request(path: str, body: bytes, headers: Any) -> dict[str, ) if unknown: raise StartChatRequestError("InvalidParameter", "Unknown StartChat parameter: {}".format(unknown[0])) - query = parameters.get("Query") - if query is None or not query.strip(): - raise StartChatRequestError("InvalidParameter.Query", "Query is required.") + stream_action = parameters.get("StreamOptions.Action") + reconnecting = stream_action == "Reconnect" + if stream_action not in (None, "Reconnect"): + raise StartChatRequestError("InvalidParameter.StreamOptions.Action", "Stream action is not supported.") + if reconnecting: + allowed = {"AgentVersion", "SessionId", "StreamOptions.Action", "StreamOptions.Cursor"} + extra = sorted(key for key in parameters if key in START_CHAT_PARAMETERS and key not in allowed) + if extra: + raise StartChatRequestError( + "InvalidParameter.StreamOptions.Action", + "Reconnect includes a business parameter: {}".format(extra[0]), + ) + if not parameters.get("SessionId"): + raise StartChatRequestError("InvalidParameter.SessionId", "SessionId is required for Reconnect.") + cursor = parameters.get("StreamOptions.Cursor") + if not isinstance(cursor, str) or re.fullmatch(r"v1\..+\.[1-9][0-9]*", cursor) is None: + raise StartChatRequestError( + "InvalidParameter.StreamOptions.Cursor", "A complete stream cursor is required." + ) + else: + query = parameters.get("Query") + if query is None or not query.strip(): + raise StartChatRequestError("InvalidParameter.Query", "Query is required.") mode = parameters.get("Mode", "IaCCodeNormal") if mode not in _MODES: raise StartChatRequestError("InvalidParameter.Mode", "Mode is not supported.") @@ -288,6 +339,9 @@ class _UpstreamCall: acknowledged_input_ids: set[str] = field(default_factory=set) thread: threading.Thread | None = None last_task_state: str | None = None + stream_id: str = field(default_factory=lambda: uuid.uuid4().hex) + history: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + next_sequence: int = 1 @dataclass @@ -298,6 +352,7 @@ class _Session: active_call: _UpstreamCall | None = None pending_sideband: dict[str, dict[str, Any]] = field(default_factory=dict) normal_handoff_ready: bool = False + streams: dict[str, _UpstreamCall] = field(default_factory=dict) state_lock: threading.Lock = field(default_factory=threading.Lock) @@ -317,6 +372,8 @@ def __init__( upstream_timeout: float = 15.0, heartbeat_interval: float = 15.0, metrics_path: str | None = None, + disconnect_after_events: int | None = None, + replay_from_head_on_reconnects: int = 0, ) -> None: super().__init__(server_address, _StartChatHandler) self.a2a_url = a2a_url @@ -331,21 +388,32 @@ def __init__( self.metrics_path = pathlib.Path(metrics_path) if metrics_path else None self.metrics_lock = threading.Lock() self.request_metrics: list[dict[str, Any]] = [] + self.disconnect_after_events = disconnect_after_events + self.disconnects_remaining = 1 if disconnect_after_events is not None else 0 + self.replay_from_head_on_reconnects = replay_from_head_on_reconnects ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 self.socket = ssl_context.wrap_socket(self.socket, server_side=True) def begin_request_metric(self, session: _Session, parameters: dict[str, str]) -> dict[str, Any]: + reconnecting = parameters.get("StreamOptions.Action") == "Reconnect" + query = parameters.get("Query", "") metric: dict[str, Any] = { "action": "StartChat", "startedAtUnixMs": int(time.time() * 1000), "sessionId": session.session_id, - "mode": parameters.get("Mode", "IaCCodeNormal"), - "queryBytes": len(parameters["Query"].encode("utf-8")), - "queryKind": "permission" if _permission_query(parameters["Query"]) is not None else "conversation", + "mode": session.mode, + "queryBytes": len(query.encode("utf-8")), + "queryKind": "reconnect" + if reconnecting + else "permission" + if _permission_query(query) is not None + else "conversation", "returnedEventCount": 0, "returnedSseBytes": 0, "eventKinds": {}, } + if reconnecting: + metric["streamCursor"] = parameters["StreamOptions.Cursor"] with self.metrics_lock: self.request_metrics.append(metric) return metric @@ -386,7 +454,7 @@ def resolve_session(self, parameters: dict[str, str]) -> tuple[_Session, bool]: session = self.sessions.get(requested) if session is None: raise StartChatRequestError("SessionNotFound", "The requested SessionId does not exist.") - if session.mode != mode: + if parameters.get("StreamOptions.Action") != "Reconnect" and session.mode != mode: raise StartChatRequestError("InvalidParameter.Mode", "A session cannot change mode.") return session, False session_id = str(uuid.uuid4()) @@ -430,6 +498,8 @@ def start_a2a_call(self, session: _Session, parameters: dict[str, str]) -> _Upst }, } call = _UpstreamCall() + with session.state_lock: + session.streams[call.stream_id] = call upstream_url = self.pipeline_a2a_url if session.mode == "IaCCodePipeline" else self.a2a_url call.thread = threading.Thread( target=self._consume_a2a, @@ -625,6 +695,17 @@ def _do_start_chat(self, body: bytes) -> None: self.close_connection = True self._active_session = session + if parameters.get("StreamOptions.Action") == "Reconnect": + cursor = parameters["StreamOptions.Cursor"] + identity, _separator, raw_sequence = cursor.rpartition(".") + stream_id = identity[3:] + with session.state_lock: + call = session.streams.get(stream_id) + if call is None: + self._request_metric["errorCode"] = "InvalidParameter.StreamOptions.Cursor" + return + self._relay_reconnect(call, int(raw_sequence)) + return permission_payload = _permission_query(parameters["Query"]) if permission_payload is not None: input_id = permission_payload.get("inputId") @@ -688,12 +769,36 @@ def _do_stop_chat(self, body: bytes) -> None: self.wfile.write(response) self.close_connection = True - def _write_event(self, event: dict[str, Any]) -> bool: + def _write_event( + self, + event: dict[str, Any], + call: _UpstreamCall | None = None, + event_id: str | None = None, + ) -> bool: try: data = json.dumps(event, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - self.wfile.write(b"data: " + data + b"\n\n") + if str(event.get("object", "")).lower() in {"heartbeat", "keepalive"}: + prefix = b"" + else: + if event_id is None: + if call is None: + raise ValueError("business events require a stream") + event_id = "v1.{}.{}".format(call.stream_id, call.next_sequence) + call.next_sequence += 1 + call.history.append((event_id, event)) + prefix = "id: {}\n".format(event_id).encode("utf-8") + self.wfile.write(prefix + b"data: " + data + b"\n\n") self.wfile.flush() - self._observe_returned_event(event, len(data) + 8) + self._observe_returned_event(event, len(prefix) + len(data) + 8) + if ( + self.server.disconnects_remaining > 0 + and self.server.disconnect_after_events is not None + and self._request_metric is not None + and self._request_metric.get("queryKind") != "reconnect" + and self._request_metric["returnedEventCount"] >= self.server.disconnect_after_events + ): + self.server.disconnects_remaining -= 1 + return False return True except (BrokenPipeError, ConnectionResetError): return False @@ -703,13 +808,13 @@ def _relay_until_end(self, call: _UpstreamCall) -> bool: try: item = call.events.get(timeout=self.server.heartbeat_interval) except queue.Empty: - if not self._write_event({"object": "heartbeat"}): + if not self._write_event({"object": "heartbeat"}, call): return False continue if item is _END: return True assert isinstance(item, dict) - if not self._write_event(item): + if not self._write_event(item, call): return False def _relay_until_serial_boundary(self, call: _UpstreamCall) -> bool: @@ -717,17 +822,29 @@ def _relay_until_serial_boundary(self, call: _UpstreamCall) -> bool: try: item = call.events.get(timeout=self.server.heartbeat_interval) except queue.Empty: - if not self._write_event({"object": "heartbeat"}): + if not self._write_event({"object": "heartbeat"}, call): return False continue if item is _END: return True assert isinstance(item, dict) - if not self._write_event(item): + if not self._write_event(item, call): return False if _is_serial_input_boundary(item): return False + def _relay_reconnect(self, call: _UpstreamCall, sequence: int) -> bool: + if self.server.replay_from_head_on_reconnects > 0: + self.server.replay_from_head_on_reconnects -= 1 + sequence = 0 + for event_id, event in list(call.history): + _identity, _separator, raw_sequence = event_id.rpartition(".") + if int(raw_sequence) <= sequence: + continue + if not self._write_event(event, call, event_id): + return False + return self._relay_until_end(call) + def _observe_returned_event(self, event: dict[str, Any], wire_bytes: int) -> None: if self._request_metric is None: return diff --git a/tests/skill_bridge/test_alicloud_ros_agent_bridge.py b/tests/skill_bridge/test_alicloud_ros_agent_bridge.py index 9b7c778e8..797cea342 100644 --- a/tests/skill_bridge/test_alicloud_ros_agent_bridge.py +++ b/tests/skill_bridge/test_alicloud_ros_agent_bridge.py @@ -53,6 +53,25 @@ def _write_fake_aliyun(tmp_path: Path, source: str) -> Path: return launcher +class _FakeCLIProcess: + def __init__(self, output: str, return_code: int = 0) -> None: + self.stdout = io.StringIO(output) + self.return_code = return_code + + def wait(self, timeout=None): + del timeout + return self.return_code + + def poll(self): + return self.return_code + + def terminate(self): + return None + + def kill(self): + return None + + def _clear_code_credential_env(monkeypatch) -> None: for name in ( "ALIBABA_CLOUD_ACCESS_KEY_ID", @@ -92,6 +111,10 @@ def _chat_args(**overrides): return argparse.Namespace(**values) +def _command_body(command): + return json.loads(command[command.index("--body") + 1]) + + def _status_event(*, state="TASK_STATE_WORKING", text="", metadata=None): message = {"role": "ROLE_AGENT", "parts": [{"text": text}]} if text else None status = {"state": state} @@ -547,7 +570,8 @@ def test_check_returns_safe_default_chain_and_effective_skill_policy(monkeypatch "managerIdleSeconds": bridge.MANAGER_IDLE_SECONDS, "enableThinking": True, "aliyunCLIProfile": "", - "currentProfile": {"configured": True, "mode": "DefaultCredentialChain", "regionId": "cn-hangzhou"}, + "currentProfile": {"configured": True, "mode": "DefaultCredentialChain", "regionId": "cn-hangzhou"}, + "startChatReconnectReady": True, } assert captured == { "sdk": {"sdk": True}, @@ -719,7 +743,7 @@ def fake_run(command, **_kwargs): monkeypatch.setattr( bridge, "_local_ros_plugin_status", - lambda: {"installed": True, "ready": True, "version": "0.7.2"}, + lambda: {"installed": True, "ready": True, "reconnectReady": True, "version": "0.9.1"}, ) monkeypatch.setattr(bridge.subprocess, "run", fake_run) monkeypatch.setattr(bridge, "_load_code_sdk", lambda: pytest.fail("CLI transport must not load SDK packages")) @@ -734,15 +758,16 @@ def fake_run(command, **_kwargs): assert result["rosPluginReady"] is True assert result["pluginInstallRequired"] is False assert result["pluginAutoInstallEnabled"] is False - assert result["rosPluginVersion"] == "0.7.2" + assert result["rosPluginVersion"] == "0.9.1" @pytest.mark.parametrize( ("plugin_status", "auto_install", "install_required"), [ ({"installed": False, "ready": False}, False, True), - ({"installed": False, "ready": False}, True, False), + ({"installed": False, "ready": False}, True, True), ({"installed": True, "ready": False, "version": "0.7.1"}, True, True), + ({"installed": True, "ready": True, "reconnectReady": False}, False, True), ], ) def test_local_cli_check_reports_when_skill_must_install_ros_plugin( @@ -770,7 +795,7 @@ def test_local_cli_check_reports_when_skill_must_install_ros_plugin( result = bridge.run_check(args) - assert result["rosPluginReady"] is False + assert result["rosPluginReady"] is plugin_status["ready"] assert result["pluginAutoInstallEnabled"] is auto_install assert result["pluginInstallRequired"] is install_required @@ -867,7 +892,12 @@ def test_local_ros_plugin_status_requires_binary_and_start_stop_commands(monkeyp ) monkeypatch.setenv("ALIBABA_CLOUD_CLI_PLUGINS_DIR", str(tmp_path)) - assert bridge._local_ros_plugin_status() == {"installed": True, "ready": True, "version": "0.7.2"} + assert bridge._local_ros_plugin_status() == { + "installed": True, + "ready": True, + "reconnectReady": False, + "version": "0.7.2", + } manifest.write_text( json.dumps( @@ -883,7 +913,28 @@ def test_local_ros_plugin_status_requires_binary_and_start_stop_commands(monkeyp ), encoding="utf-8", ) - assert bridge._local_ros_plugin_status() == {"installed": True, "ready": False, "version": "0.7.1"} + assert bridge._local_ros_plugin_status() == { + "installed": True, + "ready": False, + "reconnectReady": False, + "version": "0.7.1", + } + + manifest.write_text( + json.dumps( + { + "plugins": { + "aliyun-cli-ros": { + "version": "0.9.1", + "path": str(plugin_root), + "cmdNames": ["start-chat", "stop-chat"], + } + } + } + ), + encoding="utf-8", + ) + assert bridge._local_ros_plugin_status()["reconnectReady"] is True def test_cli_transport_rejects_client_context_and_remote_profile() -> None: @@ -1115,17 +1166,19 @@ def test_sse_parser_handles_heartbeats_multiline_data_and_raw_json() -> None: ": comment\n", 'data: {"object":"heartbeat"}\n', "\n", + "id: v1.stream.with.dots.1\n", 'data: {"value":\n', "data: 1}\n", "\n", '{"result":{"ok":true}}\n', ] events = list(bridge.iter_sse_payloads(lines)) - assert [event[0] for event in events] == [ + assert [event["payload"] for event in events] == [ {"object": "heartbeat"}, {"value": 1}, {"result": {"ok": True}}, ] + assert [event["id"] for event in events] == [None, "v1.stream.with.dots.1", None] def test_cli_plugin_parser_streams_and_unwraps_each_json_line() -> None: @@ -1134,33 +1187,38 @@ def test_cli_plugin_parser_streams_and_unwraps_each_json_line() -> None: events = list( bridge.iter_cli_plugin_payloads( - [ - json.dumps({"data": first}) + "\n", - json.dumps({"data": second}) + "\n", + [ + json.dumps({"id": "v1.stream.1", "data": first}) + "\n", + json.dumps({"id": "v1.stream.2", "data": second}) + "\n", ] ) ) - assert events == [(first, json.dumps({"data": first})), (second, json.dumps({"data": second}))] + assert [(event["id"], event["payload"]) for event in events] == [ + ("v1.stream.1", first), + ("v1.stream.2", second), + ] def test_cli_plugin_parser_accepts_pretty_printed_objects_and_arrays() -> None: first = {"result": {"statusUpdate": {"status": {"state": "TASK_STATE_WORKING"}}}} second = {"result": {"statusUpdate": {"status": {"state": "TASK_STATE_COMPLETED"}}}} - pretty_object = json.dumps({"data": first}, indent=2).splitlines(keepends=True) - pretty_array = json.dumps([{"data": first}, {"data": second}], indent=2).splitlines(keepends=True) + pretty_object = json.dumps({"id": "v1.stream.1", "data": first}, indent=2).splitlines(keepends=True) + pretty_array = json.dumps( + [{"id": "v1.stream.1", "data": first}, {"id": "v1.stream.2", "data": second}], indent=2 + ).splitlines(keepends=True) object_events = list(bridge.iter_cli_plugin_payloads(pretty_object)) array_events = list(bridge.iter_cli_plugin_payloads(pretty_array)) - assert [value for value, _raw in object_events] == [first] - assert [value for value, _raw in array_events] == [first, second] + assert [event["payload"] for event in object_events] == [first] + assert [event["payload"] for event in array_events] == [first, second] def test_cli_plugin_parser_reports_an_unterminated_buffer_as_malformed() -> None: events = list(bridge.iter_cli_plugin_payloads(['{\n', ' "data": {\n'])) - assert events == [(None, '{\n "data": {')] + assert events == [{"id": None, "payload": None, "raw": '{\n "data": {'}] def test_sse_parser_rejects_an_unterminated_event_as_soon_as_its_cumulative_limit_is_exceeded( @@ -1958,6 +2016,7 @@ def test_run_chat_consumes_fake_cli_stream_without_network(monkeypatch, tmp_path prompt.write_text("hello", encoding="utf-8") output = json.dumps( { + "id": "v1.stream.1", "data": _status_event( state="TASK_STATE_INPUT_REQUIRED", text="done", @@ -2033,7 +2092,13 @@ class FakeResponse: def __init__(self): self.closed = False - self.lines = iter([("data: " + json.dumps(event, separators=(",", ":")) + "\n\n").encode()]) + self.lines = iter( + [ + b"id: v1.stream.1\n", + ("data: " + json.dumps(event, separators=(",", ":")) + "\n").encode(), + b"\n", + ] + ) def __iter__(self): return self @@ -2365,9 +2430,7 @@ def test_run_respond_sends_json_as_the_only_start_chat_control_payload(monkeypat ), encoding="utf-8", ) - output = json.dumps( - { - "data": { + acknowledgement = { "result": { "messageId": "permission-ack-1", "taskId": "task-1", @@ -2386,9 +2449,14 @@ def test_run_respond_sends_json_as_the_only_start_chat_control_payload(monkeypat }, } ], - }, + } } - }, + completed = _status_event(state="TASK_STATE_COMPLETED") + output = json.dumps( + [ + {"id": "v1.response.1", "data": acknowledgement}, + {"id": "v1.response.2", "data": completed}, + ], separators=(",", ":"), ) output += "\n" @@ -2438,7 +2506,7 @@ def fake_popen(command, **kwargs): assert query_text.startswith(bridge.PERMISSION_QUERY_PREFIX + " ") query = json.loads(query_text[len(bridge.PERMISSION_QUERY_PREFIX) :]) - assert result["state"] == "permission-responded" + assert result["state"] == "completed" assert result["permissionResponse"]["decision"] == "deny" assert query["decision"] == "deny" assert command[command.index("--enable-thinking") + 1] == "false" @@ -2527,7 +2595,7 @@ def test_manager_idle_countdown_starts_after_sse_worker_exits(monkeypatch, tmp_p + "'status': {'state': 'TASK_STATE_INPUT_REQUIRED', 'message': {'role': 'ROLE_AGENT', " + "'parts': [{'text': 'done'}]}}, 'metadata': {'iac_code': {'assistantFinal': " + "{'complete': True}}, 'iacCodeSessionId': 'iac-1'}}}}\n" - + "print(json.dumps({'data': event}), flush=True)\n", + + "print(json.dumps({'id': 'v1.stream.1', 'data': event}), flush=True)\n", encoding="utf-8", ) @@ -2591,8 +2659,11 @@ def test_managed_worker_outlives_start_and_follow_returns_step_start_before_fina tmp_path, "import json, time\n" + "from pathlib import Path\n" + + "counter = 0\n" + "def emit(value):\n" - + " print(json.dumps({'data': value}), flush=True)\n" + + " global counter\n" + + " counter += 1\n" + + " print(json.dumps({'id': 'v1.stream.{}'.format(counter), 'data': value}), flush=True)\n" + "def status(state, text='', metadata=None):\n" + " body = {'state': state}\n" + " if text:\n" @@ -2644,6 +2715,98 @@ def test_managed_worker_outlives_start_and_follow_returns_step_start_before_fina assert not bridge._pid_alive(started["workerPid"]) +def test_managed_remote_bootstrap_is_committed_before_ack_and_recovers_lost_result( + monkeypatch, tmp_path: Path +) -> None: + state_root = tmp_path / "state" + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(state_root)) + workspace = tmp_path / "workspace" + workspace.mkdir() + counter_path = tmp_path / "remote-count" + calls_path = tmp_path / "remote-calls.jsonl" + observation_path = tmp_path / "ack-observation.json" + fake_cli = _write_fake_aliyun( + tmp_path, + "import json, os, sys, time\n" + + "from pathlib import Path\n" + + "counter_path = Path({!r})\n".format(str(counter_path)) + + "calls_path = Path({!r})\n".format(str(calls_path)) + + "observation_path = Path({!r})\n".format(str(observation_path)) + + "count = int(counter_path.read_text() or '0') + 1 if counter_path.exists() else 1\n" + + "counter_path.write_text(str(count))\n" + + "with calls_path.open('a', encoding='utf-8') as handle:\n" + + " handle.write(json.dumps(sys.argv[1:]) + '\\n')\n" + + "def status(state, text='', final=False):\n" + + " body = {'state': state}\n" + + " if text:\n" + + " body['message'] = {'role': 'ROLE_AGENT', 'parts': [{'text': text}]}\n" + + " metadata = {'iac_code': {}, 'iacCodeSessionId': 'iac-remote-1'}\n" + + " if final:\n" + + " metadata['iac_code']['assistantFinal'] = {'complete': True}\n" + + " return {'result': {'statusUpdate': {'taskId': 'task-remote-1', " + + "'contextId': 'session-remote-1', 'status': body, 'metadata': metadata}}}\n" + + "first = status('TASK_STATE_WORKING', 'first')\n" + + "if count == 1:\n" + + " invocation = os.environ['ALICLOUD_ROS_AGENT_INVOCATION_ID']\n" + + " ack_path = Path(os.environ['ALICLOUD_ROS_AGENT_BOOTSTRAP_ACK_FILE'])\n" + + " print(json.dumps({'invocationId': invocation, 'event': " + + "{'id': 'v1.remote-managed.1', 'data': first}, 'sessionId': 'session-remote-1'}), flush=True)\n" + + " deadline = time.monotonic() + 5\n" + + " while not ack_path.exists() and time.monotonic() < deadline:\n" + + " time.sleep(0.01)\n" + + " state_root = Path(os.environ['ALICLOUD_ROS_AGENT_STATE_DIR'])\n" + + " job_path = next((state_root / 'jobs').glob('*/job.json'))\n" + + " spool_path = job_path.with_name('events.jsonl')\n" + + " job = json.loads(job_path.read_text())\n" + + " records = [json.loads(line) for line in spool_path.read_text().splitlines()]\n" + + " ack = json.loads(ack_path.read_text()) if ack_path.exists() else {}\n" + + " observation_path.write_text(json.dumps({'ack': ack, 'sessionId': job.get('sessionId'), " + + "'streamCursor': job.get('streamCursor'), 'spoolIds': " + + "[item.get('streamEventId') for item in records]}))\n" + + " print(json.dumps({'code': 'ExecutorTimeout'}), file=sys.stderr)\n" + + " raise SystemExit(1)\n" + + "completed = status('TASK_STATE_COMPLETED', 'done', True)\n" + + "print(json.dumps([{'id': 'v1.remote-managed.1', 'data': first}, " + + "{'id': 'v1.remote-managed.2', 'data': completed}]), flush=True)\n", + ) + + started = bridge._start_job_local( + { + "workspace": str(workspace), + "prompt": "safe remote reconnect test", + "mode": "normal", + "transport": "aliyun_cli", + "aliyunCLIExecutionMode": "remote", + "endpoint": "ros-pre.aliyuncs.com", + "regionId": "cn-hangzhou", + "aliyunPath": str(fake_cli), + } + ) + result = bridge._follow_job_local(started["jobId"], 0, 10) + + assert result["state"] == "turn-completed" + assert result["finalText"] == "done" + calls = [json.loads(line) for line in calls_path.read_text().splitlines()] + assert len(calls) == 2 + assert "--query" in calls[0] + assert "--query" not in calls[1] + assert "--stream-options" not in calls[1] + assert _command_body(calls[1]) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.remote-managed.1", + } + observation = json.loads(observation_path.read_text()) + assert observation["ack"] == { + "committed": True, + "eventId": "v1.remote-managed.1", + "invocationId": observation["ack"]["invocationId"], + } + assert observation["sessionId"] == "session-remote-1" + assert observation["streamCursor"] == "v1.remote-managed.1" + assert observation["spoolIds"] == ["v1.remote-managed.1"] + _wait_for_pid_exit(started["workerPid"]) + + def test_worker_failed_start_cleans_request_and_marks_job_failed(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) job_id = "0" * 32 @@ -3586,9 +3749,9 @@ def spawn(captured_job_id, request): ("stop_status", "result_state", "persisted_state", "ok"), [ ("Stopped", "canceled", "canceled", True), - ("Stopping", "canceling", "working", True), - ("NoActiveStream", "not-active", "working", True), - ("Failed", "cancel-failed", "working", False), + ("Stopping", "canceling", "canceling", True), + ("NoActiveStream", "not-active", "not-active", True), + ("Failed", "cancel-failed", "failed", False), ], ) def test_cancel_managed_job_calls_stop_chat_and_preserves_authoritative_state( @@ -3646,7 +3809,7 @@ def stop_chat(job, session_id): if stop_status == "Stopped": assert "inputRequired" not in job else: - assert job["inputRequired"]["kind"] == "permission" + assert "inputRequired" not in job or stop_status == "Failed" def test_parser_exposes_managed_commands_without_synchronous_chat() -> None: @@ -3777,7 +3940,13 @@ def test_remote_cli_refreshes_starops_hil_across_manager_proxy_and_buffered_pop_ raise SystemExit(2) print(json.dumps(json.loads(%r), ensure_ascii=False, indent=2)) """ - % json.dumps([{"data": working}, {"data": completed}], ensure_ascii=False), + % json.dumps( + [ + {"id": "v1.remote-stream.1", "data": working}, + {"id": "v1.remote-stream.2", "data": completed}, + ], + ensure_ascii=False, + ), ) front_cli = _write_fake_aliyun( front_dir, @@ -4614,3 +4783,833 @@ def test_follow_result_remains_bounded_with_large_final_text(monkeypatch, tmp_pa result = bridge._job_result(job_id, 0) assert len(bridge._json_bytes(result)) <= bridge.MAX_FOLLOW_BYTES assert result["finalTextComplete"] is False + + +def test_reconnect_builders_send_only_the_stream_anchor(monkeypatch) -> None: + args = _chat_args(session_id="session-1") + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + + assert bridge.build_reconnect_start_chat_parameters("session-1", "v1.stream.9") == { + "AgentVersion": "V2", + "SessionId": "session-1", + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.stream.9", + } + command = bridge.build_reconnect_command(args, "session-1", "v1.stream.9") + assert command[command.index("--session-id") + 1] == "session-1" + assert command.count("--body") == 1 + assert "--stream-options" not in command + assert _command_body(command) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.stream.9", + } + for forbidden in ("--query", "--biz-mode", "--enable-thinking", "--attachments", "--biz-region-id"): + assert forbidden not in command + + +@pytest.mark.parametrize( + ("event_id", "expected"), + [ + ("v1.stream.1", ("v1.stream", 1)), + ("v1.stream.with.dots.42", ("v1.stream.with.dots", 42)), + ("v1.stream.0002", ("v1.stream", 2)), + ], +) +def test_stream_event_id_is_parsed_from_the_right_without_rebuilding(event_id, expected) -> None: + assert bridge.parse_stream_event_id(event_id) == expected + + +@pytest.mark.parametrize("event_id", [None, "", "1", "v2.stream.1", "v1..1", "v1.stream.0", "v1.stream.x"]) +def test_invalid_stream_event_id_is_deterministic(event_id) -> None: + with pytest.raises(bridge.BridgeError) as error: + bridge.parse_stream_event_id(event_id) + assert error.value.code == "invalid_stream_event_id" + assert error.value.retryable is False + + +@pytest.mark.parametrize("execution_mode", ["local", "remote"]) +def test_cli_reconnect_keeps_cursor_monotonic_during_repeated_head_replay( + monkeypatch, tmp_path: Path, execution_mode: str +) -> None: + first = _status_event(state="TASK_STATE_WORKING", text="one") + second = _status_event(state="TASK_STATE_WORKING", text="two") + completed = _status_event( + state="TASK_STATE_COMPLETED", text="done", metadata={"assistantFinal": {"complete": True}} + ) + attempts = [ + ( + json.dumps( + [ + {"id": "v1.stream.with.dots.1", "data": first}, + {"id": "v1.stream.with.dots.2", "data": second}, + ] + ), + 1, + ), + (json.dumps({"id": "v1.stream.with.dots.1", "data": first}), 1), + ( + json.dumps( + [ + {"id": "v1.stream.with.dots.1", "data": first}, + {"id": "v1.stream.with.dots.2", "data": second}, + {"id": "v1.stream.with.dots.3", "data": completed}, + ] + ), + 0, + ), + ] + commands = [] + invocation_ids = [] + + def popen(command, **kwargs): + output, return_code = attempts.pop(0) + commands.append(command) + environment = kwargs.get("env") + if isinstance(environment, dict): + invocation_ids.append(environment[bridge.REMOTE_BOOTSTRAP_INVOCATION_ENV]) + if return_code: + kwargs["stderr"].write(b'{"code":"ReadTimeout"}') + return _FakeCLIProcess(output, return_code) + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + args = _chat_args( + transport="aliyun_cli", + aliyun_cli_execution_mode=execution_mode, + session_id=None, + ) + + result = bridge._consume_start_chat(args, tmp_path, "hello", None, []) + + assert result["state"] == "turn-completed" + assert result["finalText"] == "done" + assert result["eventCount"] == 3 + assert len(commands) == 3 + assert "--query" in commands[0] + for command in commands[1:]: + assert "--query" not in command + assert "--stream-options" not in command + assert _command_body(command) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.stream.with.dots.2", + } + if execution_mode == "remote": + assert len(set(invocation_ids)) == 3 + + +def test_code_reconnect_recovers_terminal_tail_without_resending_query(monkeypatch, tmp_path: Path) -> None: + terminal = _status_event(state="TASK_STATE_COMPLETED", text="final") + artifact = { + "result": { + "artifactUpdate": { + "contextId": "session-1", + "artifact": { + "artifactId": "template", + "name": "template.yaml", + "parts": [{"url": "file:///workspace/template.yaml"}], + }, + } + } + } + opened = [] + + class Response: + headers = {"Content-Type": "text/event-stream"} + + def __init__(self, lines, failure=None): + self.lines = iter(lines) + self.failure = failure + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.lines) + except StopIteration: + if self.failure is not None: + failure = self.failure + self.failure = None + raise failure + raise + + def close(self): + return None + + responses = [ + Response( + [ + b"id: v1.tail.1\n", + ("data: " + json.dumps(terminal) + "\n").encode(), + b"\n", + ], + ConnectionResetError("cut"), + ), + Response( + [ + b"id: v1.tail.2\n", + ("data: " + json.dumps(artifact) + "\n").encode(), + b"\n", + ] + ), + ] + + def open_request(_operation, parameters, *_args, **_kwargs): + opened.append(parameters) + return responses.pop(0) + + monkeypatch.setattr(bridge, "_open_code_request", open_request) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + args = _chat_args(transport="code", session_id=None) + + result = bridge._consume_start_chat(args, tmp_path, "hello", None, []) + + assert result["state"] == "turn-completed" + assert result["artifacts"][0]["id"] == "template" + assert opened[0]["Query"] == "hello" + assert opened[1] == { + "AgentVersion": "V2", + "SessionId": "session-1", + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.tail.1", + } + + +def test_remote_bootstrap_is_correlated_committed_and_replayed_idempotently(monkeypatch, tmp_path: Path) -> None: + first = _status_event(state="TASK_STATE_WORKING", text="first") + completed = _status_event( + state="TASK_STATE_COMPLETED", text="done", metadata={"assistantFinal": {"complete": True}} + ) + calls = [] + + def popen(command, **kwargs): + environment = kwargs["env"] + invocation_id = environment[bridge.REMOTE_BOOTSTRAP_INVOCATION_ENV] + calls.append((command, invocation_id, environment[bridge.REMOTE_BOOTSTRAP_ACK_FILE_ENV])) + if len(calls) == 1: + wrong = { + "invocationId": "0" * 32, + "event": {"id": "v1.wrong.1", "data": first}, + "sessionId": "wrong-session", + } + correct = { + "invocationId": invocation_id, + "event": {"id": "v1.bootstrap.1", "data": first}, + "sessionId": "session-1", + } + kwargs["stderr"].write(b'{"code":"ExecutorTimeout"}') + return _FakeCLIProcess(json.dumps(wrong) + "\n" + json.dumps(correct), 1) + output = json.dumps( + [ + {"id": "v1.bootstrap.1", "data": first}, + {"id": "v1.bootstrap.2", "data": completed}, + ] + ) + return _FakeCLIProcess(output) + + acknowledgements = [] + original_atomic_json = bridge._atomic_json + + def atomic_json(path, value, mode=0o600): + if Path(path).name == "bootstrap-ack.json": + acknowledgements.append(dict(value)) + return original_atomic_json(path, value, mode) + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/remote/aliyun") + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + monkeypatch.setattr(bridge, "_atomic_json", atomic_json) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + args = _chat_args(transport="aliyun_cli", aliyun_cli_execution_mode="remote", session_id=None) + + result = bridge._consume_start_chat(args, tmp_path, "hello", None, []) + + assert result["state"] == "turn-completed" + assert result["eventCount"] == 2 + assert result["sessionId"] == "session-1" + assert acknowledgements == [ + {"invocationId": calls[0][1], "eventId": "v1.bootstrap.1", "committed": True} + ] + assert _command_body(calls[1][0]) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.bootstrap.1", + } + assert calls[0][1] != calls[1][1] + + +def test_checkpoint_spools_private_event_id_once_and_keeps_worker_cursors_separate(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) + job_id = "1" * 32 + root, job_path, spool = bridge._job_paths(job_id) + bridge._secure_directory(root) + spool.touch() + bridge._atomic_json( + job_path, + { + "schemaVersion": 1, + "jobId": job_id, + "state": "working", + "mode": "pipeline", + "activeRequestSeq": 1, + "sidebandWorkerToken": "sideband-1", + "artifacts": [], + }, + ) + primary = {"type": "milestone", "requestSeq": 1, "milestones": [{"eventType": "step_started"}]} + sideband = dict(primary, workerRole="sideband", workerToken="sideband-1") + + assert bridge._append_projection(job_id, primary, "v1.primary.1") is True + assert bridge._append_projection(job_id, primary, "v1.primary.1") is True + assert bridge._append_projection(job_id, sideband, "v1.sideband.1") is True + + records = bridge._read_spool(spool) + assert [record["streamEventId"] for record in records] == ["v1.primary.1", "v1.sideband.1"] + job = bridge._load_state_json(job_path) + assert job["streamCursor"] == "v1.primary.1" + assert job["sidebandStreamCursor"] == "v1.sideband.1" + assert "streamEventId" not in bridge._job_result(job_id, 0) + + +def test_cancel_marks_intent_before_waiting_for_a_delayed_session(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) + monkeypatch.setattr(bridge, "STOP_SESSION_WAIT_SECONDS", 0.0) + job_id = "2" * 32 + root, job_path, spool = bridge._job_paths(job_id) + bridge._secure_directory(root) + spool.touch() + bridge._atomic_json( + job_path, + { + "schemaVersion": 1, + "jobId": job_id, + "state": "working", + "mode": "normal", + "activeRequestSeq": 1, + "turn": 1, + "artifacts": [], + }, + ) + + result = bridge._cancel_job_local({"jobId": job_id}) + + assert result["state"] == "canceling" + job = bridge._load_state_json(job_path) + assert isinstance(job["stopRequestedAt"], int) + assert job["state"] == "canceling" + assert "stopDispatchStartedAt" not in job + + +def test_canceled_worker_only_records_delayed_session_and_stop_is_claimed_once(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) + job_id = "4" * 32 + root, job_path, spool = bridge._job_paths(job_id) + bridge._secure_directory(root) + spool.touch() + bridge._atomic_json( + job_path, + { + "schemaVersion": 1, + "jobId": job_id, + "state": "working", + "mode": "normal", + "activeRequestSeq": 1, + "stopRequestedAt": 1, + "artifacts": [], + }, + ) + projection = { + "type": "status", + "requestSeq": 1, + "sessionId": "session-delayed", + "latestText": "must not be published", + } + + assert bridge._append_projection(job_id, projection, "v1.delayed.1") is False + + job = bridge._load_state_json(job_path) + assert job["sessionId"] == "session-delayed" + assert "latestText" not in job + assert "streamCursor" not in job + assert bridge._read_spool(spool) == [] + claimed = bridge._claim_stop_dispatch(job_id, "session-delayed") + assert claimed is not None + assert bridge._claim_stop_dispatch(job_id, "session-delayed") is None + + +def test_canceled_worker_saves_session_and_stops_even_when_first_business_event_has_no_id( + monkeypatch, tmp_path: Path +) -> None: + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) + job_id = "5" * 32 + request_token = "6" * 32 + root, job_path, spool = bridge._job_paths(job_id) + bridge._secure_directory(root) + spool.touch() + bridge._atomic_json( + job_path, + { + "schemaVersion": 1, + "jobId": job_id, + "state": "canceling", + "mode": "normal", + "activeRequestSeq": 1, + "workerPid": os.getpid(), + "stopRequestedAt": 1, + "artifacts": [], + }, + ) + bridge._atomic_json( + root / ("request-{}.json".format(request_token)), + { + "requestSeq": 1, + "workspace": str(tmp_path), + "prompt": "hello", + "mode": "normal", + "summaryMode": "normal", + "endpoint": "ros.aliyuncs.com", + "transport": "aliyun_cli", + "aliyunCLIExecutionMode": "local", + "aliyunCLIForwardEnv": [], + "sessionId": None, + "regionId": "cn-hangzhou", + "profile": None, + "noThinking": False, + "connectTimeout": 10, + "readTimeout": 600, + "aliyunPath": "aliyun", + "clientContext": None, + "attachments": [], + }, + ) + payload = _status_event(state="TASK_STATE_WORKING", text="must not be projected") + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr( + bridge.subprocess, + "Popen", + lambda _command, **_kwargs: _FakeCLIProcess(json.dumps({"data": payload})), + ) + stopped = [] + + def stop_chat(_job, session_id): + stopped.append(session_id) + return {"status": "Stopped", "sessionId": session_id} + + monkeypatch.setattr(bridge, "_run_stop_chat", stop_chat) + + assert bridge.run_worker(job_id, request_token) == 0 + + job = bridge._load_state_json(job_path) + assert stopped == ["session-1"] + assert job["sessionId"] == "session-1" + assert job["state"] == "canceled" + assert "streamCursor" not in job + assert bridge._read_spool(spool) == [] + + +def test_remote_bootstrap_session_mismatch_fails_before_projection_or_ack(monkeypatch, tmp_path: Path) -> None: + payload = _status_event(state="TASK_STATE_WORKING") + payload["result"]["statusUpdate"]["contextId"] = "session-from-payload" + ack_writes = [] + + def popen(_command, **kwargs): + invocation = kwargs["env"][bridge.REMOTE_BOOTSTRAP_INVOCATION_ENV] + output = json.dumps( + { + "invocationId": invocation, + "event": {"id": "v1.session-mismatch.1", "data": payload}, + "sessionId": "session-from-bootstrap", + } + ) + return _FakeCLIProcess(output) + + original_atomic_json = bridge._atomic_json + + def atomic_json(path, value, mode=0o600): + if Path(path).name == "bootstrap-ack.json": + ack_writes.append(dict(value)) + return original_atomic_json(path, value, mode) + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/remote/aliyun") + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + monkeypatch.setattr(bridge, "_atomic_json", atomic_json) + + with pytest.raises(bridge.BridgeError) as error: + bridge._consume_start_chat( + _chat_args(transport="aliyun_cli", aliyun_cli_execution_mode="remote"), + tmp_path, + "hello", + None, + [], + ) + + assert error.value.code == "stream_session_mismatch" + assert ack_writes == [] + + +@pytest.mark.parametrize("failure", ["exception", "failed"]) +def test_stop_chat_failure_is_persisted_as_failed(monkeypatch, tmp_path: Path, failure: str) -> None: + monkeypatch.setenv(bridge.STATE_DIR_ENV, str(tmp_path / "state")) + job_id = "3" * 32 + root, job_path, spool = bridge._job_paths(job_id) + bridge._secure_directory(root) + spool.touch() + bridge._atomic_json( + job_path, + { + "schemaVersion": 1, + "jobId": job_id, + "state": "working", + "mode": "normal", + "sessionId": "session-1", + "activeRequestSeq": 1, + "turn": 1, + "artifacts": [], + }, + ) + + def stop_chat(_job, _session_id): + if failure == "exception": + raise bridge.BridgeError("stop_chat_failed", "timeout", True) + return {"status": "Failed", "sessionId": "session-1"} + + monkeypatch.setattr(bridge, "_run_stop_chat", stop_chat) + + result = bridge._cancel_job_local({"jobId": job_id}) + + assert result["state"] == "cancel-failed" + job = bridge._load_state_json(job_path) + assert job["state"] == "failed" + assert job["stopStatus"] == "Failed" + assert job["error"]["code"] == "stop_chat_failed" + assert job["error"]["retryable"] is True + + +def test_remote_check_exposes_executor_reconnect_release_gate(monkeypatch) -> None: + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/remote/aliyun") + monkeypatch.setenv(bridge.REMOTE_EXECUTOR_VERSION_ENV, "executor-1.2.3") + monkeypatch.setenv( + bridge.REMOTE_EXECUTOR_CAPABILITIES_ENV, + "other", + ) + args = argparse.Namespace(command="check", aliyun_path="aliyun") + bridge.apply_skill_config( + args, + { + "transport": "aliyun_cli", + "aliyunCLIExecutionMode": "remote", + "endpoint": "ros-pre.aliyuncs.com", + }, + ) + + incomplete = bridge.run_check(args) + + assert incomplete["startChatReconnectReady"] is False + assert "remote_bootstrap_capability_unavailable" in incomplete["startChatReconnectBlockers"] + + monkeypatch.setenv( + bridge.REMOTE_EXECUTOR_CAPABILITIES_ENV, + bridge.REMOTE_BOOTSTRAP_CAPABILITY, + ) + result = bridge.run_check(args) + + assert result["startChatReconnectReady"] is True + assert result["remoteExecutorVersion"] == "executor-1.2.3" + assert bridge.REMOTE_BOOTSTRAP_CAPABILITY in result["remoteExecutorCapabilities"] + assert "startChatReconnectBlockers" not in result + + +def test_cli_size_limits_are_distinct_and_enforced_after_wrapper_decode(monkeypatch) -> None: + assert bridge.MAX_CLI_BATCH_BYTES > bridge.MAX_SERVER_REPLAY_BYTES > bridge.MAX_SSE_EVENT_BYTES + monkeypatch.setattr(bridge, "MAX_SSE_EVENT_BYTES", 100) + monkeypatch.setattr(bridge, "MAX_SERVER_REPLAY_BYTES", 150) + monkeypatch.setattr(bridge, "MAX_CLI_BATCH_BYTES", 400) + first = {"contextId": "session-1", "value": "x" * 45} + second = {"contextId": "session-1", "value": "y" * 45} + wrapped = json.dumps( + [{"id": "v1.size.1", "data": first}, {"id": "v1.size.2", "data": second}] + ) + + with pytest.raises(bridge.BridgeError) as error: + list(bridge.iter_cli_plugin_payloads([wrapped])) + assert error.value.code == "server_replay_too_large" + assert error.value.retryable is False + + monkeypatch.setattr(bridge, "MAX_CLI_BATCH_BYTES", 20) + with pytest.raises(bridge.BridgeError) as error: + list(bridge.iter_cli_plugin_payloads([wrapped])) + assert error.value.code == "cli_output_too_large" + + monkeypatch.setattr(bridge, "MAX_CLI_BATCH_BYTES", 100) + small = json.dumps({"id": "v1.size.1", "data": {"value": "x" * 24}}) + "\n" + assert len(small.encode("utf-8")) < bridge.MAX_CLI_BATCH_BYTES + with pytest.raises(bridge.BridgeError) as error: + list(bridge.iter_cli_plugin_payloads([small, small])) + assert error.value.code == "cli_output_too_large" + + +def test_bootstrap_copy_does_not_double_count_server_replay_budget(monkeypatch) -> None: + monkeypatch.setattr(bridge, "MAX_SSE_EVENT_BYTES", 1000) + monkeypatch.setattr(bridge, "MAX_SERVER_REPLAY_BYTES", 180) + monkeypatch.setattr(bridge, "MAX_CLI_BATCH_BYTES", 2000) + payload = {"contextId": "session-1", "value": "x" * 80} + bootstrap = { + "invocationId": "invocation-1", + "event": {"id": "v1.bootstrap-size.1", "data": payload}, + "sessionId": "session-1", + } + normal = [ + {"id": "v1.bootstrap-size.1", "data": payload}, + {"id": "v1.bootstrap-size.2", "data": {"contextId": "session-1", "value": "tail"}}, + ] + + events = list(bridge.iter_cli_plugin_payloads([json.dumps(bootstrap) + "\n", json.dumps(normal)])) + + assert [event["id"] for event in events] == [ + "v1.bootstrap-size.1", + "v1.bootstrap-size.1", + "v1.bootstrap-size.2", + ] + + +@pytest.mark.parametrize("transport", ["code", "aliyun_cli"]) +def test_business_event_without_id_fails_before_projection(monkeypatch, tmp_path: Path, transport: str) -> None: + payload = _status_event(state="TASK_STATE_WORKING") + projected = [] + args = _chat_args(transport=transport, session_id=None) + if transport == "code": + + class Response: + headers = {"Content-Type": "text/event-stream"} + + def __iter__(self): + return iter([("data: " + json.dumps(payload) + "\n").encode(), b"\n"]) + + def close(self): + return None + + monkeypatch.setattr(bridge, "_open_code_request", lambda *_args, **_kwargs: Response()) + else: + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr( + bridge.subprocess, + "Popen", + lambda _command, **_kwargs: _FakeCLIProcess(json.dumps({"data": payload})), + ) + + with pytest.raises(bridge.BridgeError) as error: + bridge._consume_start_chat( + args, + tmp_path, + "hello", + None, + [], + on_payload=lambda *_args: projected.append(True), + ) + + assert error.value.code == "missing_stream_event_id" + assert projected == [] + + +def test_stream_identity_change_fails_without_reconnect(monkeypatch, tmp_path: Path) -> None: + payload = _status_event(state="TASK_STATE_WORKING") + output = json.dumps( + [ + {"id": "v1.first.1", "data": payload}, + {"id": "v1.second.2", "data": payload}, + ] + ) + calls = [] + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr( + bridge.subprocess, + "Popen", + lambda command, **_kwargs: (calls.append(command) or _FakeCLIProcess(output)), + ) + + with pytest.raises(bridge.BridgeError) as error: + bridge._consume_start_chat(_chat_args(transport="aliyun_cli"), tmp_path, "hello", None, []) + + assert error.value.code == "stream_event_identity_mismatch" + assert len(calls) == 1 + + +def test_primary_permission_ack_reconnects_but_sideband_ack_is_an_immediate_boundary( + monkeypatch, tmp_path: Path +) -> None: + response = {"inputId": "input-1", "toolUseId": "tool-1", "decision": "allow_once"} + acknowledgement = { + "result": { + "message": { + "contextId": "session-1", + "parts": [ + { + "mediaType": "application/json", + "data": { + "schemaVersion": 1, + "kind": "permission_ack", + "inputId": "input-1", + "toolUseId": "tool-1", + "decision": "allow_once", + "accepted": True, + }, + } + ], + } + } + } + completed = _status_event(state="TASK_STATE_COMPLETED") + + def run(role): + attempts = [ + json.dumps({"id": "v1.permission.1", "data": acknowledgement}), + json.dumps({"id": "v1.permission.2", "data": completed}), + ] + commands = [] + + def popen(command, **_kwargs): + commands.append(command) + return _FakeCLIProcess(attempts.pop(0)) + + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + result = bridge._consume_start_chat( + _chat_args(transport="aliyun_cli", session_id="session-1"), + tmp_path, + "permission", + None, + [], + worker_role=role, + permission_response=response, + ) + return result, commands + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + primary_result, primary_commands = run("primary") + sideband_result, sideband_commands = run("sideband") + + assert primary_result["state"] == "turn-completed" + assert len(primary_commands) == 2 + assert "--stream-options" not in primary_commands[1] + assert _command_body(primary_commands[1]) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.permission.1", + } + assert sideband_result["state"] == "permission-responded" + assert len(sideband_commands) == 1 + + +def test_retryable_initial_failure_does_not_resend_the_business_query(monkeypatch, tmp_path: Path) -> None: + calls = [] + + def open_request(_operation, parameters, *_args, **_kwargs): + calls.append(parameters) + raise bridge.BridgeError("start_chat_failed", "throttled", True) + + monkeypatch.setattr(bridge, "_open_code_request", open_request) + + with pytest.raises(bridge.BridgeError) as error: + bridge._consume_start_chat(_chat_args(transport="code"), tmp_path, "hello", None, []) + + assert error.value.code == "missing_recovery_anchor" + assert len(calls) == 1 + assert calls[0]["Query"] == "hello" + + +def test_unknown_cli_nonzero_exit_is_not_retried(monkeypatch, tmp_path: Path) -> None: + calls = [] + + def popen(command, **kwargs): + calls.append(command) + kwargs["stderr"].write(b"unknown local failure") + return _FakeCLIProcess("", 2) + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + + result = bridge._consume_start_chat(_chat_args(transport="aliyun_cli"), tmp_path, "hello", None, []) + + assert result["state"] == "failed" + assert result["error"]["code"] == "aliyun_cli_failed" + assert len(calls) == 1 + + +def test_structured_retryable_cli_error_on_stdout_reconnects_from_anchor(monkeypatch, tmp_path: Path) -> None: + working = _status_event(state="TASK_STATE_WORKING", text="working") + completed = _status_event( + state="TASK_STATE_COMPLETED", text="done", metadata={"assistantFinal": {"complete": True}} + ) + calls = [] + + def popen(command, **_kwargs): + calls.append(command) + if len(calls) == 1: + output = "\n".join( + [ + json.dumps({"id": "v1.stdout-error.1", "data": working}), + json.dumps({"statusCode": 503, "code": "ServiceUnavailable"}), + ] + ) + return _FakeCLIProcess(output, 1) + return _FakeCLIProcess(json.dumps({"id": "v1.stdout-error.2", "data": completed})) + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr(bridge.subprocess, "Popen", popen) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + + result = bridge._consume_start_chat(_chat_args(transport="aliyun_cli"), tmp_path, "hello", None, []) + + assert result["state"] == "turn-completed" + assert len(calls) == 2 + assert _command_body(calls[1]) == { + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.stdout-error.1", + } + + +def test_same_worker_apply_checkpoint_gap_does_not_duplicate_summary(monkeypatch, tmp_path: Path) -> None: + first = _status_event(state="TASK_STATE_WORKING", text="a") + second = _status_event(state="TASK_STATE_WORKING", text="b") + completed = _status_event( + state="TASK_STATE_COMPLETED", text="done", metadata={"assistantFinal": {"complete": True}} + ) + attempts = [ + json.dumps( + [ + {"id": "v1.gap.1", "data": first}, + {"id": "v1.gap.2", "data": second}, + ] + ), + json.dumps( + [ + {"id": "v1.gap.2", "data": second}, + {"id": "v1.gap.3", "data": completed}, + ] + ), + ] + callback_calls = [] + + def callback(event_id, _payload, _summary, already_applied): + callback_calls.append((event_id, already_applied)) + if event_id == "v1.gap.2" and not already_applied: + raise bridge.BridgeError("stream_failed", "checkpoint write interrupted", True) + return True + + monkeypatch.setattr(bridge, "resolve_aliyun", lambda _path: "/usr/local/bin/aliyun") + monkeypatch.setattr( + bridge.subprocess, + "Popen", + lambda _command, **_kwargs: _FakeCLIProcess(attempts.pop(0)), + ) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + + result = bridge._consume_start_chat( + _chat_args(transport="aliyun_cli"), tmp_path, "hello", None, [], on_payload=callback + ) + + assert result["eventCount"] == 3 + assert callback_calls == [ + ("v1.gap.1", False), + ("v1.gap.2", False), + ("v1.gap.2", True), + ("v1.gap.3", False), + ] diff --git a/tests/skill_bridge/test_start_chat_relay.py b/tests/skill_bridge/test_start_chat_relay.py index e2029d2a1..a3fec5086 100644 --- a/tests/skill_bridge/test_start_chat_relay.py +++ b/tests/skill_bridge/test_start_chat_relay.py @@ -44,6 +44,72 @@ def _load_module(name: str, path: Path): bridge = _load_module("start_chat_test_bridge", BRIDGE_PATH) +def _write_relay_cli(tmp_path: Path) -> Path: + source = r'''import json +import ssl +import sys +import urllib.parse +import urllib.request + +arguments = sys.argv[1:] + +def option(name): + index = arguments.index(name) + return arguments[index + 1] + +parameters = {"AgentVersion": "V2"} +mapping = { + "--query": "Query", + "--session-id": "SessionId", + "--enable-partial-message": "EnablePartialMessage", + "--enable-thinking": "EnableThinking", + "--biz-mode": "Mode", + "--biz-region-id": "RegionId", +} +for flag, parameter in mapping.items(): + if flag in arguments: + parameters[parameter] = option(flag) +endpoint = option("--endpoint") +url = "https://{}/?{}".format(endpoint, urllib.parse.urlencode(parameters)) +body = option("--body").encode("utf-8") if "--body" in arguments else b"" +headers = {"x-acs-action": "StartChat"} +if body: + headers["content-type"] = "application/json" +request = urllib.request.Request(url, data=body, headers=headers, method="POST") +context = ssl.create_default_context() +context.check_hostname = False +context.verify_mode = ssl.CERT_NONE +with urllib.request.urlopen(request, context=context, timeout=15) as response: + event_id = None + data = [] + for raw_line in response: + line = raw_line.decode("utf-8").rstrip("\r\n") + if not line: + if data: + payload = json.loads("\n".join(data)) + print(json.dumps({"id": event_id, "data": payload}, separators=(",", ":")), flush=True) + event_id = None + data = [] + elif line.startswith("id:"): + event_id = line[3:].lstrip() + elif line.startswith("data:"): + data.append(line[5:].lstrip()) +''' + script = tmp_path / "relay_cli.py" + script.write_text(source, encoding="utf-8") + if sys.platform == "win32": + launcher = tmp_path / "aliyun.cmd" + launcher.write_text( + "@echo off\r\n{} %*\r\n".format(subprocess.list2cmdline([sys.executable, str(script)])), + encoding="utf-8", + ) + return launcher + launcher = tmp_path / "aliyun" + launcher.write_text("#!{}\n{}".format(sys.executable, source), encoding="utf-8") + launcher.chmod(0o755) + return launcher + + def _clear_code_credential_env(monkeypatch: pytest.MonkeyPatch) -> None: for name in ( "ALIBABA_CLOUD_ACCESS_KEY_ID", @@ -96,6 +162,24 @@ def test_relay_accepts_only_published_start_chat_parameters() -> None: ) +def test_relay_accepts_reconnect_stream_options_in_json_body() -> None: + parameters = relay.parse_start_chat_request( + "/?Action=StartChat&SessionId=session-1&AgentVersion=V2", + json.dumps( + {"StreamOptions.Action": "Reconnect", "StreamOptions.Cursor": "v1.stream.3"}, + separators=(",", ":"), + ).encode("utf-8"), + {"x-acs-action": "StartChat", "content-type": "application/json"}, + ) + + assert parameters == { + "AgentVersion": "V2", + "SessionId": "session-1", + "StreamOptions.Action": "Reconnect", + "StreamOptions.Cursor": "v1.stream.3", + } + + def test_relay_accepts_only_published_stop_chat_parameters() -> None: parameters = relay.parse_stop_chat_request( "/?Action=StopChat&Version=2019-09-10&SessionId=session-1&AgentVersion=V2", @@ -463,10 +547,11 @@ def _summarize_sse( ) -> dict: summary = bridge.StreamSummary(session_id, mode=mode) diagnostics = [] - for payload, raw in bridge.iter_cli_plugin_payloads(stdout.splitlines(keepends=True)): + for event in bridge.iter_cli_plugin_payloads(stdout.splitlines(keepends=True)): + payload = event["payload"] if payload is None: summary.malformed_event_count += 1 - diagnostics.append(raw) + diagnostics.append(event["raw"]) else: summary.apply(payload) return summary.to_result(0, stderr or "\n".join(diagnostics)) @@ -502,6 +587,8 @@ def test_code_transport_streams_through_sdk_to_endpoint_hook( a2a_url="http://127.0.0.1:1/", workspace=str(tmp_path), ssl_context=_tls_context(tmp_path), + disconnect_after_events=1, + replay_from_head_on_reconnects=1, ) captured = {} working_event = { @@ -536,7 +623,11 @@ def test_code_transport_streams_through_sdk_to_endpoint_hook( def start_a2a_call(session, parameters): captured["parameters"] = parameters + for event in (working_event, completed_event): + event["result"]["statusUpdate"]["contextId"] = session.session_id call = relay._UpstreamCall() + with session.state_lock: + session.streams[call.stream_id] = call captured["call"] = call call.events.put(working_event) return call @@ -594,7 +685,7 @@ def consume() -> None: "create a VPC", None, [], - on_payload=lambda _payload, _summary: first_payload.set(), + on_payload=lambda _event_id, _payload, _summary, _already_applied: first_payload.set(), ) except BaseException as exc: # pragma: no cover - asserted in the main test thread outcome["error"] = exc @@ -617,9 +708,16 @@ def consume() -> None: assert result["state"] == "turn-completed" assert result["finalText"] == "code transport done" + assert result["eventCount"] == 2 assert captured["profile"] == "sdk-profile" assert captured["parameters"]["Query"] == "create a VPC" assert captured["parameters"]["Mode"] == "IaCCodeNormal" + assert len(relay_server.request_metrics) == 2 + reconnect_metric = relay_server.request_metrics[1] + assert reconnect_metric["queryKind"] == "reconnect" + assert reconnect_metric["queryBytes"] == 0 + assert reconnect_metric["streamCursor"].endswith(".1") + assert reconnect_metric["returnedEventCount"] == 2 finally: if consumer_thread.is_alive() and "call" in captured: captured["call"].events.put(completed_event) @@ -630,6 +728,91 @@ def consume() -> None: relay_thread.join(timeout=5) +def test_cli_local_reconnects_to_relay_without_resending_query( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + working_event = { + "result": { + "statusUpdate": { + "taskId": "task-cli-1", + "contextId": "placeholder", + "status": {"state": "TASK_STATE_WORKING"}, + "metadata": {"iac_code": {}, "iacCodeSessionId": "iac-cli-1"}, + } + } + } + completed_event = { + "result": { + "statusUpdate": { + "taskId": "task-cli-1", + "contextId": "placeholder", + "status": { + "state": "TASK_STATE_COMPLETED", + "message": {"role": "ROLE_AGENT", "parts": [{"text": "cli reconnect done"}]}, + }, + "metadata": { + "iac_code": {"assistantFinal": {"complete": True}}, + "iacCodeSessionId": "iac-cli-1", + }, + } + } + } + relay_server = relay.StartChatRelay( + ("127.0.0.1", 0), + a2a_url="http://127.0.0.1:1/", + workspace=str(tmp_path), + ssl_context=_tls_context(tmp_path), + disconnect_after_events=1, + ) + + def start_a2a_call(session, _parameters): + for event in (working_event, completed_event): + event["result"]["statusUpdate"]["contextId"] = session.session_id + call = relay._UpstreamCall() + with session.state_lock: + session.streams[call.stream_id] = call + call.events.put(working_event) + call.events.put(completed_event) + call.events.put(relay._END) + return call + + relay_server.start_a2a_call = start_a2a_call + relay_thread = threading.Thread(target=relay_server.serve_forever, name="test-cli-reconnect-relay", daemon=True) + relay_thread.start() + endpoint = "127.0.0.1:{}".format(relay_server.server_address[1]) + fake_cli = _write_relay_cli(tmp_path) + args = SimpleNamespace( + aliyun_path=str(fake_cli), + transport="aliyun_cli", + aliyun_cli_execution_mode="local", + endpoint=endpoint, + connect_timeout=3, + read_timeout=15, + profile=None, + region_id="cn-hangzhou", + no_thinking=True, + mode="normal", + session_id=None, + ) + monkeypatch.setattr(bridge.time, "sleep", lambda _seconds: None) + + try: + result = bridge._consume_start_chat(args, tmp_path, "create a VPC", None, []) + + assert result["state"] == "turn-completed" + assert result["finalText"] == "cli reconnect done" + assert result["eventCount"] == 2 + assert [metric["queryKind"] for metric in relay_server.request_metrics] == ["conversation", "reconnect"] + assert relay_server.request_metrics[0]["queryBytes"] > 0 + assert relay_server.request_metrics[1]["queryBytes"] == 0 + assert relay_server.request_metrics[1]["streamCursor"].endswith(".1") + finally: + relay_server.shutdown() + relay_server.server_close() + relay_thread.join(timeout=5) + + def test_stop_chat_round_trip_through_real_aliyun_cli( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From ae3f0df1a99b3f0b0ec5d6d173c95b313ec6bd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A1=82=E9=A9=AC?= Date: Wed, 16 Sep 2026 16:46:49 +0800 Subject: [PATCH 2/4] test: specify UTF-8 in reconnect fixtures --- .../test_alicloud_ros_agent_bridge.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/skill_bridge/test_alicloud_ros_agent_bridge.py b/tests/skill_bridge/test_alicloud_ros_agent_bridge.py index 797cea342..be3773e40 100644 --- a/tests/skill_bridge/test_alicloud_ros_agent_bridge.py +++ b/tests/skill_bridge/test_alicloud_ros_agent_bridge.py @@ -2732,8 +2732,8 @@ def test_managed_remote_bootstrap_is_committed_before_ack_and_recovers_lost_resu + "counter_path = Path({!r})\n".format(str(counter_path)) + "calls_path = Path({!r})\n".format(str(calls_path)) + "observation_path = Path({!r})\n".format(str(observation_path)) - + "count = int(counter_path.read_text() or '0') + 1 if counter_path.exists() else 1\n" - + "counter_path.write_text(str(count))\n" + + "count = int(counter_path.read_text(encoding='utf-8') or '0') + 1 if counter_path.exists() else 1\n" + + "counter_path.write_text(str(count), encoding='utf-8')\n" + "with calls_path.open('a', encoding='utf-8') as handle:\n" + " handle.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "def status(state, text='', final=False):\n" @@ -2757,12 +2757,12 @@ def test_managed_remote_bootstrap_is_committed_before_ack_and_recovers_lost_resu + " state_root = Path(os.environ['ALICLOUD_ROS_AGENT_STATE_DIR'])\n" + " job_path = next((state_root / 'jobs').glob('*/job.json'))\n" + " spool_path = job_path.with_name('events.jsonl')\n" - + " job = json.loads(job_path.read_text())\n" - + " records = [json.loads(line) for line in spool_path.read_text().splitlines()]\n" - + " ack = json.loads(ack_path.read_text()) if ack_path.exists() else {}\n" + + " job = json.loads(job_path.read_text(encoding='utf-8'))\n" + + " records = [json.loads(line) for line in spool_path.read_text(encoding='utf-8').splitlines()]\n" + + " ack = json.loads(ack_path.read_text(encoding='utf-8')) if ack_path.exists() else {}\n" + " observation_path.write_text(json.dumps({'ack': ack, 'sessionId': job.get('sessionId'), " + "'streamCursor': job.get('streamCursor'), 'spoolIds': " - + "[item.get('streamEventId') for item in records]}))\n" + + "[item.get('streamEventId') for item in records]}), encoding='utf-8')\n" + " print(json.dumps({'code': 'ExecutorTimeout'}), file=sys.stderr)\n" + " raise SystemExit(1)\n" + "completed = status('TASK_STATE_COMPLETED', 'done', True)\n" @@ -2786,7 +2786,7 @@ def test_managed_remote_bootstrap_is_committed_before_ack_and_recovers_lost_resu assert result["state"] == "turn-completed" assert result["finalText"] == "done" - calls = [json.loads(line) for line in calls_path.read_text().splitlines()] + calls = [json.loads(line) for line in calls_path.read_text(encoding="utf-8").splitlines()] assert len(calls) == 2 assert "--query" in calls[0] assert "--query" not in calls[1] @@ -2795,7 +2795,7 @@ def test_managed_remote_bootstrap_is_committed_before_ack_and_recovers_lost_resu "StreamOptions.Action": "Reconnect", "StreamOptions.Cursor": "v1.remote-managed.1", } - observation = json.loads(observation_path.read_text()) + observation = json.loads(observation_path.read_text(encoding="utf-8")) assert observation["ack"] == { "committed": True, "eventId": "v1.remote-managed.1", From 9bed115363b351e859b7296681efa9455e44109a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A1=82=E9=A9=AC?= Date: Wed, 16 Sep 2026 17:10:03 +0800 Subject: [PATCH 3/4] test: support reconnect harness on Windows --- scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py | 5 ++++- tests/a2a_e2e/test_qoder_mcp_reconnect.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py b/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py index ed66fd655..84eee28e7 100755 --- a/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py +++ b/scripts/a2a/e2e/reconnect/aliyun_cli_mcp_server.py @@ -276,7 +276,10 @@ async def _call_real_cli(argv: list[str], ctx: Context) -> str: "the initial or reconnect StartChat command shape violated the E2E contract", ) - command = [real_aliyun, *argv, "--profile", profile] + executable = [real_aliyun] + if Path(real_aliyun).suffix.lower() == ".py": + executable = [os.environ.get("IAC_CODE_E2E_PYTHON") or sys.executable, real_aliyun] + command = [*executable, *argv, "--profile", profile] started = time.monotonic() try: process = await asyncio.create_subprocess_exec( diff --git a/tests/a2a_e2e/test_qoder_mcp_reconnect.py b/tests/a2a_e2e/test_qoder_mcp_reconnect.py index 12de6fdd2..1993dd539 100644 --- a/tests/a2a_e2e/test_qoder_mcp_reconnect.py +++ b/tests/a2a_e2e/test_qoder_mcp_reconnect.py @@ -90,7 +90,7 @@ def test_installed_skill_is_temporarily_patched_to_remote_fake_cli(tmp_path) -> assert config["endpoint"] == "ros-pre.aliyuncs.com" assert config["allowedAgentModes"] == ["normal"] assert config["aliyunCLIForwardEnv"] == list(runner.FORWARDED_ENV) - assert runtime.count(str(fake_cli.resolve())) == 2 + assert runtime.count(json.dumps(str(fake_cli.resolve()))) == 2 runner._restore_skills(backups) From 4a11a834271b01e0f49efc4dd87c491b4560eaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A1=82=E9=A9=AC?= Date: Wed, 16 Sep 2026 17:29:41 +0800 Subject: [PATCH 4/4] test: normalize reconnect harness output --- scripts/a2a/e2e/reconnect/fake_aliyun_cli.py | 8 +++++++- tests/a2a_e2e/test_qoder_mcp_reconnect.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py b/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py index 686526c0b..aae4d947e 100755 --- a/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py +++ b/scripts/a2a/e2e/reconnect/fake_aliyun_cli.py @@ -54,6 +54,12 @@ def _result_text(result: Any) -> str: return "".join(pieces) +def _normalize_newlines(value: str) -> str: + """Avoid translating captured Windows CRLF a second time on output.""" + + return value.replace("\r\n", "\n").replace("\r", "\n") + + async def _call_mcp(argv: list[str]) -> dict[str, Any]: server_path = Path(os.environ["IAC_CODE_E2E_MCP_SERVER"]).expanduser().resolve() python = os.environ.get("IAC_CODE_E2E_PYTHON") or sys.executable @@ -136,7 +142,7 @@ async def _main(argv: list[str]) -> int: stdout = result.get("stdout") if not isinstance(stdout, str): return _write_error("MCPCallFailed", "the E2E MCP result did not contain CLI stdout") - sys.stdout.write(stdout) + sys.stdout.write(_normalize_newlines(stdout)) sys.stdout.flush() return int(result.get("returnCode") or 0) diff --git a/tests/a2a_e2e/test_qoder_mcp_reconnect.py b/tests/a2a_e2e/test_qoder_mcp_reconnect.py index 1993dd539..87acc87c2 100644 --- a/tests/a2a_e2e/test_qoder_mcp_reconnect.py +++ b/tests/a2a_e2e/test_qoder_mcp_reconnect.py @@ -35,6 +35,10 @@ def _mcp_server(): return _load("qoder_mcp_reconnect_server", MCP_SERVER_PATH) +def _fake_cli(): + return _load("qoder_mcp_reconnect_fake_cli", FAKE_CLI_PATH) + + def test_runner_requires_explicit_real_cloud_opt_in(tmp_path) -> None: runner = _runner() @@ -98,6 +102,10 @@ def test_installed_skill_is_temporarily_patched_to_remote_fake_cli(tmp_path) -> assert not (destination / "config.json").exists() +def test_fake_cli_normalizes_captured_windows_newlines_before_writing() -> None: + assert _fake_cli()._normalize_newlines("first\r\nsecond\r\n") == "first\nsecond\n" + + @pytest.mark.skipif( not hasattr(os, "chflags") or not hasattr(stat, "UF_IMMUTABLE"), reason="user immutable flags are not supported",