Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/zcode-acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ jobs:
- name: Set up Node
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: '24.15.0'
# Includes Windows shutdown and file-watcher fixes (nodejs/node#61999, #65118).
node-version: '24.21.0'

- name: Install the host package manager
run: npm install --global pnpm@10.33.2
Expand Down
14 changes: 12 additions & 2 deletions integrations/zcode/acceptance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@ It does not download a host, deploy a remote service, or operate the desktop UI.

[ZCode acceptance](../../../.github/workflows/zcode-acceptance.yml) runs on every pull request, every push to master,
weekly on Sunday at 03:23 UTC, and by manual dispatch. It builds the actual open-source CLI on Windows from ZCode commit
`29628c9acdb81b703bbd4080c207a0e7ce5e276e` (ZCode 3.14.3 / CLI 0.16.9), using Node 24.15.0, pnpm 10.33.2 and the
`29628c9acdb81b703bbd4080c207a0e7ce5e276e` (ZCode 3.14.3 / CLI 0.16.9), using Node 24.21.0, pnpm 10.33.2 and the
host's frozen lockfile. Dependency install scripts are disabled; only the CLI and its required workspace builds run.

Node 24.21.0 includes the [Windows shutdown fix](https://github.com/nodejs/node/pull/61999) for a delayed worker task
posting to a closing libuv async handle, and the [file-watcher fix](https://github.com/nodejs/node/pull/65118) for a
separate path assertion in Node 24.16.0–24.20.0. Both faults can abort with `0xC0000409`; the async-handle fault can occur
after emitting a CLI result. The acceptance requires a zero process exit code for every invocation.

The job runs all plugin Node tests, including the actual host test, then explicitly selects two independent
controlled-model acceptances covering P3-01 through P3-09. Host build errors, missing prerequisites and failed scenarios
fail the job. The controlled inference fixtures need no model keys or repository secrets. This is actual CLI + real
Expand All @@ -27,7 +32,8 @@ fixed host against current master and dependency availability; they do not silen
## Prerequisites

- Install this checkout with its development dependencies and Server extras.
- Build the open-source ZCode CLI separately; Node 24 or later must be on PATH.
- Build the open-source ZCode CLI separately; Node 24 or later must be on PATH. On Windows, use the patched Node
24.21.0 runtime pinned by CI.
- Use a directory writable by the same Windows user that runs Node and Python. An inaccessible plugin directory
is an installation failure, even when its path appears in `plugins.dirs`.
- Keep `.powercontext/` ignored by Git. A run's profile contains private model configuration and may contain
Expand Down Expand Up @@ -84,6 +90,10 @@ Each run writes `.powercontext/zcode-acceptance/<run-id>/summary.json` with sche
Missing prerequisites, rejected model requests, timeouts and failed assertions exit nonzero. Daily pytest runs deselect
these tests. Deselection and unsupported features are not acceptance success.

For a nonzero prompt-process exit, `host_process_exit` in the summary retains the return code, the number of result
complete result events already emitted, and the fixed `libuv_async_handle_closing` or `libuv_fs_event_path_mismatch`
signature when present. Raw stderr is excluded. A truncated frame from a crashed process does not hide its exit code.

| Scenario | Evidence |
| --- | --- |
| P3-01 | Managed installation, upgrade preservation and actual CLI discovery |
Expand Down
35 changes: 34 additions & 1 deletion tests/e2e/zcode_acceptance/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@

REPOSITORY = Path(__file__).resolve().parents[3]
PLUGIN = REPOSITORY / "integrations/zcode/plugins/powercontext"
_NATIVE_ASSERTIONS = {
"libuv_async_handle_closing": r"Assertion failed: !\(handle->flags & UV_HANDLE_CLOSING\), "
r"file (?:deps[/\\]uv[/\\])?src[/\\]win[/\\]async\.c, line [0-9]+",
"libuv_fs_event_path_mismatch": r"Assertion failed: !_wcsnicmp\(filename, dir, dirlen\), "
r"file (?:deps[/\\]uv[/\\])?src[/\\]win[/\\]fs-event\.c, line [0-9]+",
}


def write_json(path: Path, value: Any) -> None:
Expand All @@ -66,6 +72,20 @@ def error_code(error: Exception) -> str:
return code if re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_/]{0,95}", code) else type(error).__name__


def _cli_events(output: str, *, failed: bool) -> list[dict[str, Any]]:
"""Decode CLI frames without hiding a process failure behind truncated output."""
events = []
for line in output.splitlines():
if not line.strip().startswith("{"):
continue
try:
events.append(json.loads(line))
except json.JSONDecodeError:
if not failed:
raise
return events


@contextmanager
def serve(app: Any) -> Iterator[str]:
"""Own a pre-bound loopback socket and only stop this run's Server."""
Expand Down Expand Up @@ -264,7 +284,7 @@ def invoke(
if resume:
args.extend(["--resume", resume])
completed = self.process(args, environment=environment)
events = [json.loads(line) for line in completed.stdout.splitlines() if line.strip().startswith("{")]
events = _cli_events(completed.stdout, failed=completed.returncode != 0)
self.last_events = events
if self.live:
started = [
Expand All @@ -280,6 +300,19 @@ def invoke(
"host_selected_different_model_from_requested"
)
if completed.returncode:
self.summary["host_process_exit"] = {
"returncode": completed.returncode,
"result_event_count": sum(event.get("type") == "result" for event in events),
# Retain a fixed native signature without exposing stderr's paths, prompts or credentials.
"native_assertion": next(
(
code
for code, pattern in _NATIVE_ASSERTIONS.items()
if any(re.fullmatch(pattern, line) for line in completed.stderr.splitlines())
),
None,
),
}
failures = [payload for event in events if (payload := event.get("payload", {})).get("errorCode")]
codes = [
payload["errorCode"]
Expand Down
48 changes: 48 additions & 0 deletions tests/test_zcode_acceptance_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import pytest

from tests.e2e.zcode_acceptance.host import NativeHost
from tests.e2e.zcode_acceptance.runner import AcceptanceRun


def make_host(tmp_path, source: str) -> NativeHost:
Expand Down Expand Up @@ -121,3 +122,50 @@ def test_normal_shutdown_does_not_emit_host_failure_evidence(tmp_path):
host.close()
assert host.process.returncode == 0
assert not (tmp_path / "evidence").exists()


@pytest.mark.parametrize(
("native_assertion", "result_emitted", "truncated"),
[
("libuv_async_handle_closing", False, True),
("libuv_async_handle_closing", True, False),
("libuv_fs_event_path_mismatch", False, False),
(None, True, False),
],
)
def test_nonzero_cli_exit_stays_failed_and_retains_only_safe_diagnostics(
tmp_path, native_assertion, result_emitted, truncated
):
cli = tmp_path / "host.py"
diagnostic = "private-provider-token private-profile-path\n"
if native_assertion == "libuv_async_handle_closing":
diagnostic += "Assertion failed: !(handle->flags & UV_HANDLE_CLOSING), file src\\win\\async.c, line 76\n"
elif native_assertion == "libuv_fs_event_path_mismatch":
diagnostic += "Assertion failed: !_wcsnicmp(filename, dir, dirlen), file src\\win\\fs-event.c, line 72\n"
cli.write_text(
"import json, sys\n"
f"sys.stderr.write({diagnostic!r})\n"
+ (
"print(json.dumps({'type': 'result', 'sessionId': 'private-session', 'response': 'private-response'}))\n"
if result_emitted
else ""
)
+ ("print('{\"type\":')\n" if truncated else "")
+ "sys.exit(7)\n",
encoding="utf-8",
)
run = AcceptanceRun(tmp_path / "runs", live=False, model_config=None, generation_env=None)
run.workspace.mkdir(parents=True)
run.node = getattr(sys, "_base_executable", sys.executable)
run.cli = cli
run.model = None

with pytest.raises(AssertionError, match=r"^host_process_exit_7$"):
run.invoke("private-prompt")

assert run.summary["host_process_exit"] == {
"returncode": 7,
"result_event_count": int(result_emitted),
"native_assertion": native_assertion,
}
assert "private" not in json.dumps(run.summary)
Loading