diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..a5848ac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug Report +description: Report a reproducible problem or unexpected error in OpenOrbit +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report an issue! Please provide as much detail as possible to help us investigate and resolve it quickly. + - type: textarea + id: description + attributes: + label: Problem Description + description: A clear and concise description of the bug. + placeholder: Describe what happened... + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to Reproduce + description: Exact steps to reproduce the behavior. + placeholder: | + 1. Start the application with '...' + 2. Navigate to '...' + 3. Click on '...' + 4. Observe error + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + placeholder: What did you expect to happen? + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual Behavior + description: What actually occurred, including any error messages or unexpected states. + placeholder: What happened instead? + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment & Versions + description: Relevant environment information (OS, Node version, Python version, browser, OpenOrbit version). + placeholder: | + - OS: [e.g. Linux Ubuntu 22.04 / macOS 14 / Windows 11] + - Python version: [e.g. 3.13.0] + - Node.js version: [e.g. 24.0.0] + - OpenOrbit version: [e.g. v0.8.0 or commit hash] + validations: + required: true + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context, logs, terminal outputs, or screenshots about the problem here. + placeholder: Optional additional logs or screenshots... + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..fde139f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,41 @@ +name: Feature Request +description: Propose a new idea, enhancement, or capability for OpenOrbit +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thank you for suggesting a feature! Please provide details about what problem this feature solves and how it should work. + - type: textarea + id: problem + attributes: + label: Problem Statement / Motivation + description: Is your feature request related to a problem or limitation? Please describe. + placeholder: A clear and concise description of what the problem or use case is... + validations: + required: true + - type: textarea + id: proposed-solution + attributes: + label: Proposed Solution & Expected Behavior + description: A clear and concise description of what you want to happen and how it should look or behave. + placeholder: Describe your proposed design, UI behavior, or API capability... + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: Any alternative solutions or workarounds you have considered. + placeholder: Optional description of other approaches evaluated... + validations: + required: false + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context, mockup screenshots, or implementation suggestions here. + placeholder: Optional mockups, architectural notes, or links... + validations: + required: false diff --git a/.gitignore b/.gitignore index f6cc555..a19a93e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ test-results/ openorbit-state.zip conversations/ .orbit/ +AGENTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ac9ae53..931313d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ All notable changes to OpenOrbit are documented in this file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project uses [Semantic Versioning](https://semver.org/). +## [0.12.0] - 2026-09-23 + +### Added + +- Added a Visual Runner editor that lets authors compose Runner workflows visually from the Runner Code screen. +- Added localized palette tooltips that explain each Visual Runner node's purpose, type, inputs, and outputs. +- Added reusable visual workflow definitions for bundled Quick Starts and Runner templates. + +### Changed + +- Unified Visual Runner and Workflow Graph node presentation around the same component and layout behavior. +- Aligned initial Visual Runner node placement with Workflow Graph layout, and improved loop, reverse-link, and edge-label rendering. +- Expanded English, Korean, and Japanese Visual Runner node guidance with practical template-oriented descriptions. + +### Fixed + +- Fixed Visual Runner Python generation for boolean and null configuration values, browser runtime Playwright resolution, and duplicate site-exploration visits. +- Fixed runner-template execution environment support for source-contract and Selenium command settings. +- Improved SupportOps example adapter compatibility with external AI-experience Runner actions. + ## [0.11.0] - 2026-09-15 ### Added @@ -263,3 +283,4 @@ and this project uses [Semantic Versioning](https://semver.org/). [0.9.0]: https://github.com/forthfate/openorbit/releases/tag/v0.9.0 [0.10.0]: https://github.com/forthfate/openorbit/releases/tag/v0.10.0 [0.11.0]: https://github.com/forthfate/openorbit/releases/tag/v0.11.0 +[0.12.0]: https://github.com/forthfate/openorbit/releases/tag/v0.12.0 diff --git a/README.md b/README.md index bd1665d..eaeea5e 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,7 @@ OpenOrbit exposes a local, versioned API: - Swagger UI: `http://localhost:3000/api/docs` - OpenAPI document: `http://localhost:3000/api/openapi.json` - API base: `http://localhost:3000/api/v1` +- MCP (Streamable HTTP): `http://localhost:3000/mcp/` Read the [API reference](docs/API.md) for endpoint details. To add reusable automation, create a Python runner with explicit lifecycle phases: diff --git a/backend/app/assistant_graph.py b/backend/app/assistant_graph.py new file mode 100644 index 0000000..4959182 --- /dev/null +++ b/backend/app/assistant_graph.py @@ -0,0 +1,159 @@ +"""LangGraph orchestration for the local Orbit assistant.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, Callable, TypedDict + +from coding_agents import execution_environment_context +from langgraph.graph import END, START, StateGraph +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +from .assistant_tools import DEFAULT_MCP_URL, AssistantToolExecutor +from .assistant_ui import AssistantUiToolExecutor +from .providers import AzureOpenAIProvider, BedrockProvider, ModelSettings + + +class AssistantState(TypedDict, total=False): + prompt: str + response: str + + +class LocalMcpTools: + """Expose the mounted OpenOrbit MCP server as native model function tools.""" + + def __init__(self, url: str = DEFAULT_MCP_URL): + self.url = url + + async def _list_tools(self) -> list[dict[str, Any]]: + async with streamablehttp_client(self.url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + return [ + { + "name": tool.name, + "description": tool.description or "OpenOrbit MCP tool.", + "parameters": tool.inputSchema, + } + for tool in result.tools + ] + + async def _call_tool(self, name: str, arguments: dict[str, Any]) -> str: + async with streamablehttp_client(self.url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool(name, arguments) + if result.isError: + return json.dumps({"error": "MCP tool failed", "content": result.content}, default=str) + return json.dumps({"content": result.content}, default=str, ensure_ascii=False) + + @staticmethod + def _run(coro: Any) -> Any: + return asyncio.run(coro) + + def definitions(self) -> list[dict[str, Any]]: + try: + return self._run(self._list_tools()) + except Exception: + # The assistant remains usable while the local server is starting. + return [] + + def execute(self, name: str, arguments: dict[str, Any]) -> str: + try: + return self._run(self._call_tool(name, arguments)) + except Exception as error: + return json.dumps({"error": f"MCP unavailable: {error}"}) + + +class OrbitAssistantGraph: + """A small, explicit LangGraph for one bounded assistant turn. + + The model node can use both the local bounded workspace tools and the + mounted MCP control-room tools. Provider adapters retain the native tool + call protocol, while LangGraph owns the turn state and final response. + """ + + def __init__( + self, + provider: AzureOpenAIProvider | BedrockProvider, + settings: ModelSettings, + local_tools: AssistantToolExecutor, + mcp_tools: LocalMcpTools | None = None, + ui_tools: AssistantUiToolExecutor | None = None, + on_activity: Callable[[str, str | None], None] | None = None, + ): + self.provider = provider + self.settings = settings + self.local_tools = local_tools + self.mcp_tools = mcp_tools or LocalMcpTools(local_tools.settings["mcp_server_url"]) + self.ui_tools = ui_tools + self.on_activity = on_activity + graph = StateGraph(AssistantState) + graph.add_node("model", self._model) + graph.add_edge(START, "model") + graph.add_edge("model", END) + self.graph = graph.compile() + + def _model(self, state: AssistantState) -> AssistantState: + self._activity("thinking") + local_definitions = self.local_tools.definitions() + mcp_definitions = self.mcp_tools.definitions() + ui_definitions = self.ui_tools.definitions() if self.ui_tools else [] + definitions = [*mcp_definitions, *local_definitions, *ui_definitions] + + def execute(name: str, arguments: dict[str, Any]) -> str: + self._activity("working", name) + if any(tool["name"] == name for tool in mcp_definitions): + result = self.mcp_tools.execute(name, arguments) + elif self.ui_tools and any(tool["name"] == name for tool in ui_definitions): + result = self.ui_tools.execute(name, arguments) + else: + result = self.local_tools.execute(name, arguments) + self._activity("thinking") + return result + + response = ( + self.provider.complete_with_tools(self.settings, state["prompt"], definitions, execute) + if definitions + else self.provider.complete(self.settings, state["prompt"]) + ) + return {"response": response} + + def _activity(self, phase: str, tool: str | None = None) -> None: + if self.on_activity: + self.on_activity(phase, tool) + + def invoke(self, prompt: str) -> str: + result = self.graph.invoke({"prompt": prompt}) + return str(result["response"]) + + +def build_assistant_prompt( + content: str, history: list[tuple[str, str]], locale: str | None, ui_enabled: bool = False +) -> str: + prompt = ( + "You are Orbit, a concise assistant for the local OpenOrbit control room.\n" + "Use the OpenOrbit MCP tools as the source of truth for local control-room state. " + "Read state before suggesting or taking any state-changing action.\n" + + execution_environment_context() + ) + if ui_enabled: + prompt += ( + "Browser UI tools are available for this turn. For any question about the current Orbit " + "screen, visible component, selected item, open panel, dialog, form value, or any request " + "to change the Orbit UI, first call ui_get_context. Use ui_interact only after reading the " + "current UI context. Do not use file tools to infer browser UI state. File tools remain for " + "workspace files and coding tasks only.\n" + ) + if locale: + prompt += f"Respond in BCP 47 locale '{locale}'.\n" + if history: + prompt += ( + "Conversation so far:\n" + + "\n".join(f"{role.title()}: {message}" for role, message in history) + + "\n\n" + ) + return prompt + f"User: {content}\nAssistant:" diff --git a/backend/app/assistant_tools.py b/backend/app/assistant_tools.py index e0b97a1..d12bdc4 100644 --- a/backend/app/assistant_tools.py +++ b/backend/app/assistant_tools.py @@ -5,14 +5,24 @@ import json import os import re +import shutil import subprocess from pathlib import Path from typing import Any +from coding_agents import CODING_AGENT_PROVIDERS, build_coding_agent_command, build_coding_agent_prompt + MAX_RESULT_BYTES = 16 * 1024 MAX_FILE_LINES = 120 MAX_SEARCH_RESULTS = 12 BLOCKED_NAMES = {".env", ".git", ".ssh", "id_rsa", "id_ed25519"} +DEFAULT_MCP_URL = "http://127.0.0.1:3000/mcp/" + + +def _terminal_visible() -> bool: + """Deployment flag; terminal visibility is not an operator UI setting.""" + value = os.environ.get("ORBIT_ASSISTANT_TERMINAL_VISIBLE", "true").strip().lower() + return value not in {"0", "false", "no", "off"} def defaults() -> dict[str, Any]: @@ -21,8 +31,12 @@ def defaults() -> dict[str, Any]: "file_read_enabled": True, "file_search_enabled": True, "run_process_enabled": True, + "coding_agent_enabled": True, + "ui_context_enabled": True, + "ui_interaction_enabled": True, "terminal_enabled": True, - "terminal_visible": True, + "terminal_visible": _terminal_visible(), + "mcp_server_url": DEFAULT_MCP_URL, } @@ -36,24 +50,32 @@ def normalize_settings(value: object) -> dict[str, Any]: "file_read_enabled", "file_search_enabled", "run_process_enabled", + "coding_agent_enabled", + "ui_context_enabled", + "ui_interaction_enabled", "terminal_enabled", - "terminal_visible", ): result[key] = bool(value.get(key, result[key])) + result["terminal_visible"] = _terminal_visible() + result["mcp_server_url"] = ( + str(value.get("mcp_server_url", result["mcp_server_url"])).strip() or result["mcp_server_url"] + ) return result class AssistantToolExecutor: - def __init__(self, settings: dict[str, Any]): + def __init__(self, settings: dict[str, Any], coding_agent_provider: str = "none"): self.settings = normalize_settings(settings) root = self.settings["workspace_root"] self.root = Path(root).expanduser().resolve() if root else None + self.coding_agent_provider = ( + coding_agent_provider if coding_agent_provider in CODING_AGENT_PROVIDERS else "none" + ) def definitions(self) -> list[dict[str, Any]]: - if not self.root or not self.root.is_dir(): - return [] definitions = [] - if self.settings["file_read_enabled"]: + workspace_ready = bool(self.root and self.root.is_dir()) + if workspace_ready and self.settings["file_read_enabled"]: definitions.append( { "name": "file_read", @@ -70,7 +92,7 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) - if self.settings["file_search_enabled"]: + if workspace_ready and self.settings["file_search_enabled"]: definitions.append( { "name": "file_search", @@ -89,7 +111,7 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) - if self.settings["run_process_enabled"]: + if workspace_ready and self.settings["run_process_enabled"]: definitions.append( { "name": "run_process", @@ -107,6 +129,22 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) + if self.settings["coding_agent_enabled"]: + definitions.append( + { + "name": "coding_agent", + "description": "Ask the configured local coding agent to inspect, modify, or validate the configured workspace. If no coding agent is selected, returns instructions for configuring one in Settings.", + "parameters": { + "type": "object", + "properties": { + "task": {"type": "string", "minLength": 1}, + "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 1800}, + }, + "required": ["task"], + "additionalProperties": False, + }, + } + ) return definitions def execute(self, name: str, arguments: dict[str, Any]) -> str: @@ -117,6 +155,8 @@ def execute(self, name: str, arguments: dict[str, Any]) -> str: return self._file_search(arguments) if name == "run_process" and self.settings["run_process_enabled"]: return self._run_process(arguments) + if name == "coding_agent" and self.settings["coding_agent_enabled"]: + return self._coding_agent(arguments) return self._result(error="Tool is disabled or unavailable.") except (OSError, UnicodeError, ValueError, re.error, subprocess.SubprocessError) as error: return self._result(error=str(error)) @@ -230,3 +270,43 @@ def _run_process(self, arguments: dict[str, Any]) -> str: output=output, truncated=len((result.stdout + result.stderr).encode("utf-8")) > MAX_RESULT_BYTES, ) + + def _coding_agent(self, arguments: dict[str, Any]) -> str: + if self.coding_agent_provider == "none": + return self._result( + error="No Coding Agent is selected. Select Kiro, Claude Code, or Codex in Settings.", + error_code="coding_agent_not_configured", + settings_page="settings", + ) + if not self.root or not self.root.is_dir(): + return self._result( + error="The Assistant workspace is missing. Configure a valid workspace in the Assistant tool settings.", + error_code="assistant_workspace_unavailable", + ) + prompt = build_coding_agent_prompt(str(arguments["task"]), self.root) + command = build_coding_agent_command(self.coding_agent_provider, prompt=prompt) + if not shutil.which(command[0]): + return self._result( + error=f"The selected Coding Agent ({self.coding_agent_provider}) is not installed or is unavailable on PATH.", + error_code="coding_agent_not_installed", + provider=self.coding_agent_provider, + ) + result = subprocess.run( + command, + cwd=self.root, + shell=False, + capture_output=True, + text=True, + timeout=min(max(int(arguments.get("timeout_seconds", 600)), 1), 1800), + env=os.environ.copy(), + ) + combined_output = result.stdout + result.stderr + output = combined_output.encode("utf-8")[:MAX_RESULT_BYTES].decode("utf-8", errors="ignore") + return self._result( + provider=self.coding_agent_provider, + command=command[:-1], + working_directory=str(self.root), + exit_code=result.returncode, + output=output, + truncated=len(combined_output.encode("utf-8")) > MAX_RESULT_BYTES, + ) diff --git a/backend/app/assistant_ui.py b/backend/app/assistant_ui.py new file mode 100644 index 0000000..35dd561 --- /dev/null +++ b/backend/app/assistant_ui.py @@ -0,0 +1,149 @@ +"""Request-scoped bridge between Assistant tools and one browser UI session.""" + +from __future__ import annotations + +import json +import logging +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +logger = logging.getLogger(__name__) + + +@dataclass +class PendingUiCommand: + event: threading.Event = field(default_factory=threading.Event) + result: dict[str, Any] | None = None + + +@dataclass +class UiSession: + send: Callable[[str, dict[str, Any]], None] + pending: dict[str, PendingUiCommand] = field(default_factory=dict) + + +class AssistantUiBroker: + """Route synchronous model tool calls through the active chat stream.""" + + def __init__(self) -> None: + self._sessions: dict[str, UiSession] = {} + self._lock = threading.Lock() + + def open(self, session_id: str, send: Callable[[str, dict[str, Any]], None]) -> None: + with self._lock: + self._sessions[session_id] = UiSession(send=send) + + def close(self, session_id: str) -> None: + with self._lock: + session = self._sessions.pop(session_id, None) + if session: + for pending in session.pending.values(): + pending.result = {"ok": False, "error": "UI session closed."} + pending.event.set() + + def request(self, session_id: str, command: dict[str, Any], timeout: float = 30) -> dict[str, Any]: + command_id = uuid.uuid4().hex + pending = PendingUiCommand() + with self._lock: + session = self._sessions.get(session_id) + if not session: + logger.info("assistant UI command rejected: unavailable session=%s", session_id) + return {"ok": False, "error": "The browser UI session is unavailable."} + session.pending[command_id] = pending + logger.info( + "assistant UI command sent: session=%s id=%s type=%s", + session_id, + command_id, + command.get("type"), + ) + session.send(command_id, command) + if not pending.event.wait(timeout): + with self._lock: + session = self._sessions.get(session_id) + if session: + session.pending.pop(command_id, None) + return {"ok": False, "error": "The browser UI did not respond in time."} + return pending.result or {"ok": False, "error": "The browser UI returned no result."} + + def resolve(self, session_id: str, command_id: str, result: dict[str, Any]) -> bool: + with self._lock: + session = self._sessions.get(session_id) + pending = session.pending.pop(command_id, None) if session else None + if not pending: + logger.info("assistant UI result ignored: session=%s id=%s", session_id, command_id) + return False + pending.result = result + logger.info( + "assistant UI result received: session=%s id=%s ok=%s components=%s diagnostics=%s", + session_id, + command_id, + result.get("ok"), + len(result.get("components", [])) if isinstance(result.get("components"), list) else "n/a", + result.get("diagnostics"), + ) + pending.event.set() + return True + + +class AssistantUiToolExecutor: + """Expose semantic, browser-owned UI capabilities to the Assistant.""" + + def __init__( + self, broker: AssistantUiBroker, session_id: str, *, context_enabled: bool, interaction_enabled: bool + ): + self.broker = broker + self.session_id = session_id + self.context_enabled = context_enabled + self.interaction_enabled = interaction_enabled and context_enabled + + def definitions(self) -> list[dict[str, Any]]: + definitions = [] + if self.context_enabled: + definitions.append( + { + "name": "ui_get_context", + "description": "Read the current browser page, visible Assistant-enabled components, controls, and actions. Read this before interacting with the UI.", + "parameters": { + "type": "object", + "properties": {"component_id": {"type": "string"}}, + "additionalProperties": False, + }, + } + ) + if self.interaction_enabled: + definitions.append( + { + "name": "ui_interact", + "description": "Set one visible Assistant-enabled UI control or invoke one listed UI action. Use the revision returned by ui_get_context. High-impact actions only open the normal user confirmation dialog.", + "parameters": { + "type": "object", + "properties": { + "component_id": {"type": "string", "minLength": 1}, + "revision": {"type": "integer", "minimum": 1}, + "operation": {"type": "string", "enum": ["set_control_value", "invoke_action"]}, + "target_id": {"type": "string", "minLength": 1}, + "value": {}, + }, + "required": ["component_id", "revision", "operation", "target_id"], + "additionalProperties": False, + }, + } + ) + return definitions + + def execute(self, name: str, arguments: dict[str, Any]) -> str: + if name == "ui_get_context" and self.context_enabled: + result = self.broker.request( + self.session_id, + {"type": "get_context", "component_id": arguments.get("component_id")}, + ) + elif name == "ui_interact" and self.interaction_enabled: + result = self.broker.request( + self.session_id, + {"type": "interact", **arguments}, + ) + else: + result = {"ok": False, "error": "Unknown browser UI tool."} + return json.dumps(result, ensure_ascii=False) diff --git a/backend/app/main.py b/backend/app/main.py index f70be62..c83c101 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,21 +1,41 @@ from __future__ import annotations +import asyncio import json +import os import re +import shutil +from contextlib import asynccontextmanager from pathlib import Path from typing import Literal from fastapi import FastAPI, File, HTTPException, Query, Request, Response, UploadFile, WebSocket from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, StreamingResponse +from orbit_sdk.visual import visual_nodes from pydantic import BaseModel, Field from . import store as store_module +from .assistant_graph import OrbitAssistantGraph, build_assistant_prompt from .assistant_tools import AssistantToolExecutor +from .assistant_ui import AssistantUiBroker, AssistantUiToolExecutor from .docker import preflight_docker +from .mcp_server import create_mcp_server from .providers import AzureOpenAIProvider, BedrockProvider, ModelSettings from .store import ConsoleStore from .terminal import serve_terminal +from .visual_runners import generate_source, validate_blueprint + + +@asynccontextmanager +async def application_lifespan(_: FastAPI): + """Run the MCP session manager for the same lifetime as the API server.""" + async with mcp_server.session_manager.run(): + try: + yield + finally: + store.shutdown() + app = FastAPI( title="OpenOrbit API", @@ -54,6 +74,7 @@ {"name": "Improvements", "description": "Read-only supervisor feedback and analytics."}, {"name": "System", "description": "Local service health and capabilities."}, ], + lifespan=application_lifespan, ) app.add_middleware( CORSMiddleware, @@ -61,17 +82,14 @@ allow_methods=["*"], allow_headers=["*"], ) -store = ConsoleStore() +# The API service owns in-memory scheduler threads, so it alone may mark a +# leftover local pipeline as interrupted when it starts. +store = ConsoleStore(recover_interrupted_runs=True) +assistant_ui_broker = AssistantUiBroker() WEB_DIST = Path(__file__).resolve().parents[2] / "frontend" / "dist" SDK_DOCS_DIST = Path(__file__).resolve().parents[2] / "site" -@app.on_event("shutdown") -def stop_active_runner_processes() -> None: - """Prevent browsers from outliving the local API process on reload.""" - store.shutdown() - - def safely(action): try: return action() @@ -137,6 +155,56 @@ def health_v1(): return health() +@app.get("/api/system/readiness", tags=["System"], summary="Get required local system capabilities") +def system_readiness(): + """Report prerequisites required by the control room as a whole. + + Keep this intentionally separate from build-specific validation: these + checks decide whether the application can safely offer its core features, + while repository, browser, and Docker requirements vary per build. + """ + application = store.application_settings() + profiles = {profile["profile_name"]: profile for profile in store.profiles()} + selected_profile = str(application.get("chat_model_profile_name", "")).strip() + profile = profiles.get(selected_profile) + + ai_detail = "ready" + if not selected_profile: + ai_detail = "profile_not_selected" + elif profile is None: + ai_detail = "profile_not_found" + elif not profile.get("model", "").strip(): + ai_detail = "model_not_set" + elif profile.get("provider") == "azure-openai": + if not profile.get("endpoint", "").strip(): + ai_detail = "endpoint_not_set" + elif not profile.get("secret_env", "").strip() or not os.environ.get(profile["secret_env"].strip()): + ai_detail = "secret_not_available" + elif profile.get("provider") == "aws-bedrock": + has_environment_credentials = bool( + os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY") + ) + if not profile.get("aws_profile", "").strip() and not has_environment_credentials: + ai_detail = "credentials_not_available" + + git_executable = shutil.which("git") + checks = [ + { + "id": "system_ai", + "status": "ready" if ai_detail == "ready" else "blocked", + "detail": ai_detail, + "settings_page": "settings", + }, + { + "id": "git", + "status": "ready" if git_executable else "blocked", + "detail": "ready" if git_executable else "not_installed", + "settings_page": None, + }, + ] + return {"ready": all(check["status"] == "ready" for check in checks), "checks": checks} + + @app.get("/api/v1", tags=["System"], operation_id="getApiInfo", summary="Get API entry-point information") def api_info_v1(): return { @@ -204,16 +272,6 @@ def run_telemetry(run_id: str): return safely(lambda: store.run_telemetry(run_id)) -@app.get("/api/runs/{run_id}/prompt-revisions") -def prompt_revisions(run_id: str): - return safely(lambda: store.prompt_revisions(run_id)) - - -@app.get("/api/runs/{run_id}/commit-changes") -def commit_changes(run_id: str): - return safely(lambda: store.commit_changes(run_id)) - - @app.get("/api/runs/{run_id}/artifacts/{loop_index}/{artifact_path:path}") def run_artifact(run_id: str, loop_index: int, artifact_path: str): return safely(lambda: FileResponse(store.run_artifact(run_id, loop_index, artifact_path))) @@ -254,6 +312,11 @@ def target_environments(): return store.target_environments() +@app.get("/api/personas") +def personas(): + return store.personas() + + class PromptTemplateUpdate(BaseModel): name: str = Field(min_length=1, max_length=120) content: str = Field(min_length=1, max_length=100_000) @@ -313,6 +376,19 @@ class TargetEnvironmentUpdate(BaseModel): managed_prompt_path: str = "" +class PersonaUpdate(BaseModel): + name: str = Field(min_length=1, max_length=120) + locale: str = Field(min_length=2, max_length=32) + timezone: str = Field(min_length=1, max_length=64) + activity_windows: list[dict] = Field(default_factory=list) + definition: str = Field(min_length=1, max_length=20_000) + context: dict = Field(default_factory=dict) + + +class PersonaCreate(PersonaUpdate): + id: str = Field(pattern=r"^[a-z][a-z0-9-]{2,63}$") + + @app.put("/api/prompt-templates/{template_id}") def update_prompt_template(template_id: str, values: PromptTemplateUpdate): return safely(lambda: store.update_prompt_template(template_id, values.model_dump())) @@ -338,6 +414,11 @@ def create_target_environment(values: TargetEnvironmentCreate): return safely(lambda: store.create_target_environment(values.model_dump())) +@app.post("/api/personas") +def create_persona(values: PersonaCreate): + return safely(lambda: store.create_persona(values.model_dump())) + + @app.put("/api/execution-environments/{environment_id}") def update_execution_environment(environment_id: str, values: ExecutionEnvironmentUpdate): return safely(lambda: store.update_execution_environment(environment_id, values.model_dump())) @@ -348,6 +429,11 @@ def update_target_environment(environment_id: str, values: TargetEnvironmentUpda return safely(lambda: store.update_target_environment(environment_id, values.model_dump())) +@app.put("/api/personas/{persona_id}") +def update_persona(persona_id: str, values: PersonaUpdate): + return safely(lambda: store.update_persona(persona_id, values.model_dump())) + + @app.delete("/api/execution-environments/{environment_id}") def delete_execution_environment(environment_id: str): return safely(lambda: store.delete_execution_environment(environment_id)) @@ -358,6 +444,11 @@ def delete_target_environment(environment_id: str): return safely(lambda: store.delete_target_environment(environment_id)) +@app.delete("/api/personas/{persona_id}") +def delete_persona(persona_id: str): + return safely(lambda: store.delete_persona(persona_id)) + + @app.delete("/api/prompt-templates/{template_id}") def delete_prompt_template(template_id: str): return safely(lambda: store.delete_prompt_template(template_id)) @@ -425,6 +516,7 @@ class BuildCreate(BaseModel): manager_template_id: str = "manager-default-v1" model_profile_name: str = "Default" test_case_set_id: str = Field(min_length=1, max_length=64) + persona_ids: list[str] = Field(default_factory=list) browser_base_url: str = Field(default="", max_length=2_000) browser_executable_path: str = Field(default="", max_length=4_000) browser_library_path: str = Field(default="", max_length=4_000) @@ -476,11 +568,16 @@ class IssueManagementDelete(BaseModel): proposal_ids: list[str] = Field(min_length=1, max_length=200) +class AgentIssueDecision(BaseModel): + decision: Literal["approve", "reject"] + + class RunnerAssetUpdate(BaseModel): name: str = Field(min_length=1, max_length=120) description: str = Field(min_length=1, max_length=500) source: str = Field(min_length=1, max_length=250_000) template_id: str | None = Field(default=None, max_length=64) + visual_blueprint: dict | None = None class RunnerAssetCreate(RunnerAssetUpdate): @@ -496,6 +593,16 @@ class RunnerGraphDraft(BaseModel): source: str = Field(min_length=1, max_length=250_000) +class VisualRunnerPreview(BaseModel): + blueprint: dict + + +class VisualRunnerUpdate(BaseModel): + name: str = Field(min_length=1, max_length=120) + description: str = Field(min_length=1, max_length=500) + blueprint: dict + + class RunnerTemplateValues(BaseModel): id: str = Field(pattern=r"^[a-z][a-z0-9-]{2,63}$") name: str = Field(min_length=1, max_length=120) @@ -594,6 +701,29 @@ def preview_runner_graph(values: RunnerGraphPreview): return safely(lambda: store.preview_runner_graph(values.source)) +@app.post("/api/visual-runners/preview") +def preview_visual_runner(values: VisualRunnerPreview): + blueprint = validate_blueprint(values.blueprint) + source = generate_source(blueprint) + return {"blueprint": blueprint, "source": source} + + +@app.get("/api/visual-runners/catalog") +def visual_runner_catalog(): + """Expose SDK-owned visual node metadata for editor clients.""" + return { + "nodes": [node for node in visual_nodes.catalog() if node.get("palette_visible", True)], + # Templates are selected before entering Visual Mode. The palette is + # reserved for reusable composition materials. + "starters": [], + } + + +@app.put("/api/runners/{runner_id}/visual") +def update_visual_runner(runner_id: str, values: VisualRunnerUpdate): + return safely(lambda: store.update_visual_runner(runner_id, values.model_dump())) + + @app.post("/api/runners") def create_runner(values: RunnerAssetCreate): return safely(lambda: store.create_runner(values.model_dump())) @@ -975,13 +1105,23 @@ def application_settings(): return store.application_settings() +@app.get("/api/assistant-mcp-config") +def assistant_mcp_config(): + return store.assistant_mcp_config() + + class ApplicationSettingsUpdate(BaseModel): manager_prompt_template: str = Field(default="", max_length=100_000) manager_output_locale: str = Field(default="", max_length=100) chat_model_profile_name: str = Field(default="", max_length=200) + coding_agent_provider: Literal["none", "kiro", "claude-code", "codex"] = "none" assistant_tools: dict | None = None +class AssistantMcpConfigUpdate(BaseModel): + content: str = Field(min_length=2, max_length=1_000_000) + + class ApplicationDataLocationUpdate(BaseModel): path: str = Field(min_length=1, max_length=4_096) @@ -1016,7 +1156,7 @@ def update_application_data(values: ApplicationDataLocationUpdate): if any(run.status in {"queued", "running", "awaiting_approval"} for run in store.runs()): raise ValueError("Stop active evaluation runs before changing the app data location") store_module.configure_application_data(values.path) - store = ConsoleStore() + store = ConsoleStore(recover_interrupted_runs=True) return application_data_summary() @@ -1025,6 +1165,16 @@ def update_application_settings(values: ApplicationSettingsUpdate): return store.save_application_settings(values.model_dump(exclude_unset=True)) +@app.put("/api/assistant-mcp-config") +def save_assistant_mcp_config(values: AssistantMcpConfigUpdate): + return safely(lambda: store.save_assistant_mcp_config(values.content)) + + +@app.post("/api/assistant-mcp-config/open-vscode") +def open_assistant_mcp_config_in_vscode(): + return safely(store.open_assistant_mcp_config_in_vscode) + + class ChatTurn(BaseModel): role: str = Field(pattern="^(user|assistant)$") content: str = Field(min_length=1, max_length=20_000) @@ -1033,6 +1183,13 @@ class ChatTurn(BaseModel): class ChatMessage(BaseModel): content: str = Field(min_length=1, max_length=20_000) history: list[ChatTurn] = Field(default_factory=list, max_length=12) + ui_session_id: str = Field(default="", max_length=200) + + +class AssistantUiResult(BaseModel): + session_id: str = Field(min_length=1, max_length=200) + command_id: str = Field(min_length=1, max_length=200) + result: dict class TemplateTranslationRequest(BaseModel): @@ -1168,29 +1325,105 @@ def chat(values: ChatMessage, request: Request): ) try: provider = AzureOpenAIProvider() if settings.provider == "azure-openai" else BedrockProvider() - history = "\n".join(f"{turn.role.title()}: {turn.content}" for turn in values.history) - prompt = ( - "You are Orbit, a concise assistant for the local OpenOrbit control room.\n" - "The local OpenAPI contract is available at /api/openapi.json; use it as the source of truth " - "when explaining API endpoints, parameters, and response shapes.\n" + application = store.application_settings() + tool_executor = AssistantToolExecutor( + application["assistant_tools"], application["coding_agent_provider"] ) - if locale := request_locale(request): - prompt += f"Respond in BCP 47 locale '{locale}'.\n" - if history: - prompt += f"Conversation so far:\n{history}\n\n" - prompt += f"User: {values.content}\nAssistant:" - tool_executor = AssistantToolExecutor(store.application_settings()["assistant_tools"]) - definitions = tool_executor.definitions() - response = ( - provider.complete_with_tools(settings, prompt, definitions, tool_executor.execute) - if definitions - else provider.complete(settings, prompt) + prompt = build_assistant_prompt( + values.content, + [(turn.role, turn.content) for turn in values.history], + request_locale(request), ) + response = OrbitAssistantGraph(provider, settings, tool_executor).invoke(prompt) return {"response": response, "profile_name": profile_name} except RuntimeError as error: raise HTTPException(409, str(error)) +@app.post("/api/chat/stream") +async def chat_stream(values: ChatMessage, request: Request): + profile_name = store.application_settings()["chat_model_profile_name"] + if not profile_name: + raise HTTPException(409, "Select an AI model profile for the chat assistant in Settings.") + configured = profile(store.profiles(), profile_name) + settings = ModelSettings( + **{key: value for key, value in configured.items() if key in ModelSettings.__dataclass_fields__} + ) + provider = AzureOpenAIProvider() if settings.provider == "azure-openai" else BedrockProvider() + application = store.application_settings() + tool_executor = AssistantToolExecutor( + application["assistant_tools"], application["coding_agent_provider"] + ) + prompt = build_assistant_prompt( + values.content, + [(turn.role, turn.content) for turn in values.history], + request_locale(request), + ui_enabled=bool(values.ui_session_id and application["assistant_tools"]["ui_context_enabled"]), + ) + + async def events(): + queue: asyncio.Queue[dict[str, object]] = asyncio.Queue() + loop = asyncio.get_running_loop() + + def send_ui_command(command_id: str, command: dict[str, object]) -> None: + loop.call_soon_threadsafe( + queue.put_nowait, + {"type": "ui_command", "command_id": command_id, "command": command}, + ) + + ui_tools = None + if values.ui_session_id and application["assistant_tools"]["ui_context_enabled"]: + assistant_ui_broker.open(values.ui_session_id, send_ui_command) + ui_tools = AssistantUiToolExecutor( + assistant_ui_broker, + values.ui_session_id, + context_enabled=True, + interaction_enabled=application["assistant_tools"]["ui_interaction_enabled"], + ) + + def activity(phase: str, tool: str | None = None) -> None: + event = {"type": "activity", "phase": phase} + if tool: + event["tool"] = tool + loop.call_soon_threadsafe(queue.put_nowait, event) + + async def run() -> None: + try: + response = await asyncio.to_thread( + OrbitAssistantGraph( + provider, settings, tool_executor, ui_tools=ui_tools, on_activity=activity + ).invoke, + prompt, + ) + await queue.put({"type": "response", "response": response}) + except RuntimeError as error: + await queue.put({"type": "error", "message": str(error)}) + except Exception: + await queue.put({"type": "error", "message": "Chat request failed."}) + + task = asyncio.create_task(run()) + try: + while True: + event = await queue.get() + yield json.dumps(event, ensure_ascii=False) + "\n" + if event["type"] in {"response", "error"}: + break + finally: + if not task.done(): + task.cancel() + if values.ui_session_id: + assistant_ui_broker.close(values.ui_session_id) + + return StreamingResponse(events(), media_type="application/x-ndjson") + + +@app.post("/api/chat/ui-results") +def chat_ui_result(values: AssistantUiResult): + if not assistant_ui_broker.resolve(values.session_id, values.command_id, values.result): + raise HTTPException(404, "Browser UI command was not found or has expired.") + return {"ok": True} + + @app.websocket("/api/terminal") async def terminal(websocket: WebSocket): await serve_terminal(websocket, store.application_settings()["assistant_tools"]) @@ -1326,32 +1559,27 @@ def list_improvements_v1(): return store.improvements() -@app.get( - "/api/v1/improvements/proposals", - tags=["Improvements"], - operation_id="listProposalLifecycles", - summary="List proposals and decisions from evaluation-run results", -) -def list_proposal_lifecycles_v1( - build_id: str | None = None, - status: Literal["proposed", "acceptable", "accepted", "rejected", "applied"] | None = None, -): - return store.proposal_lifecycles(build_id, status) +@app.get("/api/v1/issue-management", tags=["Improvements"], operation_id="listIssueManagementItems") +def list_issue_management_items_v1(): + return store.issue_management_items() @app.get( - "/api/v1/improvements/iterations", + "/api/v1/issue-management/{proposal_id}/diff", tags=["Improvements"], - operation_id="listImprovementIterationData", - summary="List SDK-saved data files by evaluation iteration", + operation_id="getIssueManagementDiff", ) -def list_improvement_iteration_data_v1(build_id: str | None = None): - return store.improvement_iteration_data(build_id) +def issue_management_diff_v1(proposal_id: str): + return safely(lambda: store.issue_management_diff(proposal_id)) -@app.get("/api/v1/issue-management", tags=["Improvements"], operation_id="listIssueManagementItems") -def list_issue_management_items_v1(): - return store.issue_management_items() +@app.post( + "/api/v1/issue-management/{proposal_id}/decision", + tags=["Improvements"], + operation_id="decideAgentIssue", +) +def decide_agent_issue_v1(proposal_id: str, values: AgentIssueDecision): + return safely(lambda: store.decide_agent_issue(proposal_id, values.decision)) @app.patch( @@ -1451,6 +1679,12 @@ def sdk_docs(path: str): raise HTTPException(404, "SDK documentation page was not found.") +# Streamable HTTP transport for MCP clients. The mounted application's root +# is the protocol endpoint, so the public URL is /mcp/. +mcp_server = create_mcp_server(lambda: store, app.openapi) +app.mount("/mcp", mcp_server.streamable_http_app()) + + @app.get("/{path:path}", include_in_schema=False) def local_web_app(path: str): """Serve the built React application for `orbit-agent-console run`.""" diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py new file mode 100644 index 0000000..0cc4c39 --- /dev/null +++ b/backend/app/mcp_server.py @@ -0,0 +1,122 @@ +"""MCP surface for the stable OpenOrbit v1 API. + +The tools deliberately call the same store operations as the HTTP handlers. +This keeps MCP local-first and avoids making the server call itself over HTTP. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Literal + +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from mcp.types import ToolAnnotations + +from .docker import preflight_docker +from .store import ConsoleStore + + +def _json(value: Any) -> Any: + """Convert Pydantic response models without changing ordinary JSON values.""" + return value.model_dump(mode="json") if hasattr(value, "model_dump") else value + + +def create_mcp_server( + get_store: Callable[[], ConsoleStore], openapi: Callable[[], dict[str, Any]] +) -> FastMCP: + """Create the streamable-HTTP MCP server mounted by the FastAPI app.""" + mcp = FastMCP( + "OpenOrbit", + instructions=( + "Operate the local OpenOrbit control plane. Read state before starting, approving, " + "rejecting, cancelling, or stopping pipelines. The OpenAPI resource is the complete " + "contract for the corresponding HTTP API." + ), + streamable_http_path="/", + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=["127.0.0.1:*", "localhost:*", "[::1]:*"], + ), + ) + + @mcp.resource("openorbit://openapi") + def openapi_schema() -> str: + """The generated OpenAPI 3 contract for OpenOrbit's versioned HTTP API.""" + import json + + return json.dumps(openapi(), ensure_ascii=False) + + @mcp.tool( + description="Get local service health, dashboard totals, active pipelines, and Docker readiness.", + annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False), + ) + def get_status() -> dict[str, Any]: + store = get_store() + return { + "health": {"status": "ok"}, + "dashboard": _json(store.dashboard()), + "active_pipelines": _json(store.active_evaluations()), + "docker": preflight_docker(), + } + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_projects() -> list[dict[str, Any]]: + """List all configured projects (the v1 API calls builds projects).""" + return [_json(project) for project in get_store().builds()] + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def get_project(project_id: str) -> dict[str, Any]: + """Get one project and its complete evaluation configuration.""" + return _json(get_store().build(project_id)) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_pipelines(project_id: str | None = None, status: str | None = None) -> list[dict[str, Any]]: + """List retained pipeline runs, optionally filtered by project or status.""" + pipelines = get_store().runs() + if project_id: + pipelines = [pipeline for pipeline in pipelines if pipeline.build_id == project_id] + if status: + pipelines = [pipeline for pipeline in pipelines if pipeline.status == status] + return [_json(pipeline) for pipeline in pipelines] + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def get_pipeline(pipeline_id: str) -> dict[str, Any]: + """Get a pipeline's current state, phases, evidence, and supervisor results.""" + return _json(get_store().run(pipeline_id)) + + @mcp.tool( + description="Start a project pipeline. Use test mode for a one-off safe validation run.", + annotations=ToolAnnotations(destructiveHint=False, idempotentHint=False, openWorldHint=False), + ) + def start_pipeline(project_id: str, execution_mode: Literal["run", "test"] = "run") -> dict[str, Any]: + """Start a retained run or transient test pipeline for a project.""" + return _json(get_store().invoke_remote_build(project_id, execution_mode, None)) + + @mcp.tool( + description="Approve, reject, or cancel a pipeline that is awaiting an operator action.", + annotations=ToolAnnotations(destructiveHint=True, idempotentHint=False, openWorldHint=False), + ) + def act_on_pipeline(pipeline_id: str, action: Literal["approve", "reject", "cancel"]) -> dict[str, Any]: + """Perform an approval-gated pipeline action.""" + actions = { + "approve": get_store().approve, + "reject": get_store().reject, + "cancel": get_store().cancel, + } + return _json(actions[action](pipeline_id)) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_improvements(build_id: str | None = None) -> list[dict[str, Any]]: + """List supervisor proposals and their decision lifecycle, optionally for one project.""" + return _json(get_store().proposal_lifecycles(build_id)) + + @mcp.tool( + description="Cancel every active local pipeline. Use only after confirming the affected work.", + annotations=ToolAnnotations(destructiveHint=True, idempotentHint=False, openWorldHint=False), + ) + def emergency_stop_pipelines() -> dict[str, Any]: + """Cancel all active pipelines immediately.""" + return _json(get_store().emergency_stop()) + + return mcp diff --git a/backend/app/models.py b/backend/app/models.py index f6fdb70..4612f6e 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -15,12 +15,22 @@ "teardown": "after_each", "finalize": "after_all", } -LIFECYCLE_PHASES = ("before_all", "before_each", "execute", "verify", "after_each", "after_all") +LIFECYCLE_PHASES = ( + "before_all", + "before_each", + "execute", + "verify", + "after_supervision", + "after_each", + "after_all", +) class Step(BaseModel): id: str - phase: Literal["before_all", "before_each", "execute", "verify", "after_each", "after_all"] + phase: Literal[ + "before_all", "before_each", "execute", "verify", "after_supervision", "after_each", "after_all" + ] name: str command: list[str] working_directory: str @@ -28,6 +38,7 @@ class Step(BaseModel): approval: Literal["not_required", "required"] = "not_required" on_failure: Literal["stop", "continue"] = "stop" minimum_interval_seconds: int = Field(default=0, ge=0, le=86_400) + after_supervision: bool = False @field_validator("phase", mode="before") @classmethod @@ -59,6 +70,16 @@ def lifecycle_is_complete(self, execution_mode: Literal["run", "test"] = "run") return phases in ( ["before_all", "before_each", "execute", "verify", "after_each"], ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_each", "execute", "verify", "after_supervision", "after_each"], + [ + "before_all", + "before_each", + "execute", + "verify", + "after_supervision", + "after_each", + "after_all", + ], ) diff --git a/backend/app/store.py b/backend/app/store.py index ea6939a..9d31ca8 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -26,14 +26,17 @@ import requests import yaml from opentelemetry.trace import Status, StatusCode +from orbit_sdk.visual import definition_for_runner_template from orbit import load_bundle +from .assistant_tools import DEFAULT_MCP_URL from .assistant_tools import normalize_settings as normalize_assistant_tools from .models import PHASE_ALIASES, Run, Step, Workflow from .observability import configure_telemetry from .providers import AzureOpenAIProvider, BedrockProvider, ModelSettings from .remote import RemoteInvocation +from .visual_runners import generate_source, validate_blueprint ROOT = Path(__file__).resolve().parents[2] @@ -82,8 +85,10 @@ def _application_data_dir() -> Path: TARGET_TEST_CASE_SETS = CONFIG / "target-ai-test-case-sets.yaml" EXECUTION_ENVIRONMENTS = CONFIG / "execution-environments.yaml" TARGET_ENVIRONMENTS = CONFIG / "target-environments.yaml" +PERSONAS = CONFIG / "personas.yaml" CYCLE_INTERVENTIONS = CONFIG / "cycle-interventions.yaml" ISSUE_MANAGEMENT = CONFIG / "issue-management.yaml" +ASSISTANT_MCP_CONFIG = CONFIG / "assistant-mcp.json" DEFAULT_OPERATIONAL_MANAGER_PROMPT = """You are an approval-first operations manager for recurring AI evaluations. Preserve the task safety boundary, collect observable evidence, and never claim success without stated acceptance evidence. Escalate required approvals @@ -96,8 +101,8 @@ def _application_data_dir() -> Path: Your final response must be exactly one JSON object: { \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_trace\": {\"persona_goal\":\"string\",\"current_action\":\"string\",\"decision\":\"string\",\"next_action\":\"string\",\"evidence\":\"string\"}}, - \"improvements\": [{\"title\":\"string\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], - \"reported_issues\": [{\"title\":\"string\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] + \"improvements\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], + \"reported_issues\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] } For an evaluated AI, include behavior_trace and fill every field. This is an evidence-backed persona journey, not the evaluator's procedure and not hidden reasoning. Keep the visible journey concise and written from the persona's perspective: persona_goal is the persona's stable wish; current_action is the one meaningful action taken in this iteration; decision is the resulting judgment or choice; next_action is the one specific, safe next action. Write each field as one to three natural sentences in the selected output language. Follow that language's normal grammar, ellipsis, and point of view; do not mechanically repeat a subject or pronoun across fields. Do not describe navigation, waits, screenshots, generic control inspection, or other repeated mechanics in any visible journey field. evidence is a concise source-backed factual record for the evidence drawer, not a visible journey item. Use the persona's wording where useful, but do not invent motives, feelings, beliefs, or facts beyond the declared persona and observed evidence. Do not reveal hidden reasoning or evaluator chain-of-thought. Do not include behavior_trace for non-AI targets. behavior_summary is deprecated and should be omitted. Always include both array keys, using empty arrays when there are no items.""" LEGACY_OPERATIONAL_MANAGER_PROMPT = """You are an approval-first operations manager for recurring AI evaluations. @@ -110,8 +115,8 @@ def _application_data_dir() -> Path: Your final response must be exactly one JSON object: { \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_summary\":\"string, only when the evaluated target is an AI\"}, - \"improvements\": [{\"title\":\"string\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], - \"reported_issues\": [{\"title\":\"string\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] + \"improvements\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], + \"reported_issues\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] } Include behavior_summary only when the evaluated target is an AI. It must describe the AI's observed responses, decisions, tool use, refusals, or other behavior in plain language; do not describe pass/fail outcomes, metrics, baselines, or the evaluator's actions. Omit behavior_summary for non-AI targets. Always include both array keys, using empty arrays when there are no items.""" PROPOSAL_DECISION_POLICY = """# Improvement decision policy @@ -148,12 +153,13 @@ def configure_application_data(path: str) -> Path: global APP_DATA, CONFIG, TARGET_TEST_CASE_SETS, EXECUTION_ENVIRONMENTS, TARGET_ENVIRONMENTS global CYCLE_INTERVENTIONS, DATA, RUNS, TELEMETRY, SETTINGS, TOOL_TIMES, RUNNERS - global RUNNER_TEMPLATES, QUICK_STARTS, QUICK_START_INSTANCES, TEMPLATE_TRANSLATIONS + global RUNNER_TEMPLATES, QUICK_STARTS, QUICK_START_INSTANCES, TEMPLATE_TRANSLATIONS, PERSONAS APP_DATA = target CONFIG = APP_DATA / "config" TARGET_TEST_CASE_SETS = CONFIG / "target-ai-test-case-sets.yaml" EXECUTION_ENVIRONMENTS = CONFIG / "execution-environments.yaml" TARGET_ENVIRONMENTS = CONFIG / "target-environments.yaml" + PERSONAS = CONFIG / "personas.yaml" CYCLE_INTERVENTIONS = CONFIG / "cycle-interventions.yaml" DATA = APP_DATA / "data" RUNS = DATA / "runs" @@ -175,7 +181,14 @@ def now() -> datetime: class ConsoleStore: """File-backed local state. Commands are always executed without a shell.""" - def __init__(self) -> None: + def __init__(self, *, recover_interrupted_runs: bool = False) -> None: + """Open local state without claiming ownership of the scheduler. + + Only the long-lived API service may recover unfinished local pipelines. + Diagnostic scripts also create a store to inspect assets; allowing those + short-lived processes to recover runs would incorrectly cancel the + pipeline owned by the already-running API service. + """ self._initialize_application_data() RUNS.mkdir(parents=True, exist_ok=True) RUNNERS.mkdir(parents=True, exist_ok=True) @@ -190,7 +203,8 @@ def __init__(self) -> None: self._test_sessions: dict[str, Run] = {} self._runner_graph_drafts: dict[str, tuple[str, float]] = {} self._lock = threading.Lock() - self._recover_interrupted_runs() + if recover_interrupted_runs: + self._recover_interrupted_runs() self.tracer = configure_telemetry(TELEMETRY) @staticmethod @@ -273,10 +287,10 @@ def _initialize_application_data() -> None: SETTINGS.write_text(json.dumps(document, indent=2), encoding="utf-8") @classmethod - def runner_templates(cls) -> list[dict[str, str]]: + def runner_templates(cls) -> list[dict[str, Any]]: """Load the versioned runner catalog shipped as ordinary template packages.""" root = ROOT / "templates" / "runner-templates" - templates: list[dict[str, str]] = [] + templates: list[dict[str, Any]] = [] if not root.is_dir(): return templates for directory in sorted(path for path in root.iterdir() if path.is_dir()): @@ -285,7 +299,22 @@ def runner_templates(cls) -> list[dict[str, str]]: continue values = json.loads(metadata.read_text(encoding="utf-8")) source = cls._package_source(directory, str(values.get("entrypoint", "runner.py"))) - templates.append({**values, "source": source}) + definition = definition_for_runner_template(directory.name) + templates.append( + { + **values, + "source": source, + **( + { + "visual_blueprint": deepcopy(definition.blueprint), + "visual_template_id": definition.id, + "visual_template_source_sha256": definition.source_sha256, + } + if definition + else {} + ), + } + ) return templates @staticmethod @@ -405,7 +434,11 @@ def _custom_runner_templates(self) -> list[dict[str, str]]: def available_runner_templates(self) -> list[dict[str, str]]: builtins = [{**item, "origin": "built-in"} for item in self.runner_templates()] - return [*builtins, *self._custom_runner_templates()] + builtin_ids = {str(item["id"]) for item in builtins} + custom = [ + item for item in self._custom_runner_templates() if str(item.get("id", "")) not in builtin_ids + ] + return [*builtins, *custom] def template_translation_input(self, kind: str, template_id: str) -> dict[str, Any]: """Return only display text that may safely be translated.""" @@ -854,7 +887,25 @@ def instantiate_quick_start(self, quick_start_id: str, inputs: dict[str, str]) - runner_paths = [RUNNERS / f"{generated['runner_id']}.py", RUNNERS / f"{generated['runner_id']}.json"] generated_workspace: Path | None = None try: - runner = self.create_runner({"id": generated["runner_id"], **assets["runner"]}) + runner_values: dict[str, Any] = {"id": generated["runner_id"], **assets["runner"]} + # Quick Start input substitution must travel with the Visual + # definition because a generated runner dispatches the same + # canonical template steps instead of retaining copied source. + from orbit_sdk.visual import definition_for_quick_start, instantiate_definition + + definition = definition_for_quick_start(quick_start_id) + if definition is not None: + parameter_values = { + str(item["key"]): values[str(item["key"])] for item in manifest["parameters"] + } + runner_values.update( + { + "visual_blueprint": instantiate_definition(definition, parameter_values), + "visual_template_id": definition.id, + "visual_template_source_sha256": definition.source_sha256, + } + ) + runner = self.create_runner(runner_values) prompt = self.create_prompt_template( {"id": generated["prompt_template_id"], **assets["prompt_template"]} ) @@ -1054,6 +1105,22 @@ def _runner(self, runner_id: str) -> dict[str, str]: def create_runner(self, values: dict[str, str]) -> dict[str, str]: if any(item["id"] == values["id"] for item in self.runners()): raise ValueError("runner ID already exists") + if not isinstance(values.get("visual_blueprint"), dict): + # Built-in runner templates have an authoritative Visual definition. + # User-imported templates deliberately remain source-owned. + from orbit_sdk.visual import definition_for_runner_template + + definition = definition_for_runner_template(str(values.get("template_id", ""))) + if definition is not None: + values = { + **values, + "visual_blueprint": deepcopy(definition.blueprint), + "visual_template_id": definition.id, + "visual_template_source_sha256": definition.source_sha256, + } + if isinstance(values.get("visual_blueprint"), dict): + blueprint = validate_blueprint(values["visual_blueprint"]) + values = {**values, "visual_blueprint": blueprint, "source": generate_source(blueprint)} return self._write_runner(values["id"], values) def migrate_runner_to_bundle(self, runner_id: str) -> dict[str, str]: @@ -1073,15 +1140,32 @@ def migrate_runner_to_bundle(self, runner_id: str) -> dict[str, str]: def update_runner(self, runner_id: str, values: dict[str, str]) -> dict[str, str]: existing = self._runner(runner_id) + detached_visual = bool(existing.get("visual_blueprint")) and not isinstance( + values.get("visual_blueprint"), dict + ) + if existing.get("visual_blueprint") and not isinstance(values.get("visual_blueprint"), dict): + values = {**values, "visual_blueprint": None} + merged = {**existing, **{key: value for key, value in values.items() if value is not None}} + if detached_visual: + merged.pop("visual_blueprint", None) + merged.pop("visual_template_id", None) + merged.pop("visual_template_source_sha256", None) return self._write_runner( runner_id, - {**existing, **{key: value for key, value in values.items() if value is not None}}, + merged, bundle=bool(existing.get("bundle")), ) + def update_visual_runner(self, runner_id: str, values: dict[str, Any]) -> dict[str, Any]: + """Compile and version one visual runner blueprint.""" + blueprint = validate_blueprint(dict(values["blueprint"])) + return self.update_runner( + runner_id, {**values, "source": generate_source(blueprint), "visual_blueprint": blueprint} + ) + def _write_runner( - self, runner_id: str, values: dict[str, str], *, bundle: bool = False - ) -> dict[str, str]: + self, runner_id: str, values: dict[str, Any], *, bundle: bool = False + ) -> dict[str, Any]: source = self._canonicalize_runner_source(str(values["source"])) compile(source, f"{runner_id}.py", "exec") existing_versions = list(values.get("versions") or []) @@ -1092,6 +1176,13 @@ def _write_runner( "sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(), "created_at": now().isoformat(), } + if isinstance(values.get("visual_blueprint"), dict): + version_record["visual_blueprint"] = values["visual_blueprint"] + if values.get("visual_template_id"): + version_record["visual_template_id"] = str(values["visual_template_id"]) + version_record["visual_template_source_sha256"] = str( + values.get("visual_template_source_sha256", "") + ) asset = { "id": runner_id, "name": str(values["name"]).strip(), @@ -1101,6 +1192,11 @@ def _write_runner( "version": version, "versions": [*existing_versions, version_record], } + if "visual_blueprint" in version_record: + asset["visual_blueprint"] = version_record["visual_blueprint"] + if "visual_template_id" in version_record: + asset["visual_template_id"] = version_record["visual_template_id"] + asset["visual_template_source_sha256"] = version_record["visual_template_source_sha256"] if not asset["name"] or not asset["description"]: raise ValueError("runner requires a name and description") source_path, metadata_path = ( @@ -1126,6 +1222,42 @@ def open_runner_in_vscode(self, runner_id: str) -> dict[str, str]: self._open_in_vscode(self._runner_entry_path(runner_id, int(self._runner(runner_id)["version"]))) return {"status": "opened"} + def assistant_mcp_config(self) -> dict[str, str]: + if not ASSISTANT_MCP_CONFIG.exists(): + return { + "path": str(ASSISTANT_MCP_CONFIG), + "content": json.dumps( + {"mcpServers": {"openorbit": {"url": DEFAULT_MCP_URL}}}, + indent=2, + ) + + "\n", + } + return { + "path": str(ASSISTANT_MCP_CONFIG), + "content": ASSISTANT_MCP_CONFIG.read_text(encoding="utf-8"), + } + + def save_assistant_mcp_config(self, content: str) -> dict[str, str]: + try: + document = json.loads(content) + except json.JSONDecodeError as error: + raise ValueError(f"MCP configuration must be valid JSON: {error.msg}") from error + if not isinstance(document, dict): + raise ValueError("MCP configuration must be a JSON object") + ASSISTANT_MCP_CONFIG.parent.mkdir(parents=True, exist_ok=True) + temporary = ASSISTANT_MCP_CONFIG.with_suffix(".tmp") + temporary.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + temporary.replace(ASSISTANT_MCP_CONFIG) + return self.assistant_mcp_config() + + def open_assistant_mcp_config_in_vscode(self) -> dict[str, str]: + if not ASSISTANT_MCP_CONFIG.exists(): + self.save_assistant_mcp_config( + json.dumps({"mcpServers": {"openorbit": {"url": DEFAULT_MCP_URL}}}) + ) + self._open_in_vscode(ASSISTANT_MCP_CONFIG) + return {"status": "opened"} + def delete_runner(self, runner_id: str) -> None: self._runner(runner_id) if any(build.get("runner_id") == runner_id for build in self.builds()): @@ -1136,6 +1268,47 @@ def delete_runner(self, runner_id: str) -> None: def _runner_execution_plan(self, runner_id: str, runner_version: int | None = None) -> Workflow: """Build the lifecycle declared by a runner without a workflow asset.""" runner = self._runner(runner_id) + definition = self._runner_graph_definition(runner_id, None, runner_version) + graph_nodes = definition.get("nodes", []) if isinstance(definition, dict) else [] + lifecycle_phases = {"before_all", "before_each", "execute", "verify", "after_each", "after_all"} + if isinstance(graph_nodes, list) and graph_nodes: + steps = [ + Step( + id=str(node["id"]), + phase=str(node["phase"]), + name=str(node.get("title") or node["id"]), + command=[ + sys.executable, + str(self._runner_entry_path(runner_id, runner_version)), + "--phase", + str(node["phase"]), + "--step", + str(node["id"]), + ], + working_directory=str(ROOT), + timeout_seconds=86_400 if node["phase"] == "execute" else 300, + approval="not_required", + on_failure="stop", + after_supervision=bool(node.get("after_supervision", False)), + ) + for node in graph_nodes + if isinstance(node, dict) and node.get("phase") in lifecycle_phases + ] + if steps: + return Workflow( + id=runner_id, + name=runner["name"], + description=runner["description"], + kind="improvement" + if runner.get("template_id") == "native-improvement-cycle" + else "simulation", + enabled=True, + risk="medium", + runner_id=runner_id, + runner_version=runner_version or int(runner["version"]), + steps=steps, + test_steps=deepcopy(steps), + ) source = runner["source"] if runner_version is not None: selected = next( @@ -1149,7 +1322,14 @@ def _runner_execution_plan(self, runner_id: str, runner_version: int | None = No if selected is None: raise ValueError(f"runner version {runner_version} does not exist") source = str(selected["source"]) - lifecycle_order = ("before_all", "before_each", "execute", "verify", "after_each", "after_all") + lifecycle_order = ( + "before_all", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ) declared = { PHASE_ALIASES.get(phase, phase) for phase in re.findall(r'@runner\.phase\(\s*["\']([^"\']+)["\']\s*\)', source) @@ -1280,8 +1460,10 @@ def builds(self) -> list[dict[str, Any]]: self._hydrate_build_environment(build) build.update(self._repository_metadata(str(build.get("repository", "")))) build.setdefault("created_at", fallback) - dates = [run.created_at for run in runs if run.build_id == build["id"]] + build_runs = [run for run in runs if run.build_id == build["id"]] + dates = [run.created_at for run in build_runs] build["last_run_at"] = max(dates).isoformat() if dates else None + build["run_count"] = len(build_runs) return builds def _hydrate_build_environment(self, build: dict[str, Any]) -> None: @@ -1371,6 +1553,8 @@ def _environment_variables_from_values(values: dict[str, Any]) -> dict[str, str] "ORBIT_ADAPTER_COMMAND", "ORBIT_AGENT_COMMAND", "ORBIT_PROBE_COMMAND", + "ORBIT_SELENIUM_COMMAND", + "ORBIT_SOURCE_CONTRACT_PATTERN", } normalized = {str(key).strip(): str(value) for key, value in variables.items()} if any( @@ -1639,6 +1823,103 @@ def target_test_case_sets(self) -> list[dict[str, Any]]: else [] ) + def personas(self) -> list[dict[str, Any]]: + values = yaml.safe_load(PERSONAS.read_text(encoding="utf-8")) if PERSONAS.exists() else [] + return ( + [ + {**item, "definition": self._persona_definition(item)} + for item in values + if isinstance(item, dict) + ] + if isinstance(values, list) + else [] + ) + + @staticmethod + def _persona_definition(values: dict[str, Any]) -> str: + """Migrate legacy goal/constraint arrays into one Markdown definition.""" + definition = str(values.get("definition", "")).strip() + if definition: + return definition + sections = [] + goals = [str(goal).strip() for goal in values.get("goals", []) if str(goal).strip()] + constraints = [ + str(constraint).strip() for constraint in values.get("constraints", []) if str(constraint).strip() + ] + if goals: + sections.append("# Goals\n\n" + "\n".join(f"- {goal}" for goal in goals)) + if constraints: + sections.append("# Constraints\n\n" + "\n".join(f"- {constraint}" for constraint in constraints)) + return "\n\n".join(sections) + + @staticmethod + def _validated_persona(values: dict[str, Any], persona_id: str) -> dict[str, Any]: + name = str(values.get("name", "")).strip() + locale, timezone = str(values.get("locale", "")).strip(), str(values.get("timezone", "")).strip() + definition, windows, context = ( + str(values.get("definition", "")).strip(), + values.get("activity_windows", []), + values.get("context", {}), + ) + if not name or not locale or not timezone or not definition: + raise ValueError("persona requires a name, locale, timezone, and Markdown definition") + if not isinstance(windows, list) or not all(isinstance(window, dict) for window in windows): + raise ValueError("persona activity windows must be a list of objects") + if not isinstance(context, dict): + raise ValueError("persona context must be an object") + return { + "id": persona_id, + "name": name, + "locale": locale, + "timezone": timezone, + "activity_windows": windows, + "definition": definition, + "context": context, + } + + def create_persona(self, values: dict[str, Any]) -> dict[str, Any]: + persona_id = str(values.get("id", "")).strip() + personas = self.personas() + if not persona_id or any(item.get("id") == persona_id for item in personas): + raise ValueError("persona ID is required and must be unique") + persona = self._validated_persona(values, persona_id) + persona["created_at"] = now().isoformat() + personas.append(persona) + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text(yaml.safe_dump(personas, allow_unicode=True, sort_keys=False), encoding="utf-8") + temporary.replace(PERSONAS) + return persona + + def update_persona(self, persona_id: str, values: dict[str, Any]) -> dict[str, Any]: + personas = self.personas() + index = next((i for i, item in enumerate(personas) if item.get("id") == persona_id), None) + if index is None: + raise KeyError(persona_id) + persona = self._validated_persona(values, persona_id) + persona["created_at"] = personas[index].get("created_at", now().isoformat()) + personas[index] = persona + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text(yaml.safe_dump(personas, allow_unicode=True, sort_keys=False), encoding="utf-8") + temporary.replace(PERSONAS) + return persona + + def delete_persona(self, persona_id: str) -> None: + personas = self.personas() + if not any(item.get("id") == persona_id for item in personas): + raise KeyError(persona_id) + if any(persona_id in build.get("persona_ids", []) for build in self.builds()): + raise ValueError("persona is used by a build") + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text( + yaml.safe_dump( + [item for item in personas if item.get("id") != persona_id], + allow_unicode=True, + sort_keys=False, + ), + encoding="utf-8", + ) + temporary.replace(PERSONAS) + @staticmethod def _validated_target_test_case_set(values: dict[str, Any], set_id: str) -> dict[str, Any]: name, description = str(values.get("name", "")).strip(), str(values.get("description", "")).strip() @@ -1955,6 +2236,12 @@ def create_build(self, values: dict[str, Any]) -> dict[str, Any]: raise ValueError("AI model profile does not exist") if not any(item.get("id") == values.get("test_case_set_id") for item in self.target_test_case_sets()): raise ValueError("target-AI test case set does not exist") + persona_ids = [str(item) for item in values.get("persona_ids", [])] + if len(set(persona_ids)) != len(persona_ids) or any( + not any(persona.get("id") == persona_id for persona in self.personas()) + for persona_id in persona_ids + ): + raise ValueError("persona selection contains an unknown or duplicate persona") build = { "id": build_id, "name": values["name"], @@ -1975,6 +2262,7 @@ def create_build(self, values: dict[str, Any]) -> dict[str, Any]: "model_profile_name": values.get("model_profile_name", "Default"), "task_instruction": values.get("task_instruction", ""), "test_case_set_id": values["test_case_set_id"], + "persona_ids": persona_ids, "browser_base_url": str(target_environment.get("browser_base_url", "")).strip(), "browser_executable_path": str(execution_environment.get("browser_executable_path", "")).strip(), "browser_library_path": str(execution_environment.get("browser_library_path", "")).strip(), @@ -2041,6 +2329,12 @@ def update_build(self, build_id: str, values: dict[str, Any]) -> dict[str, Any]: raise ValueError("AI model profile does not exist") if not any(item.get("id") == values.get("test_case_set_id") for item in self.target_test_case_sets()): raise ValueError("target-AI test case set does not exist") + persona_ids = [str(item) for item in values.get("persona_ids", [])] + if len(set(persona_ids)) != len(persona_ids) or any( + not any(persona.get("id") == persona_id for persona in self.personas()) + for persona_id in persona_ids + ): + raise ValueError("persona selection contains an unknown or duplicate persona") existing = builds[index] build = { "id": build_id, @@ -2062,6 +2356,7 @@ def update_build(self, build_id: str, values: dict[str, Any]) -> dict[str, Any]: "model_profile_name": values.get("model_profile_name", "Default"), "task_instruction": existing.get("task_instruction", ""), "test_case_set_id": values["test_case_set_id"], + "persona_ids": persona_ids, "browser_base_url": str(target_environment.get("browser_base_url", "")).strip(), "browser_executable_path": str(execution_environment.get("browser_executable_path", "")).strip(), "browser_library_path": str(execution_environment.get("browser_library_path", "")).strip(), @@ -2114,6 +2409,66 @@ def improvements(self) -> list[dict[str, Any]]: path = CONFIG / "improvements.yaml" return yaml.safe_load(path.read_text(encoding="utf-8")) if path.exists() else [] + @staticmethod + def _issue_category(value: Any) -> str: + """Normalize persisted categories; legacy implementation targets are never categories.""" + categories = {"accuracy_grounding", "safety_privacy", "task_completion", "user_experience", "other"} + return str(value) if str(value) in categories else "other" + + def _known_unresolved_issues(self, run: Run, *, limit: int = 12) -> list[dict[str, str]]: + """Return compact, active Issue context for a supervisor prompt. + + Issue Management is derived from immutable supervisor results, so this + deliberately scans earlier issue-assessment responses instead of a + separate mutable issue store. Resolved and deleted items are excluded: + a later observation of either may be a genuine regression. + """ + records = self._issue_management_records() + candidates = [item for item in self.runs() if item.build_id == run.build_id] + if all(item.id != run.id for item in candidates): + candidates.append(run) + values: list[dict[str, str]] = [] + for source_run in candidates: + for result in source_run.supervisor_results: + if ( + not isinstance(result, dict) + or result.get("stage", "issue_assessment") != "issue_assessment" + ): + continue + response = result.get("response") + if not isinstance(response, dict): + continue + improvements = response.get("improvements") + reported = response.get("reported_issues") + proposals = improvements if isinstance(improvements, list) and improvements else reported + if not isinstance(proposals, list): + continue + for index, proposal in enumerate(proposals): + if not isinstance(proposal, dict): + continue + proposal_id = f"{source_run.id}:{result.get('iteration', 0)}:{index}" + if proposals is reported: + proposal_id = f"{source_run.id}:{result.get('iteration', 0)}:issue:{index}" + management = records.get(proposal_id, {}) + if management.get("status") in {"resolved", "deleted"}: + continue + if str(proposal.get("status") or "").lower() == "rejected": + continue + values.append( + { + "id": proposal_id, + "title": str(proposal.get("title") or "")[:240], + "category": self._issue_category(proposal.get("category")), + "rationale": str( + proposal.get("rationale") or proposal.get("acceptanceEvidence") or "" + )[:700], + "evidence": str(proposal.get("evidence") or proposal.get("reproduction") or "")[ + :700 + ], + } + ) + return values[-limit:] + def cycle_interventions(self) -> list[dict[str, Any]]: return ( yaml.safe_load(CYCLE_INTERVENTIONS.read_text(encoding="utf-8")) @@ -2143,6 +2498,7 @@ def proposal_lifecycles( "response": run.supervisor_response, } ] + seen_agent_improvements: set[str] = set() for record in records: if not isinstance(record, dict): continue @@ -2182,15 +2538,42 @@ def proposal_lifecycles( if isinstance(persona, str) and persona not in personas: personas.append(persona) improvements = response.get("improvements", []) + reported_issues = response.get("reported_issues", []) evaluation = response.get("evaluation") score = evaluation.get("score") if isinstance(evaluation, dict) else None if isinstance(score, bool) or not isinstance(score, (int, float)): score = None - if not isinstance(improvements, list): + if not isinstance(improvements, list) or not isinstance(reported_issues, list): continue - for index, proposal in enumerate(improvements): + # The first supervision pass produces Issue proposals and their + # operator-facing rationale. A later agent-proposal assessment + # evaluates a worktree diff; its generated improvements are + # review feedback, not new persona/Issue proposals. + # Improvements are the operator-facing Issue proposals. A + # reported issue is used only when no improvement was made, + # preventing one supervisor observation from becoming two + # indistinguishable Issue Management rows. + issue_proposals = [] + if record.get("stage") != "agent_proposal_assessment": + issue_proposals = ( + [ + (f"{run.id}:{record.get('iteration', 0)}:{index}", proposal) + for index, proposal in enumerate(improvements) + ] + if improvements + else [ + (f"{run.id}:{record.get('iteration', 0)}:issue:{index}", proposal) + for index, proposal in enumerate(reported_issues) + ] + ) + for proposal_id, proposal in issue_proposals: if not isinstance(proposal, dict): continue + # A supervisor may retain fresh evidence against an active + # Issue by referring to its stable ID. It is not a new + # Issue Management row until the original issue is resolved. + if str(proposal.get("known_issue_id") or "").strip(): + continue source_status = str(proposal.get("status") or "proposed").lower() decision = ( "accepted" @@ -2203,9 +2586,11 @@ def proposal_lifecycles( ) values.append( { - "proposal_id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "proposal_id": proposal_id, "title": str(proposal.get("title") or "Untitled proposal"), - "target": str(proposal.get("target") or "prompt"), + "category": self._issue_category(proposal.get("category")), + # target is legacy execution metadata. Do not expose it as an Issue category. + "target": str(proposal.get("target") or ""), "proposal": proposal, "decision": decision, "score": score, @@ -2223,9 +2608,9 @@ def proposal_lifecycles( "prompt_version": None, "events": [ { - "id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "id": proposal_id, "event_type": "decision", - "proposal_id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "proposal_id": proposal_id, "decision": decision, "rationale": str(proposal.get("rationale") or ""), "recorded_at": record.get("recorded_at") or run.updated_at.isoformat(), @@ -2236,6 +2621,90 @@ def proposal_lifecycles( ], } ) + # Coding-agent changes are first-class review proposals too. + # They are emitted by the SDK only when the agent supplied + # explicit feedback, and their immutable worktree diff stays + # attached to the resulting Issue. + if record.get("stage") != "agent_proposal_assessment": + continue + for step in run.step_results: + if not isinstance(step, dict) or step.get("loop_index") != iteration: + continue + result = step.get("result") + agent = result.get("agent_run") if isinstance(result, dict) else None + if not isinstance(agent, dict) or not str(agent.get("feedback") or "").strip(): + continue + proposal_data = agent.get("proposal") + proposal_data = proposal_data if isinstance(proposal_data, dict) else {} + agent_proposal = result.get("agent_proposal") if isinstance(result, dict) else {} + linked_issue = agent_proposal.get("issue") if isinstance(agent_proposal, dict) else None + fingerprint = str(proposal_data.get("fingerprint") or "") + if not fingerprint or fingerprint in seen_agent_improvements: + continue + seen_agent_improvements.add(fingerprint) + approval = ( + str(evaluation.get("approval") or "pending").lower() + if isinstance(evaluation, dict) + else "pending" + ) + decision = ( + "accepted" + if approval == "approved" + else "rejected" + if approval == "rejected" + else "pending" + ) + parent_id = ( + str(linked_issue.get("_orbit_issue_id") or "") + if isinstance(linked_issue, dict) + else "" + ) + parent = next( + (item for item in reversed(values) if item["proposal_id"] == parent_id), None + ) + # Backward-compatible attachment for evidence retained before stable IDs existed. + if parent is None and isinstance(linked_issue, dict): + linked_title = str(linked_issue.get("title") or "") + parent = next( + ( + item + for item in reversed(values) + if item.get("run_id") == run.id + and item.get("iteration") == iteration + and item.get("title") == linked_title + ), + None, + ) + if parent is None: + candidates = [ + item + for item in values + if item.get("run_id") == run.id and item.get("iteration") == iteration + ] + if len(candidates) == 1: + parent = candidates[0] + if parent is None: + continue + parent_proposal = parent.get("proposal") + if not isinstance(parent_proposal, dict): + continue + parent_proposal["agent_change"] = { + "feedback": str(agent.get("feedback")).strip(), + "changed_files": agent.get("changed_files", []), + "diff": proposal_data.get("diff", ""), + "diff_artifact": proposal_data.get("diff_artifact"), + "base_revision": proposal_data.get("base_revision"), + "branch": proposal_data.get("branch", ""), + "worktree_path": proposal_data.get("worktree_path", ""), + "fingerprint": fingerprint, + "review": { + "decision": decision, + "score": score, + "summary": (evaluation or {}).get("summary", "") + if isinstance(evaluation, dict) + else "", + }, + } if status: values = [item for item in values if item["status"] == status or item["decision"] == status] return sorted(values, key=lambda item: str(item.get("recorded_at", "")), reverse=True) @@ -2271,13 +2740,31 @@ def issue_management_items(self) -> list[dict[str, Any]]: # management-level tombstone, not deletion of the underlying run. if record.get("status") == "deleted": continue + agent_change = (proposal.get("proposal") or {}).get("agent_change") or {} + review = agent_change.get("review") if isinstance(agent_change, dict) else None + review_comment = [] + if isinstance(review, dict) and str(review.get("summary") or "").strip(): + review_comment = [ + { + "body": str(review["summary"]).strip(), + "assigner": "AI supervisor", + "recorded_at": proposal.get("recorded_at", ""), + "kind": "agent_change_review", + "decision": review.get("decision"), + } + ] values.append( { **proposal, "management_status": record.get("status", "unreviewed"), + "proposal_action": record.get("proposal_action", "pending"), + "proposal_branch": str( + ((proposal.get("proposal") or {}).get("agent_change") or {}).get("branch") or "" + ), + "proposal_commit": str(record.get("proposal_commit") or ""), "assigner": record.get("assigner", ""), - "comments": record.get("comments", []), - "management_events": record.get("events", []), + "comments": [*review_comment, *record.get("comments", [])], + "management_events": [*review_comment, *record.get("events", [])], } ) return values @@ -2340,52 +2827,113 @@ def delete_issue_management_items(self, proposal_ids: list[str]) -> dict[str, in self._save_issue_management_records(records) return {"deleted": deleted} - def improvement_iteration_data(self, build_id: str | None = None) -> list[dict[str, Any]]: - """List SDK-saved data files by persisted evaluation run and iteration. + def issue_management_diff(self, proposal_id: str) -> dict[str, Any]: + """Return the immutable worktree patch attached to an agent Issue.""" + item = next( + (value for value in self.issue_management_items() if value["proposal_id"] == proposal_id), None + ) + if item is None: + raise KeyError(proposal_id) + proposal = item.get("proposal") + agent_change = proposal.get("agent_change") if isinstance(proposal, dict) else None + if agent_change is None and isinstance(proposal, dict) and proposal.get("kind") == "agent_change": + agent_change = proposal + if not isinstance(agent_change, dict): + return {"diff": "", "changed_files": []} + changed_files = agent_change.get("changed_files") + return { + "diff": str(agent_change.get("diff") or ""), + "changed_files": changed_files if isinstance(changed_files, list) else [], + "base_revision": agent_change.get("base_revision"), + } - These entries intentionally exist even when a supervisor did not make a - proposal. The proposal-decision history uses them to expose the full - iteration record rather than hiding developer-retained evidence behind - a proposal requirement. - """ - values: list[dict[str, Any]] = [] - for run in self.runs(): - if build_id and run.build_id != build_id: - continue - grouped: dict[int, list[dict[str, Any]]] = {} - timestamps: dict[int, str] = {} - for step in run.step_results: - if not isinstance(step, dict) or not isinstance(step.get("loop_index"), int): - continue - iteration = step["loop_index"] - for data_file in step.get("data_files", []): - if not isinstance(data_file, dict): - continue - path, filename = data_file.get("path"), data_file.get("filename") - if not isinstance(path, str) or not isinstance(filename, str): - continue - grouped.setdefault(iteration, []).append( - { - "label": str(data_file.get("label") or "")[:256], - "filename": filename, - "path": path, - "relative_path": str(data_file.get("relative_path") or ""), - "content_type": str(data_file.get("content_type") or ""), - } - ) - timestamps.setdefault(iteration, str(step.get("ended_at") or run.updated_at.isoformat())) - for iteration, data_files in grouped.items(): - values.append( - { - "build_id": run.build_id, - "build_name": run.build_name, - "run_id": run.id, - "iteration": iteration, - "recorded_at": timestamps[iteration], - "data_files": data_files, - } + def decide_agent_issue(self, proposal_id: str, decision: str) -> dict[str, Any]: + """Commit or discard a pending agent worktree proposal from its Issue.""" + if decision not in {"approve", "reject"}: + raise ValueError("decision must be approve or reject") + item = next( + (value for value in self.issue_management_items() if value["proposal_id"] == proposal_id), None + ) + if item is None: + raise KeyError(proposal_id) + proposal = item.get("proposal") + agent_change = proposal.get("agent_change") if isinstance(proposal, dict) else None + if agent_change is None and isinstance(proposal, dict) and proposal.get("kind") == "agent_change": + agent_change = proposal + if not isinstance(agent_change, dict): + raise ValueError("only agent worktree proposals can be decided from an Issue") + branch = str(agent_change.get("branch") or "").strip() + worktree_value = str(agent_change.get("worktree_path") or "").strip() + if not branch or not worktree_value: + raise ValueError("this agent proposal was created before worktree review was enabled") + worktree = Path(worktree_value).resolve() + worktree_root = (APP_DATA / "agent-worktrees").resolve() + if worktree_root not in worktree.parents: + raise ValueError("agent proposal worktree is outside Orbit storage") + source_repository = Path(self._load(str(item.get("run_id") or "")).repository).resolve() + if not source_repository.is_dir(): + raise ValueError("agent proposal source repository is no longer available") + records = self._issue_management_records() + record = records.setdefault(proposal_id, {"status": "unreviewed", "comments": [], "events": []}) + previous = str(record.get("proposal_action") or "pending") + if previous != "pending": + return next( + value for value in self.issue_management_items() if value["proposal_id"] == proposal_id + ) + timestamp = now().isoformat() + if decision == "approve": + if not worktree.is_dir(): + raise ValueError("agent proposal worktree is no longer available") + changed = subprocess.run( + ["git", "diff", "--quiet", "--"], cwd=worktree, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if changed.returncode == 0: + raise ValueError("agent proposal has no uncommitted changes") + if changed.returncode > 1: + raise ValueError( + changed.stderr.decode("utf-8", "replace").strip() or "cannot inspect agent proposal" ) - return sorted(values, key=lambda item: str(item["recorded_at"]), reverse=True) + subprocess.run( + ["git", "add", "--all"], + cwd=worktree, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + message = f"Orbit agent proposal: {str(agent_change.get('feedback') or 'approved change')[:160]}" + subprocess.run( + ["git", "commit", "-m", message], + cwd=worktree, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=worktree, check=True, text=True, stdout=subprocess.PIPE + ).stdout.strip() + record.update({"proposal_action": "approved", "proposal_commit": commit, "status": "resolved"}) + record["events"].append( + {"type": "proposal_approved", "branch": branch, "commit": commit, "recorded_at": timestamp} + ) + else: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=source_repository, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "branch", "-D", branch], + cwd=source_repository, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + record.update({"proposal_action": "rejected", "status": "resolved"}) + record["events"].append({"type": "proposal_rejected", "branch": branch, "recorded_at": timestamp}) + self._save_issue_management_records(records) + return next(value for value in self.issue_management_items() if value["proposal_id"] == proposal_id) def _save_cycle_interventions(self, values: list[dict[str, Any]]) -> None: temporary = CYCLE_INTERVENTIONS.with_suffix(".tmp") @@ -2604,51 +3152,6 @@ def prompt_revisions(self, run_id: str) -> list[dict[str, Any]]: current_by_path[path] = after return [*initial_revisions, *ordered] - def commit_changes(self, run_id: str) -> list[dict[str, Any]]: - """Return commit ranges automatically retained by SDK runner phases.""" - run = self._load(run_id) - changes: list[dict[str, Any]] = [] - for step in run.step_results: - result = step.get("result") - event = result.get("commit_change") if isinstance(result, dict) else None - if not isinstance(event, dict) and isinstance(result, dict): - jgent = result.get("jgent_paired") - candidate = jgent.get("committed_source_candidate") if isinstance(jgent, dict) else None - event = candidate if isinstance(candidate, dict) else None - if not isinstance(event, dict): - continue - before, after = str(event.get("before") or ""), str(event.get("after") or "") - if not before or not after or before == after: - continue - artifact = event.get("diff_artifact") if isinstance(event.get("diff_artifact"), dict) else None - diff: str | None = None - if artifact: - raw_path = artifact.get("path") - candidate = Path(str(raw_path)).resolve() if isinstance(raw_path, str) else None - artifacts_root = (APP_DATA / "artifacts").resolve() - if candidate and artifacts_root in candidate.parents: - try: - if candidate.stat().st_size <= 500_000: - diff = candidate.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - pass - changes.append( - { - "iteration": step.get("loop_index"), - "phase": step.get("phase"), - "recorded_at": step.get("ended_at"), - "before": before, - "after": after, - "changed_paths": event.get("changed_paths") - if isinstance(event.get("changed_paths"), list) - else [], - "commits": event.get("commits") if isinstance(event.get("commits"), list) else [], - "diff_artifact": artifact, - "diff": diff, - } - ) - return sorted(changes, key=lambda item: str(item.get("recorded_at") or ""), reverse=True) - def run_artifact(self, run_id: str, loop_index: int, relative_path: str) -> Path: """Resolve one retained run artifact without permitting path traversal.""" if loop_index < 0: @@ -3227,6 +3730,7 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": normalize_assistant_tools(None), } stored = json.loads(SETTINGS.read_text(encoding="utf-8")) @@ -3236,6 +3740,7 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": normalize_assistant_tools(None), } prompt = str(values.get("manager_prompt_template", "")).strip() @@ -3248,6 +3753,12 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": prompt or DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": output_locale, "chat_model_profile_name": str(values.get("chat_model_profile_name", "")).strip(), + "coding_agent_provider": ( + str(values.get("coding_agent_provider", "none")).strip() + if str(values.get("coding_agent_provider", "none")).strip() + in {"kiro", "claude-code", "codex"} + else "none" + ), "assistant_tools": normalize_assistant_tools(values.get("assistant_tools")), } @@ -3272,6 +3783,12 @@ def save_application_settings(self, values: dict[str, Any]) -> dict[str, Any]: ).strip(), "manager_output_locale": output_locale, "chat_model_profile_name": chat_profile_name, + "coding_agent_provider": ( + str(values.get("coding_agent_provider", current["coding_agent_provider"])).strip() + if str(values.get("coding_agent_provider", current["coding_agent_provider"])).strip() + in {"none", "kiro", "claude-code", "codex"} + else "none" + ), "assistant_tools": normalize_assistant_tools( values.get("assistant_tools", current["assistant_tools"]) ), @@ -3384,6 +3901,10 @@ def _execute(self, run_id: str) -> None: None, ) resources["test_cases"] = selected.get("cases", []) if selected else build.get("test_cases", []) + selected_personas = set(build.get("persona_ids", [])) + resources["personas"] = [ + persona for persona in self.personas() if persona.get("id") in selected_personas + ] profile_name = str(build.get("model_profile_name", "")) resources["model_profile"] = next( (profile for profile in self.profiles() if profile.get("profile_name") == profile_name), @@ -3396,7 +3917,14 @@ def _execute(self, run_id: str) -> None: if workflow.runner_id: runner_path = self._runner_entry_path(workflow.runner_id, run.runner_version) for step in [*workflow.steps, *(workflow.test_steps or [])]: - step.command = [sys.executable, str(runner_path), "--phase", step.phase] + step.command = [ + sys.executable, + str(runner_path), + "--phase", + step.phase, + "--step", + step.id, + ] step.working_directory = run.repository or str(ROOT) with self.tracer.start_as_current_span( "workflow.run", @@ -3415,8 +3943,13 @@ def _execute(self, run_id: str) -> None: self._save(run) steps = workflow.steps_for(run.execution_mode) before_all = [step for step in steps if step.phase == "before_all"] - loop_steps = [step for step in steps if step.phase in {"before_each", "execute", "verify"}] - after_each = [step for step in steps if step.phase == "after_each"] + loop_steps = [ + step + for step in steps + if step.phase in {"before_each", "execute", "verify"} and not step.after_supervision + ] + post_supervision_steps = [step for step in steps if step.after_supervision] + after_each = [step for step in steps if step.phase == "after_each" and not step.after_supervision] after_all = [step for step in steps if step.phase == "after_all"] if not self._wait_for_schedule(run_id): return @@ -3513,15 +4046,23 @@ def _execute(self, run_id: str) -> None: self._execute_step(run_id, step, loop_index, resources) if self._load(run_id).status in {"failed", "cancelled"}: break - # Cleanup is part of the lifecycle contract, not merely the - # happy path. Run it after an earlier phase fails or a user - # cancels the Run; the terminal status remains unchanged. + # Retain all iteration evidence before the supervisor selects + # an Issue. The post-supervision agent work is displayed as an + # after_each step, but runs only after that selection exists. for step in after_each: self._execute_step(run_id, step, loop_index, resources, allow_terminal=True) - if self._load(run_id).status in {"failed", "cancelled"}: - break + # The supervisor first assesses discovered issues. The agent + # may then create a proposal for an issue the supervisor did + # not reject; only that concrete proposal receives the second, + # independent score/decision. if run.execution_mode == "run" and self._load(run_id).status == "running": self._complete_supervision(run_id) + for step in post_supervision_steps: + self._execute_step(run_id, step, loop_index, resources) + if post_supervision_steps and self._load(run_id).status == "running": + self._complete_supervision(run_id, stage="agent_proposal_assessment") + if self._load(run_id).status in {"failed", "cancelled"}: + break if ( loop_index < run.loop_limit and run.repeat_interval_minutes @@ -3603,7 +4144,7 @@ def _latest_cycle_has_persona_evidence(run: Run) -> bool: return False @staticmethod - def _validated_supervisor_result(text: str) -> dict[str, Any]: + def _validated_supervisor_result(text: str, *, require_evaluation: bool = False) -> dict[str, Any]: """Validate the exact structured result required by the manager template.""" try: result = json.loads(text) @@ -3611,10 +4152,8 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: raise ValueError("supervisor did not return valid JSON") from error if not isinstance(result, dict) or not {"improvements", "reported_issues"}.issubset(result): raise ValueError("supervisor JSON must contain improvements and reported_issues") - if set(result) not in ( - {"improvements", "reported_issues"}, - {"evaluation", "improvements", "reported_issues"}, - ): + allowed_keys = {"evaluation", "persona_journeys", "improvements", "reported_issues"} + if not set(result).issubset(allowed_keys): raise ValueError("supervisor JSON contains unsupported result fields") if not all( isinstance(result[key], list) and all(isinstance(item, dict) for item in result[key]) @@ -3622,6 +4161,32 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: ): raise ValueError("supervisor improvements and reported_issues must be arrays of objects") evaluation = result.get("evaluation") + if require_evaluation and evaluation is None: + raise ValueError("supervisor evaluation is required for a registered agent result") + if evaluation is not None: + ConsoleStore._validate_evaluation(evaluation) + journeys = result.get("persona_journeys", []) + if not isinstance(journeys, list) or not all(isinstance(item, dict) for item in journeys): + raise ValueError("supervisor persona_journeys must be an array of objects") + for journey in journeys: + if ( + set(journey) != {"persona_id", "behavior_trace"} + or not isinstance(journey["persona_id"], str) + or not journey["persona_id"].strip() + ): + raise ValueError("supervisor persona journey is invalid") + ConsoleStore._validate_persona_journey_trace(journey["behavior_trace"]) + for issue in result["reported_issues"]: + issue_evaluation = issue.get("evaluation") + if issue_evaluation is not None: + ConsoleStore._validate_evaluation(issue_evaluation) + for issue in [*result["reported_issues"], *result["improvements"]]: + issue["category"] = ConsoleStore._issue_category(issue.get("category")) + return result + + @staticmethod + def _validate_evaluation(evaluation: Any) -> None: + """Validate either an issue assessment or an agent-proposal assessment.""" if evaluation is not None: if ( not isinstance(evaluation, dict) @@ -3685,9 +4250,56 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: or not all(isinstance(trace[field], str) and trace[field].strip() for field in trace) ): raise ValueError("supervisor evaluation behavior_trace is invalid") - return result - def _complete_supervision(self, run_id: str) -> None: + @staticmethod + def _validate_persona_journey_trace(trace: Any) -> None: + required = {"persona_goal", "current_action", "decision", "next_action", "evidence"} + if ( + not isinstance(trace, dict) + or set(trace) != required + or not all(isinstance(trace[field], str) and trace[field].strip() for field in required) + ): + raise ValueError("supervisor persona journey behavior_trace is invalid") + + @staticmethod + def _evaluation_request_for_iteration( + run: Run, iteration: int, candidate_id: str | None + ) -> dict[str, Any] | None: + """Return the explicit SDK evaluation request for this agent result. + + A normal iteration is evidence for improvement feedback, but is not by + itself a subject to score. Only ``RunnerContext.register_evaluation`` + can opt an agent result into score/approval evaluation. + """ + for step in reversed(run.step_results): + if step.get("loop_index") != iteration: + continue + if candidate_id is not None and step.get("candidate_id") != candidate_id: + continue + result = step.get("result") + request = result.get("evaluation_request") if isinstance(result, dict) else None + if not isinstance(request, dict): + continue + feedback = request.get("feedback") + if ( + request.get("subject") != "agent_change" + or not isinstance(feedback, str) + or not feedback.strip() + ): + continue + value = { + "subject": "agent_change", + "feedback": feedback.strip(), + "changed_files": request.get("changed_files", []), + "validation": request.get("validation", ""), + } + fingerprint = str(request.get("improvement_fingerprint") or "") + if fingerprint: + value["improvement_fingerprint"] = fingerprint + return value + return None + + def _complete_supervision(self, run_id: str, *, stage: str = "issue_assessment") -> None: """Ask the configured manager model and retain its validated JSON per Run. A missing profile is observable but never turns a successfully completed @@ -3714,6 +4326,26 @@ def _complete_supervision(self, run_id: str) -> None: ), None, ) + evaluation_request = self._evaluation_request_for_iteration(run, iteration, candidate_id) + if stage == "issue_assessment": + evaluation_request = None + elif stage != "agent_proposal_assessment": + raise ValueError(f"unsupported supervision stage: {stage}") + elif evaluation_request is None: + # No Agent feedback and no retained diff means there is no proposal + # to score. Issue assessment is already retained independently. + return + if evaluation_request and evaluation_request.get("improvement_fingerprint"): + fingerprint = evaluation_request["improvement_fingerprint"] + already_evaluated = any( + isinstance(record, dict) + and record.get("stage") == stage + and isinstance(record.get("evaluation_request"), dict) + and record["evaluation_request"].get("improvement_fingerprint") == fingerprint + for record in run.supervisor_results + ) + if already_evaluated: + return configured = next( (item for item in self.profiles() if item["profile_name"] == run.supervisor_profile_name), self.settings(), @@ -3785,6 +4417,53 @@ def supervisor_result(result: object) -> object: if cycle_evidence: supervisor_prompt += "\n\n# OpenOrbit cycle evidence\n" supervisor_prompt += json.dumps(cycle_evidence, ensure_ascii=False, default=str) + known_issues = self._known_unresolved_issues(run) if stage == "issue_assessment" else [] + if known_issues: + supervisor_prompt += "\n\n# Known unresolved issues\n" + supervisor_prompt += json.dumps(known_issues, ensure_ascii=False) + supervisor_prompt += ( + "\nThese are active problems already reported for this build. Compare every potential " + "finding by its underlying cause, user impact, and reproduction, not by title wording or " + "language. When it is the same unresolved issue, do not create a new improvement or " + "reported issue. If retaining this iteration's evidence against that existing issue is " + "necessary, return an item with its exact known_issue_id and evidence only; it will not " + "create a new Issue Management row. Create a new item only for a materially distinct cause, " + "impact, or reproduction.\n" + ) + supervisor_prompt += ( + "\n\n# Persona journey timeline required\n" + "Always include a top-level persona_journeys array. For every active or processed " + "persona supported by this iteration's evidence, emit one object with its exact persona_id " + "and behavior_trace: {persona_goal, current_action, decision, next_action, evidence}. " + "Write the trace as the supervisor's evidence-backed observation of that persona, not as " + "the runner procedure or hidden reasoning. Do not invent a journey for a persona without " + "evidence; return [] when there is none. persona_journeys is independent of top-level " + "evaluation: include it even when no iteration score is allowed.\n" + "The only permitted top-level keys are evaluation (when explicitly requested), " + "persona_journeys, improvements, and reported_issues." + ) + if stage == "issue_assessment": + supervisor_prompt += ( + "\n\n# Issue assessment required\n" + "Assess every reported_issues and improvements item independently. Each discovered item must include " + "evaluation: {score, approval, summary}; approval is approved, pending, or rejected. " + "This is the score/decision for the discovered issue, not for the iteration. " + "Do not include a top-level evaluation. An agent may only create a proposal for an issue " + "whose approval is not rejected." + ) + elif evaluation_request: + supervisor_prompt += "\n\n# Agent result to evaluate\n" + supervisor_prompt += json.dumps(evaluation_request, ensure_ascii=False, default=str) + supervisor_prompt += ( + "\n\nThe SDK explicitly registered this completed agent result for evaluation. " + "Include evaluation and score only this agent result; do not score the iteration itself." + ) + else: + supervisor_prompt += ( + "\n\nNo SDK agent result was registered for evaluation in this iteration. " + "Keep improvements and reported_issues as usual, but omit evaluation entirely. " + "Do not assign a score or approval to the iteration." + ) with self.tracer.start_as_current_span( "supervisor.evaluate", attributes={ @@ -3806,7 +4485,26 @@ def supervisor_result(result: object) -> object: response_text = provider.complete(settings, supervisor_prompt) span.set_attributes(_telemetry_text_metadata("gen_ai.response", response_text)) span.add_event("gen_ai.response.received") - result = self._validated_supervisor_result(response_text) + result = self._validated_supervisor_result( + response_text, require_evaluation=evaluation_request is not None + ) + known_issue_ids = {item["id"] for item in known_issues} + for proposal in [*result["improvements"], *result["reported_issues"]]: + known_issue_id = str(proposal.get("known_issue_id") or "").strip() + if known_issue_id and known_issue_id not in known_issue_ids: + # Never let a model-invented ID suppress a new Issue. + proposal.pop("known_issue_id", None) + if stage == "issue_assessment" and any( + not str(item.get("known_issue_id") or "").strip() + and not isinstance(item.get("evaluation"), dict) + for item in [*result["reported_issues"], *result["improvements"]] + ): + raise ValueError("every supervisor-discovered issue must include score and decision") + # Manager templates created before agent-result evaluation can + # still return an evaluation by default. A score is invalid + # without the SDK's explicit registration, so never retain it. + if evaluation_request is None: + result.pop("evaluation", None) evaluation = result.get("evaluation") if evaluation is not None: threshold = ( @@ -3835,9 +4533,11 @@ def supervisor_result(result: object) -> object: { "iteration": iteration, "candidate_id": candidate_id, + "stage": stage, "status": "completed", "prompt": supervisor_prompt, "response": result, + "evaluation_request": evaluation_request, "recorded_at": now().isoformat(), } ) diff --git a/backend/app/visual_runners.py b/backend/app/visual_runners.py new file mode 100644 index 0000000..8c82b1a --- /dev/null +++ b/backend/app/visual_runners.py @@ -0,0 +1,264 @@ +"""Schema validation and deterministic Python generation for visual runners.""" + +from __future__ import annotations + +import re +from collections import defaultdict +from collections.abc import Iterable +from typing import Any + +from orbit_sdk.visual import visual_nodes +from orbit_sdk.visual.bindings import validate as validate_bindings +from orbit_sdk.visual.bindings import validate_condition + +PHASES = ("before_all", "before_each", "execute", "verify", "after_each", "after_all") +NODE_KINDS = {item["kind"] for item in visual_nodes.catalog()} +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9_-]{0,63}$") +_PYTHON_IDENTIFIER = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_PHASE_INDEX = {phase: index for index, phase in enumerate(PHASES)} + + +def validate_blueprint(value: dict[str, Any]) -> dict[str, Any]: + """Validate a visual runner blueprint and return its normalized value.""" + if value.get("schema_version") != 1: + raise ValueError("visual runner blueprint requires schema_version 1") + nodes = value.get("nodes") + edges = value.get("edges", []) + if not isinstance(nodes, list) or not nodes: + raise ValueError("visual runner blueprint requires at least one node") + if not isinstance(edges, list): + raise ValueError("visual runner blueprint edges must be an array") + parameters = value.get("parameters", {}) + if not isinstance(parameters, dict) or not all( + isinstance(key, str) and isinstance(item, str) for key, item in parameters.items() + ): + raise ValueError("visual runner blueprint parameters must be a string object") + ids: set[str] = set() + normalized_nodes: list[dict[str, Any]] = [] + for node in nodes: + if not isinstance(node, dict): + raise ValueError("visual runner nodes must be objects") + node_id = str(node.get("id", "")) + if not _IDENTIFIER.fullmatch(node_id) or node_id in ids: + raise ValueError("visual runner node IDs must be unique lowercase identifiers") + ids.add(node_id) + kind, phase = str(node.get("kind", "")), str(node.get("phase", "")) + if kind not in NODE_KINDS or phase not in PHASES: + raise ValueError(f"visual runner node {node_id!r} has an unsupported kind or phase") + inputs = _ports(node.get("inputs", []), node_id, "input") + outputs = _ports(node.get("outputs", []), node_id, "output") + config = node.get("config") if isinstance(node.get("config"), dict) else {} + try: + validate_bindings(config) + validate_condition(node.get("when")) + except ValueError as error: + raise ValueError( + f"visual runner node {node_id!r} has invalid runtime binding: {error}" + ) from error + if kind != "custom_script": + try: + visual_nodes.validate(kind, config) + except ValueError as error: + raise ValueError(f"visual runner node {node_id!r} {error}") from error + position = node.get("position") if isinstance(node.get("position"), dict) else {} + normalized_nodes.append( + { + "id": node_id, + "kind": kind, + "title": str(node.get("title") or node_id.replace("_", " ").title()), + "description": str(node.get("description") or "") or None, + "phase": phase, + "inputs": inputs, + "outputs": outputs, + "config": config, + "when": node.get("when"), + "script": str(node.get("script") or ""), + "position": {"x": _number(position.get("x", 0)), "y": _number(position.get("y", 0))}, + } + ) + nodes_by_id = {node["id"]: node for node in normalized_nodes} + normalized_edges: list[dict[str, Any]] = [] + for edge in edges: + if not isinstance(edge, dict) or edge.get("source") not in ids or edge.get("target") not in ids: + raise ValueError("visual runner edges must connect existing nodes") + source, target = str(edge["source"]), str(edge["target"]) + kind = str(edge.get("kind", "data")) + if kind not in {"execution", "data", "condition", "loop"}: + raise ValueError("visual runner edge kind is invalid") + source_port = str(edge.get("source_port") or "") or None + target_port = str(edge.get("target_port") or "") or None + if kind == "data": + if bool(source_port) != bool(target_port): + raise ValueError("visual runner data edges must declare both ports or neither") + if source_port and ( + source_port not in nodes_by_id[source]["outputs"] + or target_port not in nodes_by_id[target]["inputs"] + ): + raise ValueError("visual runner data edges must connect declared output and input ports") + elif source_port or target_port: + raise ValueError("only visual runner data edges may use ports") + if kind == "loop": + if nodes_by_id[source]["phase"] != "after_each" or nodes_by_id[target]["phase"] not in { + "before_each", + "execute", + }: + raise ValueError("visual runner loops must connect after_each to before_each or execute") + elif _PHASE_INDEX[nodes_by_id[source]["phase"]] > _PHASE_INDEX[nodes_by_id[target]["phase"]]: + raise ValueError("visual runner edges must follow lifecycle order") + normalized_edges.append( + { + "source": source, + "target": target, + "kind": kind, + "source_port": source_port, + "target_port": target_port, + "label": str(edge.get("label") or "") or None, + } + ) + _ensure_acyclic(nodes_by_id, normalized_edges) + return { + "schema_version": 1, + "parameters": dict(parameters), + "nodes": normalized_nodes, + "edges": normalized_edges, + } + + +def _ports(value: object, node_id: str, label: str) -> list[str]: + if not isinstance(value, list) or not all(_IDENTIFIER.fullmatch(str(port)) for port in value): + raise ValueError(f"visual runner node {node_id!r} has invalid {label} ports") + ports = [str(port) for port in value] + if len(set(ports)) != len(ports): + raise ValueError(f"visual runner node {node_id!r} has duplicate {label} ports") + return ports + + +def _number(value: object) -> int | float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + return 0 + return value + + +def _ensure_acyclic(nodes: dict[str, dict[str, Any]], edges: Iterable[dict[str, Any]]) -> None: + """Reject cycles other than the explicit iteration loop edge.""" + adjacent = {node_id: [] for node_id in nodes} + for edge in edges: + if edge["kind"] != "loop": + adjacent[edge["source"]].append(edge["target"]) + visiting: set[str] = set() + visited: set[str] = set() + + def visit(node_id: str) -> None: + if node_id in visiting: + raise ValueError("visual runner edges must not contain a cycle") + if node_id not in visited: + visiting.add(node_id) + for target in adjacent[node_id]: + visit(target) + visiting.remove(node_id) + visited.add(node_id) + + for node_id in nodes: + visit(node_id) + + +def generate_source(blueprint: dict[str, Any]) -> str: + """Generate one explicit, executable runner source file from a blueprint.""" + blueprint = validate_blueprint(blueprint) + bindings_by_target: dict[str, dict[str, tuple[str, str]]] = defaultdict(dict) + for edge in blueprint["edges"]: + if edge["kind"] == "data" and edge["source_port"] and edge["target_port"]: + bindings_by_target[edge["target"]][edge["target_port"]] = ( + edge["source"], + edge["source_port"], + ) + by_phase: dict[str, list[dict[str, Any]]] = defaultdict(list) + for node in blueprint["nodes"]: + by_phase[node["phase"]].append({**node, "_bindings": bindings_by_target[node["id"]]}) + lines = [ + '"""Generated by OpenOrbit Visual Runner Editor. Do not edit by hand."""', + "", + "from orbit_sdk import graph, runner", + "", + ] + for edge in blueprint["edges"]: + arguments = [repr(edge["source"]), repr(edge["target"])] + if edge["kind"] != "execution": + arguments.append(f"kind={edge['kind']!r}") + if edge["label"]: + arguments.append(f"label={edge['label']!r}") + if edge["source_port"]: + arguments.append(f"source_port={edge['source_port']!r}") + if edge["target_port"]: + arguments.append(f"target_port={edge['target_port']!r}") + lines.append(f"graph.connect({', '.join(arguments)})") + lines.append("") + for phase in PHASES: + ordered_nodes = _ordered_nodes(by_phase[phase], blueprint["edges"]) + for node in ordered_nodes: + lines.extend(_node_source(node)) + if by_phase[phase]: + lines.append(f"@runner.phase({phase!r})") + lines.append(f"def phase_{phase}(ctx):") + lines.extend(f" {_function_name(node['id'])}(ctx)" for node in ordered_nodes) + lines.append("") + lines.extend(["if __name__ == '__main__':", " runner.main()", ""]) + return "\n".join(lines) + + +def _node_source(node: dict[str, Any]) -> list[str]: + # Generated runners are Python source, not JSON. ``json.dumps`` emits + # JSON literals such as ``true`` and ``null``, which are invalid Python. + config = repr(node["config"]) + outputs = repr(node["outputs"]) + lines = [ + f"@graph.step({node['id']!r}, title={node['title']!r}, phase={node['phase']!r}, inputs={node['inputs']!r}, outputs={outputs}, description={node['description']!r})", + f"@runner.phase({node['phase']!r}, step_id={node['id']!r})", + f"def {_function_name(node['id'])}(ctx):", + f" config = {config}", + f" outputs = ctx.visual_node_inputs({node['id']!r}, {_bindings(node)!r})", + ] + if node["when"] is not None: + lines.append(f" if not ctx.visual_should_run({node['when']!r}, inputs=outputs):") + lines.append(" return") + if node["kind"] == "custom_script": + script = node["script"] or "# Set values on outputs, then return.\npass" + lines.extend(f" {line}" if line else "" for line in script.splitlines()) + else: + lines.append( + f" outputs.update(ctx.run_visual_node({node['kind']!r}, node_id={node['id']!r}, config=config, inputs=outputs))" + ) + lines.extend([f" ctx.publish_visual_node_outputs({node['id']!r}, outputs)", ""]) + return lines + + +def _bindings(node: dict[str, Any]) -> dict[str, tuple[str, str]]: + # Set by generate_source immediately before each node is rendered. + return node.get("_bindings", {}) + + +def _function_name(node_id: str) -> str: + """Return a deterministic, safe Python function name for a graph node ID.""" + if _PYTHON_IDENTIFIER.fullmatch(node_id): + return node_id + return "node_" + re.sub(r"[^a-z0-9_]", "_", node_id) + + +def _ordered_nodes(nodes: list[dict[str, Any]], edges: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Order same-phase nodes by graph dependencies while retaining canvas order.""" + node_ids = {node["id"] for node in nodes} + predecessors = {node_id: set() for node_id in node_ids} + for edge in edges: + if edge["kind"] != "loop" and edge["source"] in node_ids and edge["target"] in node_ids: + predecessors[edge["target"]].add(edge["source"]) + ordered: list[dict[str, Any]] = [] + remaining = list(nodes) + while remaining: + ready = next((node for node in remaining if not predecessors[node["id"]]), None) + if ready is None: # validate_blueprint has already provided the user-facing error. + return nodes + remaining.remove(ready) + ordered.append(ready) + for values in predecessors.values(): + values.discard(ready["id"]) + return ordered diff --git a/backend/coding_agents.py b/backend/coding_agents.py new file mode 100644 index 0000000..6ffebe4 --- /dev/null +++ b/backend/coding_agents.py @@ -0,0 +1,59 @@ +"""Shared local coding-agent command and prompt conventions.""" + +from __future__ import annotations + +import os +import platform +import shlex +from datetime import datetime +from pathlib import Path + +CODING_AGENT_PROVIDERS = ("kiro", "claude-code", "codex") + + +def build_coding_agent_command(provider: str, *, options: str = "", prompt: str) -> list[str]: + """Build the established CLI invocation for a locally installed coding agent.""" + commands = { + # ``--approve-for-me`` already selects Codex's workspace-write sandbox. + "codex": (["codex", "exec", "--approve-for-me"], []), + "claude-code": (["claude"], ["-p"]), + "kiro": (["kiro-cli", "chat"], []), + } + if provider not in commands: + raise ValueError("AI agent provider must be codex, claude-code, or kiro") + normalized_prompt = prompt.strip() + if not normalized_prompt: + raise ValueError("AI agent prompt must not be empty") + try: + extra_arguments = shlex.split(options) + except ValueError as error: + raise ValueError("AI agent options must be a valid command argument string") from error + executable, prompt_arguments = commands[provider] + return [*executable, *extra_arguments, *prompt_arguments, normalized_prompt] + + +def execution_environment_context(working_directory: str | Path | None = None) -> str: + """Describe the host context in which a coding task will run.""" + shell_path = os.environ.get("COMSPEC") if os.name == "nt" else os.environ.get("SHELL") + shell_path = shell_path or ("cmd.exe" if os.name == "nt" else "/bin/sh") + shell_name = os.path.basename(shell_path) or shell_path + directory = Path(working_directory).expanduser().resolve() if working_directory else Path.cwd() + current_time = datetime.now().astimezone().isoformat(timespec="seconds") + return ( + "Execution environment:\n" + f"- OS: {platform.system()} {platform.release()}\n" + f"- Default shell: {shell_name} ({shell_path})\n" + f"- Current working directory: {directory}\n" + f"- Current local time: {current_time}\n" + ) + + +def build_coding_agent_prompt(task: str, working_directory: str | Path) -> str: + """Wrap an Assistant task in the shared system context understood by all CLIs.""" + return ( + "You are the coding agent used by Orbit Assistant.\n" + "Work only in the current working directory. Inspect and validate changes where appropriate.\n" + + execution_environment_context(working_directory) + + "\nUser task:\n" + + task.strip() + ) diff --git a/backend/orbit_sdk/__init__.py b/backend/orbit_sdk/__init__.py new file mode 100644 index 0000000..d967bc2 --- /dev/null +++ b/backend/orbit_sdk/__init__.py @@ -0,0 +1,50 @@ +"""Public, dependency-free API for OpenOrbit runner assets. + +The package keeps the historical ``from orbit_sdk import graph, runner`` API +stable while separating runtime primitives, declarative decorators, reusable +helpers, and visual-node definitions into dedicated modules. +""" + +import sys +from types import ModuleType + +from .core import context as _context +from .core.context import ( + ORBIT_APP_DATA, + RunnerContext, +) +from .core.lifecycle import PHASE_ALIASES, canonical_phase +from .decorators import Graph, GraphEdge, GraphNode, Runner, graph, runner + + +class _OrbitSdkModule(ModuleType): + """Keep mutable legacy module settings synchronized with ``core``. + + Existing runner tests and integrations sometimes set ``ORBIT_APP_DATA`` on + the public module. The runtime now lives in ``core.context``, so forward + that small compatibility surface rather than leaking the old layout. + """ + + def __getattr__(self, name: str): + return getattr(_context, name) + + def __setattr__(self, name: str, value: object) -> None: + if name == "ORBIT_APP_DATA": + setattr(_context, name, value) + super().__setattr__(name, value) + + +sys.modules[__name__].__class__ = _OrbitSdkModule + +__all__ = [ + "ORBIT_APP_DATA", + "PHASE_ALIASES", + "Graph", + "GraphEdge", + "GraphNode", + "Runner", + "RunnerContext", + "canonical_phase", + "graph", + "runner", +] diff --git a/backend/orbit_sdk/core/__init__.py b/backend/orbit_sdk/core/__init__.py new file mode 100644 index 0000000..bd2a1b9 --- /dev/null +++ b/backend/orbit_sdk/core/__init__.py @@ -0,0 +1,5 @@ +"""Runtime primitives used by Orbit runner code.""" + +from .context import RunnerContext + +__all__ = ["RunnerContext"] diff --git a/backend/orbit_sdk.py b/backend/orbit_sdk/core/context.py similarity index 79% rename from backend/orbit_sdk.py rename to backend/orbit_sdk/core/context.py index 51cff8d..710cc29 100644 --- a/backend/orbit_sdk.py +++ b/backend/orbit_sdk/core/context.py @@ -6,7 +6,6 @@ from __future__ import annotations -import argparse import base64 import hashlib import json @@ -15,162 +14,20 @@ import subprocess import tempfile import threading +import uuid from contextlib import contextmanager from dataclasses import dataclass, field from datetime import UTC, datetime -from functools import wraps from pathlib import Path -from typing import Any, Callable, Iterator, Literal +from typing import Iterator, Literal -PROJECT_ROOT = Path(os.environ.get("ORBIT_TARGET_REPOSITORY", Path.cwd())).resolve() -ORBIT_APP_DATA = Path(os.environ.get("ORBIT_APP_DATA", Path.home() / ".local" / "share" / "orbit")).resolve() - -# Accept pre-1.0 lifecycle names in existing runner files while emitting the -# canonical names everywhere else. -PHASE_ALIASES = { - "init": "before_all", - "setup": "before_each", - "run": "execute", - "eval": "verify", - "teardown": "after_each", - "finalize": "after_all", -} - - -def canonical_phase(name: str) -> str: - """Return the canonical lifecycle key for a current or legacy phase.""" - return PHASE_ALIASES.get(name, name) - - -@dataclass(frozen=True) -class GraphNode: - """A declarative visual-workflow node attached to a runner function.""" - - id: str - title: str - phase: str | None - inputs: tuple[str, ...] = () - outputs: tuple[str, ...] = () - description: str | None = None - - -@dataclass(frozen=True) -class GraphEdge: - """A directed relationship between visual-workflow nodes.""" - - source: str - target: str - kind: Literal["execution", "data", "condition", "loop", "error"] = "execution" - label: str | None = None - source_port: str | None = None - target_port: str | None = None - - -class Graph: - """Declare a runner's visual workflow without changing its execution. - - ``graph`` is intentionally declarative: decorators only retain metadata. - Orbit may inspect :meth:`definition` before a run, then overlay runtime - status onto the same node IDs after a run. - """ - - def __init__(self) -> None: - self._nodes: dict[str, GraphNode] = {} - self._edges: list[GraphEdge] = [] - - def step( - self, - id: str | None = None, - *, - title: str | None = None, - phase: str | None = None, - inputs: tuple[str, ...] | list[str] = (), - outputs: tuple[str, ...] | list[str] = (), - description: str | None = None, - ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - """Annotate one function as a visual workflow node. - - ``phase`` is an optional display group, not a hard-coded lifecycle - enum, so a runner can evolve its lifecycle without changing this API. - """ - - def register(handler: Callable[..., Any]) -> Callable[..., Any]: - node_id = id or handler.__name__ - if not node_id or node_id in self._nodes: - raise ValueError(f"graph node ID must be unique: {node_id!r}") - node_phase = canonical_phase(phase or getattr(handler, "__orbit_phase__", "")) or None - node = GraphNode( - id=node_id, - title=title or handler.__name__.replace("_", " ").title(), - phase=node_phase, - inputs=tuple(inputs), - outputs=tuple(outputs), - description=description, - ) - self._nodes[node_id] = node - setattr(handler, "__orbit_graph_node__", node) - - @wraps(handler) - def instrumented(*args: Any, **kwargs: Any) -> Any: - context = next( - (value for value in (*args, *kwargs.values()) if isinstance(value, RunnerContext)), - None, - ) - if context is None: - return handler(*args, **kwargs) - with context.function(node_id): - return handler(*args, **kwargs) - - setattr(instrumented, "__orbit_graph_node__", node) - setattr(handler, "__orbit_graph_wrapper__", instrumented) - return instrumented - - return register - - def connect( - self, - source: str, - target: str, - *, - kind: Literal["execution", "data", "condition", "loop", "error"] = "execution", - label: str | None = None, - source_port: str | None = None, - target_port: str | None = None, - ) -> GraphEdge: - """Declare a typed arrow; ``loop`` and ``condition`` model control flow.""" - edge = GraphEdge(source, target, kind, label, source_port, target_port) - self._edges.append(edge) - return edge - - def definition(self) -> dict[str, object]: - """Return JSON-safe graph data for a visual client or source inspector.""" - return { - "nodes": [ - { - "id": node.id, - "title": node.title, - "phase": node.phase, - "inputs": list(node.inputs), - "outputs": list(node.outputs), - "description": node.description, - } - for node in self._nodes.values() - ], - "edges": [ - { - "source": edge.source, - "target": edge.target, - "kind": edge.kind, - "label": edge.label, - "source_port": edge.source_port, - "target_port": edge.target_port, - } - for edge in self._edges - ], - } +from coding_agents import build_coding_agent_command +from orbit_sdk.core.lifecycle import canonical_phase +from orbit_sdk.helpers import command_from_environment -graph = Graph() +PROJECT_ROOT = Path(os.environ.get("ORBIT_TARGET_REPOSITORY", Path.cwd())).resolve() +ORBIT_APP_DATA = Path(os.environ.get("ORBIT_APP_DATA", Path.home() / ".local" / "share" / "orbit")).resolve() def ORBIT_PROJECT_PATH(*parts: str) -> Path: @@ -462,6 +319,85 @@ def save_state(self, name: str, value: object, *, scope: str = "runner") -> dict "size": len(payload), } + def visual_node_inputs( + self, node_id: str, bindings: dict[str, tuple[str, str]] | None = None + ) -> dict[str, object]: + """Return declared upstream values for one generated visual-runner node. + + Args: + node_id: The generated graph-node identifier requesting inputs. + bindings: Input names mapped to ``(source_node_id, output_port)``. + Generated visual runners provide this mapping from their data + edges, so unrelated node output is never exposed implicitly. + + Returns: + A dictionary keyed by the node's declared input ports. Missing + upstream values are omitted, allowing a custom script to apply a + default explicitly. + """ + document = self.load_state("orbit-visual-node-outputs", {}, scope="runner") + if not isinstance(document, dict) or document.get("iteration") != self.loop_index: + return {} + values = document.get("outputs", {}) + if not isinstance(values, dict): + return {} + if bindings is None: + return {} + inputs: dict[str, object] = {} + for input_name, binding in bindings.items(): + if not isinstance(input_name, str) or not isinstance(binding, tuple) or len(binding) != 2: + raise ValueError("visual node input bindings must map names to source node ports") + source_node, source_port = binding + source_outputs = values.get(source_node) + if isinstance(source_outputs, dict) and source_port in source_outputs: + inputs[input_name] = source_outputs[source_port] + return inputs + + def publish_visual_node_outputs(self, node_id: str, values: dict[str, object]) -> None: + """Persist JSON-safe outputs from one generated visual-runner node.""" + if not node_id.strip(): + raise ValueError("visual node ID must not be empty") + if not isinstance(values, dict): + raise ValueError("visual node outputs must be a JSON object") + current = self.load_state("orbit-visual-node-outputs", {}, scope="runner") + if not isinstance(current, dict) or current.get("iteration") != self.loop_index: + current = {"iteration": self.loop_index, "outputs": {}} + outputs = current.setdefault("outputs", {}) + if not isinstance(outputs, dict): + outputs = current["outputs"] = {} + outputs[node_id] = values + self.save_state("orbit-visual-node-outputs", current, scope="runner") + + def run_visual_node( + self, + kind: str, + *, + node_id: str, + config: dict[str, object], + inputs: dict[str, object], + ) -> dict[str, object]: + """Dispatch one SDK-owned visual operation through its registry. + + Custom Script nodes intentionally remain generated Python. All curated + operations use this dispatcher so their catalog metadata, validation, + and runtime implementation have one SDK owner. + """ + if not node_id.strip(): + raise ValueError("visual node ID must not be empty") + from orbit_sdk.visual import visual_nodes + from orbit_sdk.visual.bindings import resolve + + resolved = resolve(self, config, inputs) + if not isinstance(resolved, dict): # Defensive boundary for SDK node handlers. + raise ValueError("visual node configuration must resolve to an object") + return visual_nodes.execute(kind, self, config=resolved, inputs=inputs) + + def visual_should_run(self, condition: object, *, inputs: dict[str, object]) -> bool: + """Evaluate a blueprint's safe, declarative node condition.""" + from orbit_sdk.visual.bindings import evaluate + + return evaluate(self, condition, inputs) + def materialize_assets(self, name: str, files: dict[str, str | bytes]) -> dict[str, object]: """Atomically materialize runner-owned files outside the target repository. @@ -1459,6 +1395,21 @@ def test_cases(self) -> list[dict[str, object]]: """ return list(self.resources.get("test_cases", [])) + def require_test_case_ids( + self, required_ids: set[str] | list[str] | tuple[str, ...], *, label: str = "required test case" + ) -> None: + """Require that the declared fixed cases include every requested ID. + + Args: + required_ids: Stable test-case IDs that must be selected. + label: Singular description used in an actionable validation error. + """ + required = {str(case_id) for case_id in required_ids if str(case_id).strip()} + selected = {str(case.get("id", "")) for case in self.test_cases} + missing = sorted(required - selected) + if missing: + raise ValueError(f"Missing {label}: {', '.join(missing)}") + def resource(self, name: str, default: object = None) -> object: """Read a named value from Orbit's immutable resource snapshot. @@ -1514,6 +1465,15 @@ def complete_model(self, prompt: str) -> dict[str, str]: "response": response, } + def complete_model_json(self, prompt: str, *, description: str = "model response") -> dict[str, object]: + """Run one target-AI turn and require a JSON object response. + + Use this when a runner's prompt explicitly contracts the model to emit + structured data. The raw text remains available through + :meth:`complete_model` for free-form model tasks. + """ + return self.parse_json_object(self.complete_model(prompt)["response"], description=description) + @property def previous_supervisor_feedback(self) -> dict[str, object]: """Return the latest completed supervisor response for this run. @@ -1566,6 +1526,56 @@ def previous_supervisor_feedback(self) -> dict[str, object]: return feedback return {} + @property + def current_issue_assessment(self) -> dict[str, object]: + """Return this iteration's first supervisor-assessed actionable issue. + + The post-supervision agent phase uses this as its sole objective. A + rejected issue is deliberately returned too, so the runner can record + an explicit skip rather than treating absence as approval. + """ + run_id = self.environment.get("ORBIT_RUN_ID", "").strip() + if not run_id: + return {} + try: + values = json.loads( + (self.app_data / "data" / "runs" / f"{run_id}.json").read_text(encoding="utf-8") + ) + except (OSError, json.JSONDecodeError): + return {} + for record in reversed(values.get("supervisor_results", [])): + if not isinstance(record, dict) or record.get("stage") != "issue_assessment": + continue + if int(record.get("iteration", 0)) != self.loop_index: + continue + response = record.get("response") + if not isinstance(response, dict): + continue + issues = response.get("reported_issues") + improvements = response.get("improvements") + # Manager templates commonly express a discovered product problem + # as an improvement proposal. It is still an Issue for this + # lifecycle once the supervisor has scored and decided it. Prefer + # that user-facing proposal so the agent changes the same Issue + # shown in Issue Management rather than creating a parallel row. + candidates = [ + {**improvement, "_orbit_issue_id": f"{run_id}:{self.loop_index}:{index}"} + for index, improvement in enumerate(improvements) + if isinstance(improvements, list) and isinstance(improvement, dict) + ] + candidates.extend( + {**issue, "_orbit_issue_id": f"{run_id}:{self.loop_index}:issue:{index}"} + for index, issue in enumerate(issues) + if isinstance(issues, list) and isinstance(issue, dict) + ) + if candidates: + for issue in candidates: + assessment = issue.get("evaluation") + if not isinstance(assessment, dict) or assessment.get("approval") != "rejected": + return issue + return candidates[0] if candidates else {} + return {} + def log(self, message: str) -> None: """Write an Orbit runner-progress message to the workflow log. @@ -1648,6 +1658,155 @@ def emit_result(self, values: dict[str, object]) -> None: """ print("__ORBIT_RESULT__" + json.dumps(values, ensure_ascii=False), flush=True) + def register_evaluation( + self, + feedback: str, + *, + subject: str = "agent_change", + changed_files: list[str] | None = None, + validation: str = "", + improvement_fingerprint: str = "", + ) -> dict[str, object]: + """Register an AI-agent result that should receive a supervisor evaluation. + + Register only after the agent has completed a meaningful change or + produced actionable feedback. Merely running an iteration does not + create an evaluation, score, or approval decision. + + Args: + feedback: Concise account of the agent's completed work and its evidence. + subject: Stable evaluation subject. ``agent_change`` is the supported + default for autonomous SDK agents. + changed_files: Repository-relative files changed by the agent, if any. + validation: Verification performed by the agent after its work. + + Returns: + The structured evaluation request emitted for the current step. + """ + normalized_feedback = feedback.strip() + if not normalized_feedback: + raise ValueError("evaluation feedback must not be empty") + if subject != "agent_change": + raise ValueError("evaluation subject must be agent_change") + normalized_files = [ + str(path).strip() for path in (changed_files or []) if isinstance(path, str) and path.strip() + ] + request = { + "subject": subject, + "feedback": normalized_feedback, + "changed_files": list(dict.fromkeys(normalized_files)), + "validation": validation.strip(), + "improvement_fingerprint": improvement_fingerprint.strip(), + } + self.emit_result({"evaluation_request": request}) + return request + + def run_ai_agent( + self, + prompt: str, + *, + provider: str, + options: str = "", + timeout: int = 1_800, + ) -> dict[str, object]: + """Run a locally installed coding agent and retain its completion evidence. + + The agent must print ``ORBIT_AGENT_FEEDBACK: `` as its final + feedback line to opt its work into evaluation. This keeps a silent or + no-op agent run from creating a score for the iteration. + """ + normalized_prompt = prompt.strip() + if not normalized_prompt: + raise ValueError("AI agent prompt must not be empty") + # Agent edits are proposals, never writes to the evaluated repository. + # A proposal branch/worktree gives the coding agent a real Git checkout while + # preserving the operator's working tree and its uncommitted changes. + self._git_repository_root() + run_id = str(self.environment.get("ORBIT_RUN_ID") or "manual") + worktree_id = f"{run_id}-{self.loop_index}-{uuid.uuid4().hex[:8]}" + worktree = ORBIT_APP_DATA / "agent-worktrees" / worktree_id + worktree.parent.mkdir(parents=True, exist_ok=True) + base_revision = self.git_head() + if not base_revision: + raise ValueError("AI agent proposals require a checked-out Git revision") + branch = f"orbit/agent-proposal/{run_id}/{self.loop_index}-{uuid.uuid4().hex[:8]}" + self.exec( + ["git", "worktree", "add", "-b", branch, str(worktree), base_revision], + cwd=self.project_root, + target_log_source="ai-agent", + ) + output = "" + changed_files: list[str] = [] + patch = "" + keep_worktree = False + try: + output = self.exec( + build_coding_agent_command(provider, options=options, prompt=normalized_prompt), + cwd=worktree, + timeout=timeout, + target_log_source="ai-agent", + ) + # Include newly-created files in the review patch without creating + # a commit. Intent-to-add is local to this disposable proposal + # worktree and is finalized only after Issue approval. + self.exec(["git", "add", "--intent-to-add", "--all"], cwd=worktree) + changed_files = [ + line + for line in self.exec(["git", "diff", "--name-only", "--"], cwd=worktree).splitlines() + if line + ] + patch = self.exec(["git", "diff", "--binary", "--"], cwd=worktree) + feedback = next( + ( + line.removeprefix("ORBIT_AGENT_FEEDBACK:").strip() + for line in reversed(output.splitlines()) + if line.startswith("ORBIT_AGENT_FEEDBACK:") + and line.removeprefix("ORBIT_AGENT_FEEDBACK:").strip() + ), + "", + ) + keep_worktree = bool(feedback and patch) + finally: + if not keep_worktree: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=self.project_root, + env=self.environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + subprocess.run( + ["git", "branch", "-D", branch], + cwd=self.project_root, + env=self.environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + artifact = ( + self.write_artifact(f"agent-proposals/{worktree_id}.patch", patch, content_type="text/x-diff") + if patch + else None + ) + result = { + "provider": provider, + "feedback": feedback, + "changed_files": changed_files, + "output": output[-12_000:], + "proposal": { + "worktree_id": worktree_id, + "base_revision": base_revision, + "branch": branch if keep_worktree else "", + "worktree_path": str(worktree) if keep_worktree else "", + "diff": patch, + "fingerprint": _sha256(patch.encode("utf-8")) if patch else "", + "diff_artifact": artifact, + }, + } + self.emit_result({"agent_run": result}) + return result + def playwright_journey(self, cases: list[dict[str, object]] | None = None) -> dict[str, object]: """Run bounded, read-only Playwright page checks for the supplied cases. @@ -1683,7 +1842,7 @@ def playwright_journey(self, cases: list[dict[str, object]] | None = None) -> di artifacts.mkdir(parents=True, exist_ok=True) module = self.environment.get("ORBIT_PLAYWRIGHT_MODULE", "") if not module: - module = str(Path(__file__).resolve().parents[1] / "frontend" / "node_modules" / "playwright") + module = str(Path(__file__).resolve().parents[3] / "frontend" / "node_modules" / "playwright") payload = { "baseUrl": base_url, "cases": selected_cases, @@ -1736,6 +1895,7 @@ def exec( input: str | None = None, target_log_source: str | None = None, target_log_exclude_prefixes: tuple[str, ...] = (), + merge_stderr: bool = True, ) -> str: """Run one bounded child command and return its captured output. @@ -1754,9 +1914,13 @@ def exec( the target-log stream under this source name. target_log_exclude_prefixes: Output prefixes retained for the caller but excluded from target logs, for structured child results. + merge_stderr: When ``True`` (the default), include standard error in + the returned output. Set to ``False`` when a child reserves + standard output for a machine-readable response; standard error + is still printed and forwarded to target logs. Returns: - Combined standard output and standard error from the child. + Standard output, plus standard error when ``merge_stderr`` is true. Raises: subprocess.TimeoutExpired: If the child exceeds ``timeout``. @@ -1770,7 +1934,7 @@ def exec( text=True, stdin=subprocess.PIPE if input is not None else None, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE, ) lines: list[str] = [] @@ -1779,16 +1943,29 @@ def exec( process.stdin.write(input) process.stdin.close() - def forward_output() -> None: - assert process.stdout is not None - for line in process.stdout: - lines.append(line) + def forward_output(stream, *, retain: bool) -> None: + for line in stream: + if retain: + lines.append(line) print(line, end="", flush=True) if target_log_source and line.strip() and not line.startswith(target_log_exclude_prefixes): self.target_log(line.strip(), source=target_log_source) - reader = threading.Thread(target=forward_output, daemon=True) - reader.start() + assert process.stdout is not None + readers = [ + threading.Thread( + target=forward_output, args=(process.stdout,), kwargs={"retain": True}, daemon=True + ) + ] + if not merge_stderr: + assert process.stderr is not None + readers.append( + threading.Thread( + target=forward_output, args=(process.stderr,), kwargs={"retain": False}, daemon=True + ) + ) + for reader in readers: + reader.start() try: process.wait(timeout=timeout) except subprocess.TimeoutExpired: @@ -1796,7 +1973,8 @@ def forward_output() -> None: process.wait() raise finally: - reader.join() + for reader in readers: + reader.join() output = "".join(lines) if process.returncode: self.log(f"exec failed with exit code {process.returncode}: {' '.join(command)}") @@ -1804,72 +1982,81 @@ def forward_output() -> None: self.log(f"exec completed: {' '.join(command)}") return output + def command_from_env(self, command_env: str) -> list[str]: + """Parse one externally configured command from the runner environment. -class Runner: - """Register lifecycle handlers and dispatch the phase requested by Orbit.""" - - def __init__(self) -> None: - self._handlers: dict[str, Callable[[RunnerContext], None]] = {} - - def phase( - self, name: str - ) -> Callable[[Callable[[RunnerContext], None]], Callable[[RunnerContext], None]]: - """Register a function as a handler for one runner lifecycle phase. - - Args: - name: Lifecycle phase name, normally one of ``before_all``, - ``before_each``, ``execute``, ``verify``, ``after_each``, or - ``after_all``. Legacy names are accepted for compatibility. - - Returns: - A decorator that leaves the registered handler unchanged. + The value may be a shell-style command string or a JSON array of + strings. The executable must be present; empty arguments remain valid + because some tools use them intentionally. """ + return command_from_environment(self.environment, command_env) - def register(handler: Callable[[RunnerContext], None]) -> Callable[[RunnerContext], None]: - phase = canonical_phase(name) - self._handlers[phase] = handler - setattr(handler, "__orbit_phase__", phase) - return handler - - return register - - def main(self) -> None: - """Dispatch the phase passed by Orbit and retain any commit-range evidence. - - Place ``runner.main()`` behind an ``if __name__ == "__main__"`` guard - in every runner asset. Orbit supplies the ``--phase`` argument and - process environment; callers should not invoke this method directly. + def run_command_action( + self, + *, + command_env: str, + action: str, + timeout: int | None = None, + env: dict[str, str] | None = None, + log_source: str | None = None, + log_exclude_prefixes: tuple[str, ...] = (), + stdout_only: bool = False, + ) -> str: + """Run one action of an externally configured command. - Returns: - ``None``. The process exits after dispatching the requested phase. + Args: + command_env: Environment variable holding the command configuration. + action: Final argument passed to the external command. + timeout: Maximum duration in seconds. + env: Additional environment values for the child command. + log_source: Optional target-log source for child output. + log_exclude_prefixes: Child-output prefixes excluded from target logs. + stdout_only: Return only stdout, preserving stderr for logs. Use for + commands whose stdout is a structured response. """ - parser = argparse.ArgumentParser(description="Orbit runner phase") - command = parser.add_mutually_exclusive_group(required=True) - command.add_argument("--phase") - command.add_argument("--graph", action="store_true") - args = parser.parse_args() - if args.graph: - print(json.dumps(graph.definition(), ensure_ascii=False)) - return - phase = canonical_phase(args.phase) - handler = self._handlers.get(phase) - if handler is None: - raise SystemExit(f"runner does not define phase: {args.phase}") - context = RunnerContext( - phase, - Path(os.environ["ORBIT_TARGET_REPOSITORY"]), - os.environ.get("ORBIT_EXECUTION_MODE", "run"), - int(os.environ.get("ORBIT_LOOP_INDEX", "1")), + return self.exec( + [*self.command_from_env(command_env), action], + cwd=self.project_root, + timeout=timeout, + env=env, + target_log_source=log_source, + target_log_exclude_prefixes=log_exclude_prefixes, + merge_stderr=not stdout_only, ) - handler = getattr(handler, "__orbit_graph_wrapper__", handler) - before = context.git_head() + + def parse_json_object(self, output: str, *, description: str = "command output") -> dict[str, object]: + """Parse and require exactly one JSON object from command output.""" try: - handler(context) - finally: - try: - context.record_commit_change(before) - except (OSError, subprocess.CalledProcessError) as error: - context.log(f"Could not retain commit-change evidence: {error}") + result = json.loads(output) + except json.JSONDecodeError as error: + raise RuntimeError(f"{description} did not return JSON") from error + if not isinstance(result, dict): + raise RuntimeError(f"{description} must return a JSON object") + return result + def run_json_action( + self, + *, + command_env: str, + action: str, + input_env: str, + input_data: dict[str, object], + timeout: int | None = None, + log_source: str | None = None, + ) -> dict[str, object]: + """Run one external action that returns a JSON object on standard output.""" + payload = json.dumps({**input_data, "action": action}, ensure_ascii=False) + output = self.run_command_action( + command_env=command_env, + action=action, + timeout=timeout, + env={input_env: payload}, + log_source=log_source, + stdout_only=True, + ) + return self.parse_json_object(output, description=f"Action {action!r}") -runner = Runner() + def require_test_cases(self, *, label: str = "fixed test case") -> None: + """Require at least one declared fixed test case for a runner contract.""" + if not self.test_cases: + raise ValueError(f"Select at least one {label}") diff --git a/backend/orbit_sdk/core/lifecycle.py b/backend/orbit_sdk/core/lifecycle.py new file mode 100644 index 0000000..07fdc7a --- /dev/null +++ b/backend/orbit_sdk/core/lifecycle.py @@ -0,0 +1,16 @@ +"""Canonical runner lifecycle names and legacy aliases.""" + +PHASE_ALIASES = { + "init": "before_all", + "setup": "before_each", + "run": "execute", + "eval": "verify", + "post_supervision": "after_supervision", + "teardown": "after_each", + "finalize": "after_all", +} + + +def canonical_phase(name: str) -> str: + """Return the canonical lifecycle key for a current or legacy phase.""" + return PHASE_ALIASES.get(name, name) diff --git a/backend/orbit_sdk/decorators/__init__.py b/backend/orbit_sdk/decorators/__init__.py new file mode 100644 index 0000000..4346679 --- /dev/null +++ b/backend/orbit_sdk/decorators/__init__.py @@ -0,0 +1,10 @@ +"""Declarative runner and graph decorators. + +The public ``graph`` and ``runner`` instances remain exported from the package +root for backwards compatibility. +""" + +from .graph import Graph, GraphEdge, GraphNode, graph +from .runner import Runner, runner + +__all__ = ["Graph", "GraphEdge", "GraphNode", "Runner", "graph", "runner"] diff --git a/backend/orbit_sdk/decorators/graph.py b/backend/orbit_sdk/decorators/graph.py new file mode 100644 index 0000000..05b276f --- /dev/null +++ b/backend/orbit_sdk/decorators/graph.py @@ -0,0 +1,144 @@ +"""Declarative graph annotations for Orbit runner functions. + +The graph layer intentionally owns no runner runtime state. It only records +metadata and wraps a function when its first argument behaves like a runner +context (that is, it provides ``function(id)``). +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from functools import wraps +from typing import Any, Literal + +from ..core.lifecycle import canonical_phase + + +@dataclass(frozen=True) +class GraphNode: + """A declarative visual-workflow node attached to a runner function.""" + + id: str + title: str + phase: str | None + inputs: tuple[str, ...] = () + outputs: tuple[str, ...] = () + description: str | None = None + after_supervision: bool = False + + +@dataclass(frozen=True) +class GraphEdge: + """A directed relationship between visual-workflow nodes.""" + + source: str + target: str + kind: Literal["execution", "data", "condition", "loop", "error"] = "execution" + label: str | None = None + source_port: str | None = None + target_port: str | None = None + + +class Graph: + """Declare a runner's visual workflow without changing its execution.""" + + def __init__(self) -> None: + self._nodes: dict[str, GraphNode] = {} + self._edges: list[GraphEdge] = [] + + def step( + self, + id: str | None = None, + *, + title: str | None = None, + phase: str | None = None, + inputs: tuple[str, ...] | list[str] = (), + outputs: tuple[str, ...] | list[str] = (), + description: str | None = None, + after_supervision: bool = False, + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Annotate one function as a visual workflow node.""" + + def register(handler: Callable[..., Any]) -> Callable[..., Any]: + node_id = id or handler.__name__ + if not node_id or node_id in self._nodes: + raise ValueError(f"graph node ID must be unique: {node_id!r}") + node = GraphNode( + id=node_id, + title=title or handler.__name__.replace("_", " ").title(), + phase=canonical_phase(phase or getattr(handler, "__orbit_phase__", "")) or None, + inputs=tuple(inputs), + outputs=tuple(outputs), + description=description, + after_supervision=after_supervision, + ) + self._nodes[node_id] = node + setattr(handler, "__orbit_graph_node__", node) + + @wraps(handler) + def instrumented(*args: Any, **kwargs: Any) -> Any: + context = next( + ( + value + for value in (*args, *kwargs.values()) + if callable(getattr(value, "function", None)) + ), + None, + ) + if context is None: + return handler(*args, **kwargs) + with context.function(node_id): + return handler(*args, **kwargs) + + setattr(instrumented, "__orbit_graph_node__", node) + setattr(handler, "__orbit_graph_wrapper__", instrumented) + return instrumented + + return register + + def connect( + self, + source: str, + target: str, + *, + kind: Literal["execution", "data", "condition", "loop", "error"] = "execution", + label: str | None = None, + source_port: str | None = None, + target_port: str | None = None, + ) -> GraphEdge: + """Declare a typed arrow; loops and conditions model control flow.""" + edge = GraphEdge(source, target, kind, label, source_port, target_port) + self._edges.append(edge) + return edge + + def definition(self) -> dict[str, object]: + """Return JSON-safe graph data for a visual client or source inspector.""" + return { + "nodes": [ + { + "id": node.id, + "title": node.title, + "phase": node.phase, + "inputs": list(node.inputs), + "outputs": list(node.outputs), + "description": node.description, + **({"after_supervision": True} if node.after_supervision else {}), + } + for node in self._nodes.values() + ], + "edges": [ + { + "source": edge.source, + "target": edge.target, + "kind": edge.kind, + "label": edge.label, + "source_port": edge.source_port, + "target_port": edge.target_port, + } + for edge in self._edges + ], + } + + +graph = Graph() diff --git a/backend/orbit_sdk/decorators/runner.py b/backend/orbit_sdk/decorators/runner.py new file mode 100644 index 0000000..8217faf --- /dev/null +++ b/backend/orbit_sdk/decorators/runner.py @@ -0,0 +1,75 @@ +"""Lifecycle decorator registration and runner CLI dispatch.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING + +from ..core.lifecycle import canonical_phase +from .graph import Graph, graph + +if TYPE_CHECKING: + from ..core.context import RunnerContext + + +class Runner: + """Register lifecycle handlers without owning runtime context operations.""" + + def __init__(self, graph_definition: Graph | None = None) -> None: + """Create a runner registry, using the public graph singleton by default.""" + self._graph = graph_definition or graph + self._handlers: dict[str, Callable[[RunnerContext], None]] = {} + + def phase(self, name: str, *, step_id: str | None = None): + """Register one handler for an Orbit lifecycle phase.""" + + def register(handler: Callable[[RunnerContext], None]) -> Callable[[RunnerContext], None]: + phase = canonical_phase(name) + key = f"{phase}:{step_id}" if step_id else phase + self._handlers[key] = handler + setattr(handler, "__orbit_phase__", phase) + return handler + + return register + + def main(self) -> None: + """Dispatch the lifecycle phase requested by the Orbit process.""" + parser = argparse.ArgumentParser(description="Orbit runner phase") + command = parser.add_mutually_exclusive_group(required=True) + command.add_argument("--phase") + parser.add_argument("--step") + command.add_argument("--graph", action="store_true") + args = parser.parse_args() + if args.graph: + print(json.dumps(self._graph.definition(), ensure_ascii=False)) + return + phase = canonical_phase(args.phase) + handler = self._handlers.get(f"{phase}:{args.step}") if args.step else None + handler = handler or self._handlers.get(phase) + if handler is None: + raise SystemExit(f"runner does not define phase: {args.phase}") + from ..core.context import RunnerContext + + context = RunnerContext( + phase, + Path(os.environ["ORBIT_TARGET_REPOSITORY"]), + os.environ.get("ORBIT_EXECUTION_MODE", "run"), + int(os.environ.get("ORBIT_LOOP_INDEX", "1")), + ) + handler = getattr(handler, "__orbit_graph_wrapper__", handler) + before = context.git_head() + try: + handler(context) + finally: + try: + context.record_commit_change(before) + except (OSError, subprocess.CalledProcessError) as error: + context.log(f"Could not retain commit-change evidence: {error}") + + +runner = Runner(graph) diff --git a/backend/orbit_sdk/decorators/visual.py b/backend/orbit_sdk/decorators/visual.py new file mode 100644 index 0000000..17ca654 --- /dev/null +++ b/backend/orbit_sdk/decorators/visual.py @@ -0,0 +1,5 @@ +"""Decorator used by SDK packages to register visual node definitions.""" + +from ..visual.registry import visual_node + +__all__ = ["visual_node"] diff --git a/backend/orbit_sdk/helpers/__init__.py b/backend/orbit_sdk/helpers/__init__.py new file mode 100644 index 0000000..d5ae8f8 --- /dev/null +++ b/backend/orbit_sdk/helpers/__init__.py @@ -0,0 +1,5 @@ +"""Pure helper functions shared by runner-runtime features.""" + +from .commands import command_from_environment + +__all__ = ["command_from_environment"] diff --git a/backend/orbit_sdk/helpers/commands.py b/backend/orbit_sdk/helpers/commands.py new file mode 100644 index 0000000..87b5e41 --- /dev/null +++ b/backend/orbit_sdk/helpers/commands.py @@ -0,0 +1,26 @@ +"""Pure command-configuration helpers shared by runner operations.""" + +from __future__ import annotations + +import json +import shlex +from collections.abc import Mapping + + +def command_from_environment(environment: Mapping[str, str], command_env: str) -> list[str]: + """Parse a non-empty command from one runner environment variable.""" + configured = environment.get(command_env, "").strip() + if not configured: + raise ValueError(f"Set {command_env} to an external tool command") + try: + command = json.loads(configured) if configured.startswith("[") else shlex.split(configured) + except (json.JSONDecodeError, ValueError) as error: + raise ValueError(f"{command_env} must be a JSON string array or command") from error + if ( + not isinstance(command, list) + or not command + or not all(isinstance(item, str) for item in command) + or not command[0].strip() + ): + raise ValueError(f"{command_env} must be a non-empty JSON string array or command") + return command diff --git a/backend/orbit_sdk/visual/__init__.py b/backend/orbit_sdk/visual/__init__.py new file mode 100644 index 0000000..25f6ac3 --- /dev/null +++ b/backend/orbit_sdk/visual/__init__.py @@ -0,0 +1,31 @@ +"""Visual runner node declarations and runtime integration.""" + +from . import browser as _browser # noqa: F401 - registers browser template nodes +from . import builtin_templates as _builtin_templates # noqa: F401 - loads built-in blueprints +from . import builtins as _builtins # noqa: F401 - registers curated SDK nodes +from . import improvement as _improvement # noqa: F401 - registers improvement template nodes +from . import journeys as _journeys # noqa: F401 - registers journey template nodes +from .builtin_templates import ( + TemplateDefinition, + definition_for_quick_start, + definition_for_runner_template, + instantiate_definition, +) +from .builtin_templates import ( + catalog as builtin_template_catalog, +) +from .registry import VisualNodeDefinition, VisualNodeRegistry, visual_node, visual_nodes +from .starters import catalog as starter_catalog + +__all__ = [ + "TemplateDefinition", + "VisualNodeDefinition", + "VisualNodeRegistry", + "builtin_template_catalog", + "definition_for_quick_start", + "definition_for_runner_template", + "instantiate_definition", + "starter_catalog", + "visual_node", + "visual_nodes", +] diff --git a/backend/orbit_sdk/visual/bindings.py b/backend/orbit_sdk/visual/bindings.py new file mode 100644 index 0000000..fe40cf1 --- /dev/null +++ b/backend/orbit_sdk/visual/bindings.py @@ -0,0 +1,146 @@ +"""Safe runtime value bindings and predicates for visual runner nodes. + +Blueprints are data, never Python expressions. This module resolves a small +allowlist of runner-context values and declared input ports so generated code +can remain deterministic and safe to inspect. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from copy import deepcopy +from typing import Any + +_CONTEXT_VALUES = { + "phase", + "loop_index", + "build", + "test_cases", + "previous_supervisor_feedback", + "current_issue_assessment", + "environment", +} +_BINDING_KEYS = {"$ctx", "$input", "$resource", "$state"} + + +def is_binding(value: object) -> bool: + """Return whether a value is one supported dynamic binding object.""" + return isinstance(value, Mapping) and len(value) == 1 and next(iter(value), "") in _BINDING_KEYS + + +def validate(value: object) -> None: + """Reject unsupported dynamic binding syntax while allowing ordinary JSON.""" + if not isinstance(value, Mapping): + if isinstance(value, list): + for item in value: + validate(item) + return + keys = set(value) + reserved = keys & _BINDING_KEYS + if reserved: + if len(keys) != 1: + raise ValueError("a visual runtime binding must contain exactly one $ key") + key = next(iter(reserved)) + binding = value[key] + if key in {"$ctx", "$input", "$resource"} and not isinstance(binding, str): + raise ValueError(f"{key} binding must be a string") + if key == "$state" and (not isinstance(binding, Mapping) or not isinstance(binding.get("name"), str)): + raise ValueError("$state binding requires a state name") + return + for item in value.values(): + validate(item) + + +def resolve(ctx: Any, value: object, inputs: Mapping[str, object]) -> object: + """Resolve bindings recursively against explicitly allowed runtime values.""" + if not is_binding(value): + if isinstance(value, Mapping): + return {str(key): resolve(ctx, item, inputs) for key, item in value.items()} + if isinstance(value, list): + return [resolve(ctx, item, inputs) for item in value] + return deepcopy(value) + key, binding = next(iter(value.items())) # type: ignore[union-attr] + if key == "$ctx": + return deepcopy(_path(_context_value(ctx, binding), "")) + if key == "$input": + name, _, path = binding.partition(".") + if name not in inputs: + return None + return deepcopy(_path(inputs[name], path)) + if key == "$resource": + name, _, path = binding.partition(".") + return deepcopy(_path(ctx.resource(name, {}), path)) + # $state is deliberately explicit about scope and never exposes a filesystem path. + state = ctx.load_state(binding["name"], binding.get("default"), scope=binding.get("scope", "runner")) + return deepcopy(_path(state, str(binding.get("path", "")))) + + +def evaluate(ctx: Any, condition: object, inputs: Mapping[str, object]) -> bool: + """Evaluate a compact JSON predicate used by a node's ``when`` property.""" + if condition is None: + return True + if isinstance(condition, Mapping) and len(condition) == 1: + operator, operand = next(iter(condition.items())) + if operator == "$not": + return not evaluate(ctx, operand, inputs) + if operator in {"$all", "$any"}: + if not isinstance(operand, list): + raise ValueError(f"{operator} requires an array") + results = [evaluate(ctx, item, inputs) for item in operand] + return all(results) if operator == "$all" else any(results) + if operator in {"$equals", "$not_equals"}: + if not isinstance(operand, list) or len(operand) != 2: + raise ValueError(f"{operator} requires exactly two values") + equal = resolve(ctx, operand[0], inputs) == resolve(ctx, operand[1], inputs) + return equal if operator == "$equals" else not equal + if operator == "$exists": + return resolve(ctx, operand, inputs) is not None + return bool(resolve(ctx, condition, inputs)) + + +def validate_condition(condition: object) -> None: + """Validate predicate syntax without evaluating runner data at save time.""" + if condition is None or isinstance(condition, bool): + return + if isinstance(condition, Mapping) and len(condition) == 1: + operator, operand = next(iter(condition.items())) + if operator == "$not": + return validate_condition(operand) + if operator in {"$all", "$any"}: + if not isinstance(operand, list): + raise ValueError(f"{operator} requires an array") + for item in operand: + validate_condition(item) + return + if operator in {"$equals", "$not_equals"}: + if not isinstance(operand, list) or len(operand) != 2: + raise ValueError(f"{operator} requires exactly two values") + validate(operand[0]) + validate(operand[1]) + return + if operator == "$exists": + return validate(operand) + validate(condition) + + +def _context_value(ctx: Any, binding: str) -> object: + name, _, path = binding.partition(".") + if name not in _CONTEXT_VALUES: + raise ValueError(f"unsupported $ctx binding: {name}") + return _path(getattr(ctx, name), path) + + +def _path(value: object, path: str) -> object: + """Read a dot-separated dict/list path without attribute access or calls.""" + if not path: + return value + current = value + for part in path.split("."): + if isinstance(current, Mapping): + current = current.get(part) + elif isinstance(current, list) and part.isdigit(): + index = int(part) + current = current[index] if 0 <= index < len(current) else None + else: + return None + return current diff --git a/backend/orbit_sdk/visual/browser.py b/backend/orbit_sdk/visual/browser.py new file mode 100644 index 0000000..9144736 --- /dev/null +++ b/backend/orbit_sdk/visual/browser.py @@ -0,0 +1,187 @@ +"""Browser and source-contract operations shared by built-in runner assets.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from ..decorators.visual import visual_node + + +@visual_node( + kind="run_browser_smoke", + group_key="browser", + display_name="Run browser smoke journey", + title_key="visual.nodes.runBrowserSmoke.title", + description_key="visual.nodes.runBrowserSmoke.description", + default_config={"namespace": "browser_smoke", "fail_on_unpassed": True}, + default_outputs=("journey_evidence",), +) +def run_browser_smoke(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Run fixed browser cases and retain the exact smoke-test evidence envelope.""" + evidence = ctx.playwright_journey() + namespace = str(config.get("namespace", "browser_smoke")) + ctx.emit_result({namespace: {"iteration": ctx.loop_index, "evidence": evidence}}) + if config.get("fail_on_unpassed", True) and not all( + item.get("passed") for item in evidence.get("results", []) + ): + raise SystemExit("A browser journey failed") + return {"journey_evidence": evidence} + + +@visual_node( + kind="validate_source_contract", + group_key="browser", + display_name="Validate source contract", + title_key="visual.nodes.validateSourceContract.title", + description_key="visual.nodes.validateSourceContract.description", + default_config={ + "namespace": "source_aware_journey", + "pattern_env": "ORBIT_SOURCE_CONTRACT_PATTERN", + "glob_env": "ORBIT_SOURCE_CONTRACT_GLOB", + }, + default_outputs=("source_contract",), +) +def validate_source_contract( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Collect bounded source matches and validate browser inputs for a journey.""" + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one fixed journey case are required") + pattern = str(ctx.environment.get(config.get("pattern_env", "ORBIT_SOURCE_CONTRACT_PATTERN"), "")).strip() + if not pattern: + raise ValueError("Set ORBIT_SOURCE_CONTRACT_PATTERN to a required source expression") + glob = str(ctx.environment.get(config.get("glob_env", "ORBIT_SOURCE_CONTRACT_GLOB"), "")).strip() + command = ["rg", "-l", pattern] + if glob: + command.extend(["--glob", glob]) + command.append(".") + output = ctx.exec(command, cwd=ctx.project_root, timeout=30) + files = [path for path in output.splitlines() if path][:40] + if not files: + raise ValueError("The configured source contract was not found in the target project") + contract = {"pattern": pattern, "glob": glob or None, "files": files} + namespace = str(config.get("namespace", "source_aware_journey")) + ctx.emit_result({namespace: {"source_contract": contract}}) + ctx.log(f"Validated source contract in {len(files)} file(s)") + return {"source_contract": contract} + + +@visual_node( + kind="validate_tailwind_source", + group_key="browser", + display_name="Validate Tailwind source", + title_key="visual.nodes.validateSourceContract.title", + description_key="visual.nodes.validateSourceContract.description", + default_outputs=("source_contract",), +) +def validate_tailwind_source(ctx: Any, _: Mapping[str, Any], __: Mapping[str, object]) -> dict[str, object]: + """Require Tailwind source evidence together with a runnable browser journey.""" + files = ctx.exec( + ["sh", "-lc", "rg -l 'tailwindcss|@tailwind' --glob '*.css' . || true"], + cwd=ctx.project_root, + timeout=30, + ) + config = ctx.exec( + ["sh", "-lc", "rg --files -g 'tailwind.config.*' . || true"], cwd=ctx.project_root, timeout=30 + ) + if not files.strip() and not config.strip(): + raise ValueError("No Tailwind configuration or stylesheet was found in the target project") + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one journey case are required") + contract = {"stylesheet_files": files.splitlines()[:40], "config_files": config.splitlines()[:20]} + ctx.emit_result({"tailwind_journey": contract}) + ctx.log("Validated the Tailwind source contract") + return {"source_contract": contract} + + +@visual_node( + kind="run_source_aware_browser_journey", + group_key="browser", + display_name="Run source-aware browser journey", + title_key="visual.nodes.runSourceAwareBrowserJourney.title", + description_key="visual.nodes.runSourceAwareBrowserJourney.description", + default_config={"namespace": "source_aware_journey"}, + default_outputs=("browser_evidence",), +) +def run_source_aware_browser_journey( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Run rendered evidence independently from the source-side contract.""" + evidence = ctx.playwright_journey() + namespace = str(config.get("namespace", "source_aware_journey")) + ctx.emit_result({namespace: {"iteration": ctx.loop_index, "evidence": evidence}}) + artifact = ctx.save_data_file( + f"source-aware-journey/iteration-{ctx.loop_index}.json", + json.dumps(evidence, ensure_ascii=False, indent=2), + label="Source-aware rendered journey evidence", + content_type="application/json", + ) + return {"browser_evidence": evidence, "artifact": artifact} + + +_SITE_SCRIPT = """const {chromium}=require(process.argv[1]),i=JSON.parse(process.argv[2]),blocked=/(logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe)/i;(async()=>{const b=await chromium.launch({headless:true}),p=await b.newPage(),v=[],seen=new Set(),o=new URL(i.baseUrl).origin;try{await p.goto(i.baseUrl,{waitUntil:'domcontentloaded',timeout:30000});for(let n=0;n<=i.maxClicks;n++){seen.add(p.url());v.push({url:p.url(),title:await p.title(),text:(await p.locator('body').innerText().catch(()=>'' )).replace(/\\s+/g,' ').slice(0,1200)});if(n===i.maxClicks)break;const a=await p.locator('a[href]').evaluateAll(x=>x.map((e,k)=>({k,href:e.href,text:(e.textContent||'').trim()})));const c=a.filter(x=>{try{const u=new URL(x.href);return u.origin===o&&!seen.has(u.href)&&!blocked.test(u.pathname+' '+x.text)}catch{return false}});if(!c.length)break;await p.locator('a[href]').nth(c[0].k).click({timeout:5000});await p.waitForLoadState('domcontentloaded',{timeout:10000}).catch(()=>{})}await p.screenshot({path:i.screenshot,fullPage:true});console.log('__ORBIT_SITE_EXPLORATION_RESULT__'+JSON.stringify({visited:v,screenshot:i.screenshot}))}finally{await b.close()}})().catch(e=>{console.error(e);process.exit(1)})""" + + +@visual_node( + kind="explore_rendered_site", + group_key="browser", + display_name="Explore rendered site", + title_key="visual.nodes.exploreRenderedSite.title", + description_key="visual.nodes.exploreRenderedSite.description", + default_config={"namespace": "site_exploration", "max_clicks": 3}, + default_outputs=("rendered_pages",), +) +def explore_rendered_site(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Visit bounded safe same-origin links and retain rendered page evidence.""" + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("Set required build field(s): browser_base_url and test cases") + # The SDK package lives at backend/orbit_sdk; the checked-in frontend is at repository root. + import orbit_sdk + + module = str(Path(orbit_sdk.__file__).resolve().parents[2] / "frontend" / "node_modules" / "playwright") + screenshot = ( + ctx.app_data + / "artifacts" + / ctx.environment.get("ORBIT_RUN_ID", "manual") + / f"loop-{ctx.loop_index}" + / "site-exploration.png" + ) + screenshot.parent.mkdir(parents=True, exist_ok=True) + output = ctx.exec( + [ + "node", + "-e", + _SITE_SCRIPT, + module, + json.dumps( + { + "baseUrl": ctx.build["browser_base_url"], + "maxClicks": config.get("max_clicks", 3), + "screenshot": str(screenshot), + } + ), + ], + timeout=120, + target_log_source="site-exploration", + target_log_exclude_prefixes=("__ORBIT_SITE_EXPLORATION_RESULT__",), + ) + payload = next( + ( + line.removeprefix("__ORBIT_SITE_EXPLORATION_RESULT__") + for line in reversed(output.splitlines()) + if line.startswith("__ORBIT_SITE_EXPLORATION_RESULT__") + ), + "", + ) + if not payload: + raise RuntimeError("site exploration did not return structured evidence") + evidence = json.loads(payload) + titles = [str(page.get("title") or page.get("url")) for page in evidence.get("visited", [])] + value = { + "evidence": evidence, + "opinion": f"Explored {len(titles)} rendered page(s): {'; '.join(titles[:3])}. Review the captured pages for clarity, usefulness, and friction.", + } + ctx.emit_result({str(config.get("namespace", "site_exploration")): value}) + return {"rendered_pages": value} diff --git a/backend/orbit_sdk/visual/builtin_templates.py b/backend/orbit_sdk/visual/builtin_templates.py new file mode 100644 index 0000000..7273dd2 --- /dev/null +++ b/backend/orbit_sdk/visual/builtin_templates.py @@ -0,0 +1,592 @@ +"""Trusted built-in runner templates exposed as faithful visual blueprints. + +Unlike user-authored code, these sources ship with OpenOrbit. Their graph +annotations are read as data and each visual node delegates to the canonical +step function. This keeps Visual Mode and the original template behavior in +lockstep without attempting to parse arbitrary Python back into a graph. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +import re +from copy import deepcopy +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any, Mapping + +from .layout import initial_positions + +_ROOT = Path(__file__).resolve().parents[3] +_TEMPLATE_ROOT = _ROOT / "templates" +_CANONICAL_ROOT = Path(__file__).resolve().parent / "canonical" + + +@dataclass(frozen=True) +class TemplateStep: + """One graph-declared function from a trusted built-in runner.""" + + id: str + function_name: str + title: str + phase: str + inputs: tuple[str, ...] + outputs: tuple[str, ...] + description: str | None + + +@dataclass(frozen=True) +class BuiltinTemplate: + """Parsed graph data plus the canonical source path for one runner.""" + + id: str + display_name: str + source: Path + steps: tuple[TemplateStep, ...] + edges: tuple[dict[str, object], ...] + + +@dataclass(frozen=True) +class TemplateDefinition: + """One authoritative Visual Mode definition for a shipped runner asset. + + ``blueprint`` is the persisted representation, while ``source_sha256`` + records which canonical template revision it was derived from. The + every node is a direct SDK operation, so callers never need to know the + source-level function boundaries of a shipped template. + """ + + id: str + display_name: str + blueprint: dict[str, object] + source_sha256: str + + +def _literal(value: ast.AST, default: Any = None) -> Any: + try: + return ast.literal_eval(value) + except (ValueError, TypeError): + return default + + +def _call_name(call: ast.Call) -> str | None: + if isinstance(call.func, ast.Attribute) and isinstance(call.func.value, ast.Name): + return f"{call.func.value.id}.{call.func.attr}" + return None + + +def _template_id(path: Path) -> str: + return path.relative_to(_CANONICAL_ROOT).parent.as_posix().replace("/", ":") + + +def _display_name(path: Path) -> str: + relative = path.relative_to(_CANONICAL_ROOT) + metadata = ( + _TEMPLATE_ROOT + / relative.parent + / ("manifest.json" if "quick-starts" in relative.parts else "template.json") + ) + if metadata.is_file(): + try: + values = json.loads(metadata.read_text(encoding="utf-8")) + if isinstance(values.get("name"), str): + return values["name"] + except (OSError, ValueError): + pass + return path.parent.name.replace("-", " ").title() + + +def _parse(path: Path) -> BuiltinTemplate | None: + try: + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except (OSError, SyntaxError): + return None + steps: list[TemplateStep] = [] + edges: list[dict[str, object]] = [] + for statement in module.body: + if ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Call) + and _call_name(statement.value) == "graph.connect" + ): + call = statement.value + if len(call.args) < 2: + continue + source, target = _literal(call.args[0]), _literal(call.args[1]) + if not isinstance(source, str) or not isinstance(target, str): + continue + keywords = {item.arg: _literal(item.value) for item in call.keywords if item.arg} + edges.append( + { + "source": source, + "target": target, + "kind": keywords.get("kind", "execution"), + "label": keywords.get("label"), + "source_port": keywords.get("source_port"), + "target_port": keywords.get("target_port"), + } + ) + if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + graph: dict[str, Any] | None = None + for decorator in statement.decorator_list: + if not isinstance(decorator, ast.Call) or _call_name(decorator) != "graph.step": + continue + if not decorator.args: + continue + node_id = _literal(decorator.args[0]) + if not isinstance(node_id, str): + continue + keywords = {item.arg: _literal(item.value) for item in decorator.keywords if item.arg} + graph = {"id": node_id, **keywords} + if graph and isinstance(graph.get("phase"), str): + steps.append( + TemplateStep( + id=graph["id"], + function_name=statement.name, + title=str(graph.get("title") or graph["id"].replace("-", " ").title()), + phase=graph["phase"], + inputs=tuple(graph.get("inputs") or ()), + outputs=tuple(graph.get("outputs") or ()), + description=graph.get("description") + if isinstance(graph.get("description"), str) + else None, + ) + ) + if not steps: + return None + return BuiltinTemplate(_template_id(path), _display_name(path), path, tuple(steps), tuple(edges)) + + +@lru_cache(maxsize=1) +def templates() -> dict[str, BuiltinTemplate]: + """Discover all repository-shipped runner and Quick Start sources.""" + paths = [ + path + for path in sorted(_CANONICAL_ROOT.glob("**/runner.py")) + if path.parent.name not in {"quick-start.example", "runner-template.example"} + ] + values = [_parse(path) for path in paths] + return {value.id: value for value in values if value is not None} + + +def catalog() -> list[dict[str, Any]]: + """Return one visual starter per trusted built-in runner source.""" + return [ + { + "id": f"builtin:{definition.id}", + "group_key": "templates", + "title_key": f"visual.templates.{definition.id.replace(':', '.').replace('-', '_')}.title", + "description_key": f"visual.templates.{definition.id.replace(':', '.').replace('-', '_')}.description", + "display_name": definition.display_name, + "blueprint": deepcopy(definition.blueprint), + } + for definition in definitions().values() + ] + + +_AGENT_PAYLOAD = { + "iteration": {"$ctx": "loop_index"}, + "build": {"$ctx": "build"}, + "test_cases": {"$ctx": "test_cases"}, +} +_PROBE_PAYLOAD = { + "iteration": {"$ctx": "loop_index"}, + "build": {"$ctx": "build"}, + "probes": {"$ctx": "test_cases"}, +} + + +def _node(step: TemplateStep, kind: str, config: Mapping[str, object]) -> dict[str, object]: + """Create a persisted node while retaining the template's public graph ID.""" + return { + "id": step.id, + "kind": kind, + "title": step.title, + "phase": step.phase, + "inputs": list(step.inputs), + "outputs": list(step.outputs), + "description": step.description, + "config": dict(config), + "script": "", + } + + +def _cycle_materials( + *, + command_env: str, + namespace: str, + actions: tuple[str, str, str, str], + payload: Mapping[str, object], + action_kind: str, + contract_kind: str, + contract_config: Mapping[str, object], + close: str, + finalize: str, + log_source: str, +) -> dict[str, tuple[str, Mapping[str, object]]]: + """Return the reusable operations shared by external evaluator cycles.""" + names = ("preflight", "prepared", "result", "evidence") + step_ids = ("preflight", "prepare", "run", "collect") + values: dict[str, tuple[str, Mapping[str, object]]] = {} + for step_id, result_key, action in zip(step_ids, names, actions): + values[step_id] = ( + action_kind, + { + "command_env": command_env, + "action": action, + "namespace": namespace, + "result_key": result_key, + "input_env": "ORBIT_CYCLE_INPUT", + "input_data": payload, + "timeout": 3600, + "log_source": log_source, + "include_iteration": step_id != "preflight", + }, + ) + values.update( + { + "validate": (contract_kind, contract_config), + "close": ("log_message", {"message": close}), + "finalize": ("log_message", {"message": finalize}), + } + ) + return values + + +def _materials_for(template: BuiltinTemplate) -> dict[str, tuple[str, Mapping[str, object]]]: + """Map canonical step IDs to SDK materials, never to palette-only assets. + + The keys intentionally describe runner semantics rather than Python function + names. A visual Blueprint therefore survives an implementation refactor in + a shipped template without gaining a new template-specific node type. + """ + template_id = template.id + if template_id in { + "runner-templates:json-agent-cycle", + "quick-starts:openorbit.ai-experience-improvement", + }: + return _cycle_materials( + command_env="ORBIT_AGENT_COMMAND", + namespace="agent_cycle", + actions=("status", "prepare", "run-once", "collect-evidence"), + payload=_AGENT_PAYLOAD, + action_kind="json_cycle_action", + contract_kind="require_test_cases", + contract_config={}, + close="Completed one bounded external agent cycle", + finalize="Finalized the external agent", + log_source="agent-cycle", + ) + if template_id == "runner-templates:external-command-adapter": + return _cycle_materials( + command_env="ORBIT_ADAPTER_COMMAND", + namespace="external_adapter", + actions=("status", "prepare", "run-once", "collect-evidence"), + payload={}, + action_kind="command_cycle_action", + contract_kind="validate_command_environment", + contract_config={"command_env": "ORBIT_ADAPTER_COMMAND"}, + close="Completed one bounded external adapter cycle", + finalize="Finalized the external automation evaluation", + log_source="external-adapter", + ) + if template_id in { + "runner-templates:evidence-gated-probe-cycle", + "quick-starts:openorbit.ai-slo-drift-monitor", + }: + return _cycle_materials( + command_env="ORBIT_PROBE_COMMAND", + namespace="probe_gate", + actions=("preflight", "prepare", "run-probes", "collect-evidence"), + payload=_PROBE_PAYLOAD, + action_kind="json_cycle_action", + contract_kind="require_test_cases", + contract_config={"label": "fixed probe case"}, + close="Completed one bounded probe matrix cycle", + finalize="Finalized the probe matrix", + log_source="probe-gate", + ) + if template_id in {"runner-templates:site-exploration", "quick-starts:openorbit.site-exploration-review"}: + return { + "validate-site": ("validate_build_fields", {"fields": ["browser_base_url"]}), + "explore-site": ("explore_rendered_site", {"namespace": "site_exploration", "max_clicks": 3}), + "review-evidence": ( + "log_message", + {"message": "Retained rendered exploration evidence for review"}, + ), + "finalize-review": ("log_message", {"message": "Finalized the bounded site exploration review"}), + } + if template_id == "runner-templates:source-aware-browser-journey": + return { + "validate-source-contract": ("validate_source_contract", {"namespace": "source_aware_journey"}), + "run-browser-journey": ( + "run_source_aware_browser_journey", + {"namespace": "source_aware_journey"}, + ), + "publish-rendered-evidence": ( + "log_message", + {"message": "Published source context with rendered browser evidence"}, + ), + "close-source-aware-cycle": ( + "log_message", + {"message": "Completed one bounded source-aware browser journey"}, + ), + "finalize-source-aware-journey": ( + "log_message", + {"message": "Finalized the source-aware browser evaluation"}, + ), + } + if template_id in {"runner-templates:user-journey-cycle", "quick-starts:openorbit.critical-flow-proof"}: + namespace = "critical_flow" if "critical-flow" in template_id else "user_journey" + prefix = "critical-flow" if "critical-flow" in template_id else "user-journey" + return { + f"validate-{prefix}-runtime": ("validate_browser_runtime", {}), + f"validate-{prefix}": ("require_test_cases", {"label": "fixed journey case"}), + f"load-{prefix}-state": ("initialize_user_journey_state", {"namespace": namespace}), + f"plan-{prefix}": ("plan_user_journey", {"namespace": namespace}), + f"run-{prefix}": ("run_user_journey", {"namespace": namespace}), + f"review-{prefix}": ("publish_user_journey_handoff", {"namespace": namespace}), + f"retain-{prefix}": ("log_message", {"message": "Closed this bounded user journey"}), + f"finalize-{prefix}": ("log_message", {"message": "Finalized the user journey evaluation"}), + } + if template_id == "quick-starts:openorbit.user-journey-smoke-test": + return { + "validate-browser-runtime": ("validate_browser_runtime", {}), + "validate-browser-journey": ("require_test_cases", {"label": "browser journey case"}), + "run-browser-journey": ( + "run_browser_smoke", + {"namespace": "browser_smoke", "fail_on_unpassed": True}, + ), + "verify-browser-evidence": ( + "log_message", + {"message": "Quick start browser evaluation completed"}, + ), + "finalize-browser-evaluation": ( + "log_message", + {"message": "Finalized the one-shot browser evaluation"}, + ), + } + if template_id == "runner-templates:native-improvement-cycle": + return { + "validate-target": ("validate_git_repository", {}), + "validate-evaluation-inputs": ("validate_improvement_inputs", {}), + "snapshot-baseline": ("snapshot", {"operation": "save_before_each"}), + "prepare-prompt": ("prepare_managed_prompt", {"mode": "apply_accepted"}), + "exercise-target": ("exercise_managed_prompt", {"include_candidate": True}), + "assess-candidate": ("assess_prompt_candidate", {}), + "retain-iteration": ("retain_improvement_iteration", {}), + "restore-baseline": ("restore_improvement_baseline", {}), + } + if template_id == "quick-starts:openorbit.agent-self-improvement": + return { + "validate-target": ("validate_git_repository", {}), + "validate-evaluation-inputs": ("validate_improvement_inputs", {}), + "prepare-prompt": ("prepare_managed_prompt", {"mode": "isolated_worktree"}), + "exercise-target": ("exercise_managed_prompt", {}), + "retain-iteration": ( + "log_message", + {"message": "Retained target-AI responses and supervisor assessment evidence"}, + ), + "propose-agent-change": ( + "create_agent_proposal", + {"provider": "${agent_provider}", "options": "${agent_options}"}, + ), + } + if template_id == "quick-starts:openorbit.continuous-user-journey": + return { + "validate-browser-runtime": ("validate_browser_runtime", {}), + "validate": ("validate_persona_contract", {}), + "observe": ("observe_persona_page", {"namespace": "autonomous_persona"}), + "decide": ("plan_persona_action", {"namespace": "autonomous_persona"}), + "act": ("run_persona_action", {"namespace": "autonomous_persona"}), + "reflect": ("reflect_persona_session", {"namespace": "autonomous_persona"}), + "finalize": ("log_message", {"message": "Finalized the autonomous persona journey"}), + } + if template_id == "runner-templates:autonomous-persona-journey": + return { + "validate": ("validate_persona_contract", {}), + "observe": ("observe_persona_page", {"namespace": "autonomous_persona"}), + "decide": ("plan_persona_action", {"namespace": "autonomous_persona"}), + "act": ("run_persona_action", {"namespace": "autonomous_persona"}), + "reflect": ("reflect_persona_session", {"namespace": "autonomous_persona"}), + "finalize-persona-journey": ( + "log_message", + {"message": "Finalized the autonomous persona journey"}, + ), + } + if template_id == "runner-templates:playwright-continuous-journey": + return { + "validate": ("validate_browser_journey_contract", {}), + "plan": ("plan_user_journey", {"namespace": "continuous_journey"}), + "exercise": ("run_user_journey", {"namespace": "continuous_journey"}), + "retain": ("publish_user_journey_handoff", {"namespace": "continuous_journey"}), + "close-journey-cycle": ( + "log_message", + {"message": "Completed one bounded continuous browser journey"}, + ), + "finalize-browser-journey": ( + "log_message", + {"message": "Finalized the continuous browser journey"}, + ), + } + if template_id == "runner-templates:selenium-external-journey": + return { + "ready": ("validate_command_environment", {"command_env": "ORBIT_SELENIUM_COMMAND"}), + "prepare": ( + "command_cycle_action", + { + "command_env": "ORBIT_SELENIUM_COMMAND", + "action": "prepare", + "namespace": "selenium_journey", + "result_key": "prepared", + "timeout": 3600, + "log_source": "selenium-adapter", + }, + ), + "run": ( + "command_cycle_action", + { + "command_env": "ORBIT_SELENIUM_COMMAND", + "action": "run-once", + "namespace": "selenium_journey", + "result_key": "result", + "timeout": 3600, + "log_source": "selenium-adapter", + }, + ), + "collect": ( + "command_cycle_action", + { + "command_env": "ORBIT_SELENIUM_COMMAND", + "action": "collect-evidence", + "namespace": "selenium_journey", + "result_key": "evidence", + "timeout": 3600, + "log_source": "selenium-adapter", + }, + ), + "close-selenium-cycle": ( + "log_message", + {"message": "Completed one bounded Selenium adapter cycle"}, + ), + "finalize-selenium-journey": ( + "log_message", + {"message": "Finalized the Selenium adapter evaluation"}, + ), + } + if template_id == "runner-templates:tailwind-source-aware": + return { + "inspect-source": ("validate_tailwind_source", {}), + "run-browser": ("run_source_aware_browser_journey", {"namespace": "tailwind_journey"}), + "publish-evidence": ( + "log_message", + {"message": "Published Tailwind source context with rendered browser evidence"}, + ), + "close-tailwind-cycle": ( + "log_message", + {"message": "Completed one bounded Tailwind browser journey"}, + ), + "finalize-tailwind-journey": ( + "log_message", + {"message": "Finalized the Tailwind browser evaluation"}, + ), + } + return {} + + +def _material_for_step( + template: BuiltinTemplate, step: TemplateStep +) -> tuple[str, Mapping[str, object]] | None: + """Resolve one canonical graph ID to a reusable operation. + + The three external-cycle families use different public graph IDs while + sharing the same seven operations, so their small ID vocabulary is handled + here instead of creating per-template node kinds. + """ + materials = _materials_for(template) + direct = materials.get(step.id) + if direct: + return direct + if step.id == "check-adapter": + return materials.get("preflight") + cycle_key = next( + ( + key + for key in ("validate", "preflight", "prepare", "run", "collect", "close", "finalize") + if step.id.startswith(key) or f"-{key}-" in step.id or step.id.endswith(f"-{key}") + ), + None, + ) + return materials.get(cycle_key or "") + + +@lru_cache(maxsize=1) +def definitions() -> dict[str, TemplateDefinition]: + """Build stable TemplateDefinitions from every built-in runner graph.""" + result: dict[str, TemplateDefinition] = {} + for template in templates().values(): + # The graph and lifecycle IDs are part of a runner's observable + # contract. Preserve them verbatim so generated executions retain the + # same step evidence as the canonical source. + ids = {step.id: step.id for step in template.steps} + nodes = [] + for step in template.steps: + material = _material_for_step(template, step) + if material is None: + raise RuntimeError(f"no reusable visual material is defined for {template.id}/{step.id}") + nodes.append(_node(step, material[0], material[1])) + for node, position in zip(nodes, initial_positions(step.phase for step in template.steps)): + node["position"] = position + edges = [ + {**edge, "source": ids[edge["source"]], "target": ids[edge["target"]]} + for edge in template.edges + if edge["source"] in ids and edge["target"] in ids + ] + result[template.id] = TemplateDefinition( + id=template.id, + display_name=template.display_name, + blueprint={"schema_version": 1, "nodes": nodes, "edges": edges}, + source_sha256=hashlib.sha256(template.source.read_bytes()).hexdigest(), + ) + return result + + +def definition_for_runner_template(template_id: str) -> TemplateDefinition | None: + """Return the Visual definition for a built-in runner-template ID.""" + return definitions().get(f"runner-templates:{template_id}") + + +def definition_for_quick_start(quick_start_id: str) -> TemplateDefinition | None: + """Return the Visual definition for a built-in Quick Start runner.""" + return definitions().get(f"quick-starts:{quick_start_id}") + + +def instantiate_definition( + definition: TemplateDefinition, parameters: Mapping[str, str] | None = None +) -> dict[str, object]: + """Return a blueprint bound to approved Quick Start parameter values.""" + blueprint = deepcopy(definition.blueprint) + values = dict(parameters or {}) + if values: + blueprint["parameters"] = values + + def bind(value: object) -> object: + if isinstance(value, str): + return re.sub( + r"\$\{([a-z][a-z0-9_]*)\}", + lambda match: values.get(match.group(1), match.group(0)), + value, + ) + if isinstance(value, list): + return [bind(item) for item in value] + if isinstance(value, dict): + return {key: bind(item) for key, item in value.items()} + return value + + for node in blueprint["nodes"]: + node["config"] = bind(node["config"]) + return blueprint diff --git a/backend/orbit_sdk/visual/builtins.py b/backend/orbit_sdk/visual/builtins.py new file mode 100644 index 0000000..1f54f5d --- /dev/null +++ b/backend/orbit_sdk/visual/builtins.py @@ -0,0 +1,354 @@ +"""Curated SDK operations available to visual runner blueprints.""" + +from __future__ import annotations + +from typing import Any, Mapping + +from ..decorators.visual import visual_node +from .registry import VisualNodeDefinition, visual_nodes + +visual_nodes.register( + VisualNodeDefinition( + kind="custom_script", + group_key="advanced", + display_name="Custom Script", + title_key="visual.nodes.customScript.title", + description_key="visual.nodes.customScript.description", + is_custom_script=True, + ) +) + + +def _result(value: object) -> dict[str, object]: + return {"result": value} + + +def _validate_snapshot(config: Mapping[str, Any]) -> None: + if config.get("operation") not in {"save_before_each", "save_first_after_each", "restore_before_each"}: + raise ValueError("has an unsupported snapshot operation") + + +def _validate_evidence(config: Mapping[str, Any]) -> None: + if not isinstance(config.get("namespace", "visual_runner"), str): + raise ValueError("requires a text evidence namespace") + + +def _validate_fields(config: Mapping[str, Any]) -> None: + fields = config.get("fields") + if ( + not isinstance(fields, list) + or not fields + or not all(isinstance(field, str) and field for field in fields) + ): + raise ValueError("requires one or more build field names") + + +@visual_node( + kind="external_json_action", + group_key="actions", + display_name="External JSON action", + default_config={"command_env": "ORBIT_AGENT_COMMAND", "action": "run-once", "input_data": {}}, + title_key="visual.nodes.externalJsonAction.title", + description_key="visual.nodes.externalJsonAction.description", + default_outputs=("result",), + required_config=("command_env", "action"), +) +def external_json_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return _result( + ctx.run_json_action( + command_env=config["command_env"], + action=config["action"], + input_env=config.get("input_env", "ORBIT_CYCLE_INPUT"), + input_data=config.get("input_data", {}), + timeout=config.get("timeout"), + log_source=config.get("log_source"), + ) + ) + + +@visual_node( + kind="command_action", + group_key="actions", + display_name="Command action", + default_config={"command_env": "ORBIT_COMMAND", "action": "run"}, + title_key="visual.nodes.commandAction.title", + description_key="visual.nodes.commandAction.description", + default_outputs=("result",), + required_config=("command_env", "action"), +) +def command_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return _result( + ctx.run_command_action( + command_env=config["command_env"], + action=config["action"], + timeout=config.get("timeout"), + log_source=config.get("log_source"), + ) + ) + + +@visual_node( + kind="browser_journey", + group_key="actions", + display_name="Browser journey", + default_config={}, + title_key="visual.nodes.browserJourney.title", + description_key="visual.nodes.browserJourney.description", + default_outputs=("evidence",), +) +def browser_journey(ctx: Any, _: Mapping[str, Any], __: Mapping[str, object]) -> dict[str, object]: + return {"evidence": ctx.playwright_journey()} + + +@visual_node( + kind="state_load", + group_key="state", + display_name="Load runner state", + default_config={"name": "runner_state", "default": {}, "scope": "runner"}, + title_key="visual.nodes.stateLoad.title", + description_key="visual.nodes.stateLoad.description", + default_outputs=("state",), + required_config=("name",), +) +def state_load(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return { + "state": ctx.load_state(config["name"], config.get("default"), scope=config.get("scope", "runner")) + } + + +@visual_node( + kind="state_save", + group_key="state", + display_name="Save runner state", + default_config={"name": "runner_state", "value_input": "value", "scope": "runner"}, + title_key="visual.nodes.stateSave.title", + description_key="visual.nodes.stateSave.description", + default_inputs=("value",), + default_outputs=("state",), + required_config=("name",), +) +def state_save(ctx: Any, config: Mapping[str, Any], inputs: Mapping[str, object]) -> dict[str, object]: + return { + "state": ctx.save_state( + config["name"], + inputs.get(config.get("value_input", "value"), config.get("default")), + scope=config.get("scope", "runner"), + ) + } + + +@visual_node( + kind="model_prompt", + group_key="ai", + display_name="Model prompt", + default_config={"prompt": ""}, + title_key="visual.nodes.modelPrompt.title", + description_key="visual.nodes.modelPrompt.description", + default_outputs=("result",), + required_config=("prompt",), +) +def model_prompt(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return _result(ctx.complete_model(config["prompt"])) + + +@visual_node( + kind="model_json", + group_key="ai", + display_name="Structured model prompt", + default_config={"prompt": "", "description": "model response"}, + title_key="visual.nodes.modelJson.title", + description_key="visual.nodes.modelJson.description", + default_outputs=("result",), + required_config=("prompt",), +) +def model_json(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return _result( + ctx.complete_model_json(config["prompt"], description=config.get("description", "model response")) + ) + + +@visual_node( + kind="ai_agent", + group_key="ai", + display_name="AI agent", + default_config={"prompt": "", "provider": "codex", "options": ""}, + title_key="visual.nodes.aiAgent.title", + description_key="visual.nodes.aiAgent.description", + default_outputs=("result",), + required_config=("prompt", "provider"), +) +def ai_agent(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + return _result( + ctx.run_ai_agent( + config["prompt"], + provider=config["provider"], + options=config.get("options", ""), + timeout=config.get("timeout", 1800), + ) + ) + + +@visual_node( + kind="require_test_cases", + group_key="state", + display_name="Require test cases", + default_config={"label": "fixed test case"}, + title_key="visual.nodes.requireTestCases.title", + description_key="visual.nodes.requireTestCases.description", + default_outputs=("contract",), +) +def require_test_cases(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + ctx.require_test_cases(label=config.get("label", "fixed test case")) + return {"contract": True} + + +@visual_node( + kind="snapshot", + group_key="state", + display_name="Snapshot", + default_config={"operation": "save_before_each"}, + title_key="visual.nodes.snapshot.title", + description_key="visual.nodes.snapshot.description", + default_outputs=("snapshot",), + required_config=("operation",), + config_validator=_validate_snapshot, +) +def snapshot(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + operations = { + "save_before_each": ctx.save_before_each_snapshot, + "save_first_after_each": ctx.save_first_after_each_snapshot, + "restore_before_each": ctx.restore_before_each_snapshot, + } + try: + return {"snapshot": operations[config["operation"]]()} + except KeyError as error: + raise ValueError("unsupported snapshot operation") from error + + +@visual_node( + kind="emit_evidence", + group_key="evidence", + display_name="Emit evidence", + default_config={"namespace": "visual_runner"}, + title_key="visual.nodes.emitEvidence.title", + description_key="visual.nodes.emitEvidence.description", + config_validator=_validate_evidence, +) +def emit_evidence(ctx: Any, config: Mapping[str, Any], inputs: Mapping[str, object]) -> dict[str, object]: + ctx.emit_result({config.get("namespace", "visual_runner"): dict(inputs)}) + return {} + + +@visual_node( + kind="validate_command_environment", + group_key="contracts", + display_name="Validate command environment", + title_key="visual.nodes.validateCommandEnvironment.title", + description_key="visual.nodes.validateCommandEnvironment.description", + default_config={"command_env": "ORBIT_COMMAND"}, + required_config=("command_env",), + default_outputs=("command",), +) +def validate_command_environment( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Validate an external command exactly once before a cycle begins.""" + return {"command": ctx.command_from_env(config["command_env"])} + + +@visual_node( + kind="validate_build_fields", + group_key="contracts", + display_name="Validate build fields", + title_key="visual.nodes.validateBuildFields.title", + description_key="visual.nodes.validateBuildFields.description", + default_config={"fields": ["browser_base_url"]}, + config_validator=_validate_fields, + default_outputs=("contract",), +) +def validate_build_fields(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Require named build fields without exposing arbitrary context access.""" + missing = [field for field in config["fields"] if not ctx.build.get(field)] + if missing: + raise ValueError(f"Set required build field(s): {', '.join(missing)}") + return {"contract": True} + + +@visual_node( + kind="log_message", + group_key="evidence", + display_name="Log message", + title_key="visual.nodes.logMessage.title", + description_key="visual.nodes.logMessage.description", + default_config={"message": ""}, + required_config=("message",), +) +def log_message(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Publish one explicit lifecycle log message.""" + ctx.log(config["message"]) + return {} + + +def _emit_cycle_result(ctx: Any, config: Mapping[str, Any], result: object) -> dict[str, object]: + namespace = config.get("namespace", "visual_runner") + result_key = config.get("result_key", "result") + evidence = {str(result_key): result} + if config.get("include_iteration", True): + evidence = {"iteration": ctx.loop_index, **evidence} + ctx.emit_result({str(namespace): evidence}) + return {"result": result, "evidence": evidence} + + +@visual_node( + kind="command_cycle_action", + group_key="actions", + display_name="Command cycle action", + title_key="visual.nodes.commandCycleAction.title", + description_key="visual.nodes.commandCycleAction.description", + default_config={ + "command_env": "ORBIT_COMMAND", + "action": "run", + "namespace": "external_adapter", + "result_key": "result", + }, + required_config=("command_env", "action", "namespace", "result_key"), + default_outputs=("result", "evidence"), +) +def command_cycle_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Run a command action and emit the standard per-iteration evidence envelope.""" + result = ctx.run_command_action( + command_env=config["command_env"], + action=config["action"], + timeout=config.get("timeout"), + log_source=config.get("log_source"), + ) + return _emit_cycle_result(ctx, config, result) + + +@visual_node( + kind="json_cycle_action", + group_key="actions", + display_name="JSON cycle action", + title_key="visual.nodes.jsonCycleAction.title", + description_key="visual.nodes.jsonCycleAction.description", + default_config={ + "command_env": "ORBIT_AGENT_COMMAND", + "action": "run-once", + "namespace": "agent_cycle", + "result_key": "result", + "input_data": {}, + }, + required_config=("command_env", "action", "namespace", "result_key"), + default_outputs=("result", "evidence"), +) +def json_cycle_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Run a JSON evaluator action and emit the standard per-iteration evidence envelope.""" + result = ctx.run_json_action( + command_env=config["command_env"], + action=config["action"], + input_env=config.get("input_env", "ORBIT_CYCLE_INPUT"), + input_data=config.get("input_data", {}), + timeout=config.get("timeout"), + log_source=config.get("log_source"), + ) + return _emit_cycle_result(ctx, config, result) diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.agent-self-improvement/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.agent-self-improvement/runner.py new file mode 100644 index 0000000..6c05707 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.agent-self-improvement/runner.py @@ -0,0 +1,239 @@ +"""Create an evidence-backed agent self-improvement in an isolated worktree. + +Requires a Git repository, fixed target-AI cases, a model profile, and a +readable managed prompt. The coding agent's diff is retained for review and +never applied automatically to the source repository. +""" + +import hashlib +import json + +from orbit_sdk import graph, runner + +AGENT_PROVIDER = "${agent_provider}" +AGENT_OPTIONS = "${agent_options}" + +graph.connect("validate-target", "validate-evaluation-inputs") +graph.connect("validate-evaluation-inputs", "prepare-prompt") +graph.connect("prepare-prompt", "exercise-target", label="managed prompt") +graph.connect("exercise-target", "retain-iteration", kind="data", label="responses") +graph.connect("retain-iteration", "propose-agent-change", label="supervisor assessment") +graph.connect("propose-agent-change", "prepare-prompt", kind="loop", label="next evaluation") + + +def git(ctx, *args): + """Run Git in the configured project root without invoking a shell. + + Args: + ctx: The active Orbit runner context. + *args: Git arguments excluding the executable name. + + Returns: + Standard output from the Git command. + """ + return ctx.exec(["git", *args], cwd=ctx.project_root, timeout=300) + + +def managed_prompt_evidence(ctx): + """Read and fingerprint the prompt that the agent will improve. + + Args: + ctx: The active Orbit runner context. + + Returns: + The path, content, and SHA-256 fingerprint of the managed prompt. + + Raises: + ValueError: If no managed prompt path is configured. + """ + prompt_path = str(ctx.build.get("managed_prompt_path") or ctx.build.get("prompt_bundle") or "").strip() + if not prompt_path: + raise ValueError("native improvement cycle requires target_environment.managed_prompt_path") + content = ctx.project_path(prompt_path).read_text(encoding="utf-8") + return { + "path": prompt_path, + "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "content": content, + } + + +def agent_task(ctx): + """Build a bounded, autonomous objective for the coding agent. + + The task explicitly confines the agent's modifications to its isolated + proposal worktree; it never authorizes an automatic source-repository edit. + """ + issue = ctx.current_issue_assessment + return "\n".join( + ( + "You are the autonomous improvement agent for this repository.", + "Do not ask questions or wait for approval. First inspect the repository and the managed prompt, then make exactly one atomic, evidence-backed improvement, validate it, and finish the task completely.", + "You are working in an isolated proposal worktree. Make the complete change there; it will be captured as a reviewable diff and will not be applied to the source repository automatically. Do not commit, reset, or discard unrelated user changes.", + f"Managed prompt path: {ctx.build.get('managed_prompt_path') or ctx.build.get('prompt_bundle')}", + f"Build purpose: {ctx.build.get('purpose', '')}", + f"Fixed acceptance criteria: {json.dumps([case.get('acceptance', '') for case in ctx.test_cases], ensure_ascii=False)}", + f"Supervisor-assessed issue to resolve: {json.dumps(issue, ensure_ascii=False)}", + "If you completed meaningful feedback or a change, print one final line exactly in this format: ORBIT_AGENT_FEEDBACK: . If there is no meaningful feedback, do not print that marker.", + ) + ) + + +@graph.step( + "validate-target", title="Validate target repository", phase="before_all", outputs=["repository_target"] +) +@runner.phase("before_all", step_id="validate-target") +def before_all(ctx): + """Validate that the configured repository is a Git worktree.""" + # Repository validation runs once before the iteration loop begins. + git(ctx, "rev-parse", "--show-toplevel") + + +@graph.step( + "validate-evaluation-inputs", + title="Validate evaluation inputs", + phase="before_all", + inputs=["repository_target"], + outputs=["evaluation_contract"], +) +@runner.phase("before_all", step_id="validate-evaluation-inputs") +def validate_evaluation_inputs(ctx): + """Validate fixed target-AI cases and the selected model profile.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed target-AI prompt for a native improvement cycle") + if not isinstance(ctx.resource("model_profile", {}), dict) or not ctx.resource("model_profile", {}).get( + "model" + ): + raise ValueError("Select a configured model profile for a native improvement cycle") + ctx.log("Validated an OpenOrbit-native target-AI prompt improvement cycle") + + +@graph.step( + "prepare-prompt", + title="Read change target", + phase="before_each", + inputs=["evaluation_contract"], + outputs=["managed_prompt"], +) +@runner.phase("before_each", step_id="prepare-prompt") +def before_each(ctx): + """Expose the current prompt as read-only evidence for this iteration.""" + ctx.emit_result( + { + "improvement_cycle": { + "iteration": ctx.loop_index, + "proposal_mode": "isolated_worktree", + "managed_prompt": managed_prompt_evidence(ctx), + } + } + ) + + +@graph.step( + "exercise-target", + title="Exercise target AI", + phase="execute", + inputs=["managed_prompt"], + outputs=["target_responses"], +) +@runner.phase("execute", step_id="exercise-target") +def execute(ctx): + """Exercise the target AI and retain raw response evidence.""" + # Exercise the evaluated AI with the current managed prompt. The raw reply + # is retained as supervisor evidence instead of treating a browser page as + # proof that a prompt instruction was followed. + managed_prompt = managed_prompt_evidence(ctx) + responses = [] + for case in ctx.test_cases: + request = str(case.get("prompt") or "").strip() + if not request: + raise ValueError("each target-AI test case requires a prompt") + turn = ctx.complete_model( + "# Managed agent instructions\\n" + + managed_prompt["content"] + + "\\n\\n# User request\\n" + + request + + "\\n\\nRespond as the managed agent." + ) + responses.append( + { + "id": case.get("id"), + "name": case.get("name"), + "request": request, + "acceptance": str(case.get("acceptance") or ""), + "response": turn["response"], + "model": turn["model"], + } + ) + artifact = ctx.save_data_file( + "target-ai-responses.json", + json.dumps(responses, ensure_ascii=False, indent=2), + label="Target AI responses", + content_type="application/json", + ) + ctx.emit_result( + { + "improvement_cycle": { + "iteration": ctx.loop_index, + "evidence": {"target_ai_responses": responses, "artifact": artifact}, + } + } + ) + + +@graph.step( + "propose-agent-change", + title="Create agent proposal for assessed issue", + phase="after_each", + inputs=["iteration_snapshot"], + outputs=["agent_proposal"], + after_supervision=True, +) +@runner.phase("after_each", step_id="propose-agent-change") +def propose_agent_change(ctx): + """Create one isolated worktree proposal for a supervisor-assessed issue.""" + issue = ctx.current_issue_assessment + assessment = issue.get("evaluation") if isinstance(issue, dict) else None + decision = assessment.get("approval") if isinstance(assessment, dict) else None + if not issue: + ctx.emit_result({"agent_proposal": {"skipped": True, "reason": "no_reported_issue"}}) + return + if decision == "rejected": + ctx.emit_result({"agent_proposal": {"skipped": True, "reason": "issue_rejected", "issue": issue}}) + ctx.log("Skipped AI agent work because the supervisor rejected the issue") + return + agent = ctx.run_ai_agent(agent_task(ctx), provider=AGENT_PROVIDER, options=AGENT_OPTIONS) + proposal = agent.get("proposal", {}) + if agent["feedback"] and agent["changed_files"] and proposal.get("fingerprint"): + ctx.register_evaluation( + str(agent["feedback"]), + changed_files=[str(path) for path in agent["changed_files"]], + validation="Agent completed its autonomous repository task.", + improvement_fingerprint=str(proposal["fingerprint"]), + ) + ctx.emit_result( + { + "agent_proposal": { + "issue": issue, + "agent": {key: value for key, value in agent.items() if key != "output"}, + } + } + ) + ctx.log("Retained an isolated agent change proposal for review") + + +@graph.step( + "retain-iteration", + title="Retain assessment evidence", + phase="after_each", + inputs=["target_responses"], + outputs=["iteration_snapshot"], +) +@runner.phase("after_each", step_id="retain-iteration") +def after_each(ctx): + """Keep target-AI evidence available after supervisor assessment.""" + # Per-iteration evidence remains available for supervisor review. + ctx.log("Retained target-AI responses and supervisor assessment evidence") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-experience-improvement/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-experience-improvement/runner.py new file mode 100644 index 0000000..783cf88 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-experience-improvement/runner.py @@ -0,0 +1,121 @@ +"""Run the configured external AI evaluator through explicit runner phases. + +The evaluator receives fixed test cases through ``ORBIT_CYCLE_INPUT`` and must +return one JSON object per action. Orbit retains that structured evidence for +supervision; this runner never infers an evaluation result itself. +""" + +from orbit_sdk import graph, runner + + +def invoke(ctx, action): + """Run one external evaluator action through the SDK command contract.""" + return ctx.run_json_action( + command_env="ORBIT_AGENT_COMMAND", + action=action, + input_env="ORBIT_CYCLE_INPUT", + input_data={"iteration": ctx.loop_index, "build": ctx.build, "test_cases": ctx.test_cases}, + timeout=3600, + log_source="agent-cycle", + ) + + +graph.connect("validate-agent-cycle-contract", "preflight-agent-cycle") +graph.connect("preflight-agent-cycle", "prepare-agent-cycle", label="prepared inputs") +graph.connect("prepare-agent-cycle", "run-agent-cycle", kind="data", label="action result") +graph.connect("run-agent-cycle", "collect-agent-cycle") +graph.connect("collect-agent-cycle", "close-agent-cycle") +graph.connect("close-agent-cycle", "prepare-agent-cycle", kind="loop", label="next cycle") +graph.connect("close-agent-cycle", "finalize-agent-cycle", kind="condition", label="completed") + + +@graph.step( + "validate-agent-cycle-contract", + title="Validate external agent contract", + phase="before_all", + outputs=["contract"], +) +@runner.phase("before_all", step_id="validate-agent-cycle-contract") +def validate_contract(ctx): + """Require at least one fixed AI experience test case.""" + ctx.require_test_cases() + + +@graph.step( + "preflight-agent-cycle", + title="Check external agent readiness", + phase="before_all", + inputs=["contract"], + outputs=["preflight"], +) +@runner.phase("before_all", step_id="preflight-agent-cycle") +def preflight(ctx): + """Capture the external evaluator's readiness.""" + ctx.emit_result({"agent_cycle": {"preflight": invoke(ctx, "status")}}) + + +@graph.step( + "prepare-agent-cycle", + title="Prepare external agent", + phase="before_each", + inputs=["preflight"], + outputs=["prepared"], +) +@runner.phase("before_each", step_id="prepare-agent-cycle") +def prepare(ctx): + """Prepare the evaluator for the current iteration.""" + ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "prepared": invoke(ctx, "prepare")}}) + + +@graph.step( + "run-agent-cycle", title="Run external agent", phase="execute", inputs=["prepared"], outputs=["result"] +) +@runner.phase("execute", step_id="run-agent-cycle") +def execute(ctx): + """Run the external AI experience evaluation once.""" + ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "result": invoke(ctx, "run-once")}}) + + +@graph.step( + "collect-agent-cycle", + title="Collect external agent evidence", + phase="verify", + inputs=["result"], + outputs=["evidence"], +) +@runner.phase("verify", step_id="collect-agent-cycle") +def verify(ctx): + """Collect structured evidence for supervision.""" + ctx.emit_result( + {"agent_cycle": {"iteration": ctx.loop_index, "evidence": invoke(ctx, "collect-evidence")}} + ) + + +@graph.step( + "close-agent-cycle", + title="Close external agent cycle", + phase="after_each", + inputs=["evidence"], + outputs=["complete"], +) +@runner.phase("after_each", step_id="close-agent-cycle") +def after_each(ctx): + """Close one bounded AI experience iteration.""" + ctx.log("Completed one bounded external agent cycle") + + +@graph.step( + "finalize-agent-cycle", + title="Finalize external agent", + phase="after_all", + inputs=["complete"], + outputs=["final"], +) +@runner.phase("after_all", step_id="finalize-agent-cycle") +def after_all(ctx): + """Finalize the external evaluator run.""" + ctx.log("Finalized the external agent") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-slo-drift-monitor/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-slo-drift-monitor/runner.py new file mode 100644 index 0000000..d6eec5c --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.ai-slo-drift-monitor/runner.py @@ -0,0 +1,122 @@ +"""Run the configured AI SLO evaluator through explicit runner phases. + +The external evaluator owns probe execution. This runner passes the fixed probe +matrix, validates its JSON action responses, and retains evidence for Orbit's +supervisor to assess quality, safety, latency, or cost drift. +""" + +from orbit_sdk import graph, runner + +COMMAND_ENV = "ORBIT_PROBE_COMMAND" +NAMESPACE = "probe_gate" + + +def invoke(ctx, action): + """Run one evaluator action through the SDK command contract.""" + return ctx.run_json_action( + command_env=COMMAND_ENV, + action=action, + input_env="ORBIT_CYCLE_INPUT", + input_data={"iteration": ctx.loop_index, "build": ctx.build, "probes": ctx.test_cases}, + timeout=3600, + log_source="probe-gate", + ) + + +graph.connect("validate-probe-gate-contract", "preflight-probe-gate") +graph.connect("preflight-probe-gate", "prepare-probe-gate", label="prepared inputs") +graph.connect("prepare-probe-gate", "run-probe-gate", kind="data", label="action result") +graph.connect("run-probe-gate", "collect-probe-gate") +graph.connect("collect-probe-gate", "close-probe-gate") +graph.connect("close-probe-gate", "prepare-probe-gate", kind="loop", label="next cycle") +graph.connect("close-probe-gate", "finalize-probe-gate", kind="condition", label="completed") + + +@graph.step( + "validate-probe-gate-contract", + title="Validate probe matrix contract", + phase="before_all", + outputs=["contract"], +) +@runner.phase("before_all", step_id="validate-probe-gate-contract") +def validate_contract(ctx): + """Require at least one fixed SLO probe.""" + ctx.require_test_cases(label="fixed probe case") + + +@graph.step( + "preflight-probe-gate", + title="Check probe matrix readiness", + phase="before_all", + inputs=["contract"], + outputs=["preflight"], +) +@runner.phase("before_all", step_id="preflight-probe-gate") +def preflight(ctx): + """Capture evaluator readiness before iterative probes begin.""" + ctx.emit_result({NAMESPACE: {"preflight": invoke(ctx, "preflight")}}) + + +@graph.step( + "prepare-probe-gate", + title="Prepare probe matrix", + phase="before_each", + inputs=["preflight"], + outputs=["prepared"], +) +@runner.phase("before_each", step_id="prepare-probe-gate") +def prepare(ctx): + """Prepare the evaluator for this SLO iteration.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "prepared": invoke(ctx, "prepare")}}) + + +@graph.step( + "run-probe-gate", title="Run probe matrix", phase="execute", inputs=["prepared"], outputs=["result"] +) +@runner.phase("execute", step_id="run-probe-gate") +def execute(ctx): + """Run the fixed SLO probe matrix.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "result": invoke(ctx, "run-probes")}}) + + +@graph.step( + "collect-probe-gate", + title="Collect probe matrix evidence", + phase="verify", + inputs=["result"], + outputs=["evidence"], +) +@runner.phase("verify", step_id="collect-probe-gate") +def verify(ctx): + """Collect evidence for supervisor assessment.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "evidence": invoke(ctx, "collect-evidence")}}) + + +@graph.step( + "close-probe-gate", + title="Close probe matrix cycle", + phase="after_each", + inputs=["evidence"], + outputs=["complete"], +) +@runner.phase("after_each", step_id="close-probe-gate") +def after_each(ctx): + """Close one bounded SLO monitoring iteration.""" + ctx.log("Completed one bounded probe matrix cycle") + + +@graph.step( + "finalize-probe-gate", + title="Finalize probe matrix", + phase="after_all", + inputs=["complete"], + outputs=["final"], +) +@runner.phase("after_all", step_id="finalize-probe-gate") +def after_all(ctx): + """Finalize the SLO monitoring run.""" + ctx.log("Finalized the probe matrix") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.continuous-user-journey/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.continuous-user-journey/runner.py new file mode 100644 index 0000000..5f5b8f4 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.continuous-user-journey/runner.py @@ -0,0 +1,340 @@ +"""Run a safe, stateful persona journey with one browser action per iteration. + +The persona model chooses only from rendered, same-origin, non-destructive +actions. Persistent state records the observed evidence and next intent without +changing the evaluated product. +""" + +import json +import re +from pathlib import Path + +import orbit_sdk +from orbit_sdk import graph, runner + +BLOCKED = re.compile( + r"logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe", re.I +) +MAX_HISTORY, MAX_LEARNINGS, MAX_ISSUES = 24, 12, 8 + +graph.connect("validate-browser-runtime", "validate") +graph.connect("validate", "observe") +graph.connect("observe", "decide", kind="data", label="rendered choices") +graph.connect("decide", "act", kind="data", label="one safe action") +graph.connect("act", "reflect", kind="data", label="action evidence") +graph.connect("reflect", "observe", kind="loop", label="next visit") +graph.connect("reflect", "finalize", kind="condition", label="completed") + + +def state_path(ctx): + """Return the build-scoped path for persistent persona state. + + Args: + ctx: The active Orbit runner context. + + Returns: + A path outside the evaluated repository. + """ + slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(ctx.build.get("id") or "persona")) + directory = ctx.app_data / "autonomous-personas" + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{slug}.json" + + +def load_state(ctx): + """Load persisted persona state or initialize it from the first case.""" + if state_path(ctx).exists(): + return json.loads(state_path(ctx).read_text(encoding="utf-8")) + case = ctx.test_cases[0] + return { + "persona": str(case.get("name") or "A careful product user"), + "current_goal": str(case.get("prompt") or "Understand the product through safe visits."), + "feeling": "curious", + "next_intent": "Orient myself on the first visible page.", + "learnings": [], + "reported_issues": [], + "history": [], + } + + +def save_state(ctx, state): + """Persist only bounded persona history to keep recurring runs compact.""" + state["learnings"] = list(state.get("learnings", []))[-MAX_LEARNINGS:] + state["reported_issues"] = list(state.get("reported_issues", []))[-MAX_ISSUES:] + state["history"] = list(state.get("history", []))[-MAX_HISTORY:] + state_path(ctx).write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8") + + +def model_json(ctx, prompt): + """Request one JSON decision through the SDK's model response contract.""" + return ctx.complete_model_json(prompt, description="persona model response") + + +def browser(ctx, *, url, screenshot, allowed_href=None): + """Capture rendered browser evidence for one pre-approved safe action.""" + """Observe or follow exactly one pre-approved same-origin, non-destructive link.""" + module = str(Path(orbit_sdk.__file__).resolve().parents[2] / "frontend" / "node_modules" / "playwright") + artifacts = ( + ctx.app_data / "artifacts" / ctx.environment.get("ORBIT_RUN_ID", "manual") / f"loop-{ctx.loop_index}" + ) + artifacts.mkdir(parents=True, exist_ok=True) + executable = str(ctx.build.get("browser_executable_path") or "").strip() + payload = { + "url": url, + "screenshot": str(artifacts / screenshot), + "allowedHref": allowed_href, + "executablePath": executable, + } + script = r"""const { chromium } = require(process.argv[1]); const input = JSON.parse(process.argv[2]); +const blocked = /(logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe)/i; +(async()=>{const options={headless:true};if(input.executablePath)options.executablePath=input.executablePath;const browser=await chromium.launch(options);const page=await browser.newPage();try{ + await page.goto(input.url,{waitUntil:'domcontentloaded',timeout:30000}); const before=page.url(); + const actions=await page.locator('a[href]').evaluateAll(items=>items.map((item,index)=>({id:`link-${index}`,href:item.href,label:(item.textContent||'').trim().replace(/\s+/g,' ').slice(0,140)})).filter(item=>item.href)); + const safe=actions.filter(item=>{try{const u=new URL(item.href), origin=new URL(before).origin;return u.origin===origin&&!blocked.test(u.pathname+' '+item.label)}catch{return false}}).slice(0,30); + let acted=false;if(input.allowedHref){const candidate=safe.find(item=>item.href===input.allowedHref);if(!candidate)throw new Error('planned action is no longer an allowed visible link');await page.goto(candidate.href,{waitUntil:'domcontentloaded',timeout:30000});acted=true;} + const text=(await page.locator('body').innerText().catch(()=>'' )).replace(/\s+/g,' ').slice(0,1800);await page.screenshot({path:input.screenshot,fullPage:true}); + console.log(`browser navigation completed: ${before} -> ${page.url()}`);console.log('__ORBIT_PERSONA_RESULT__'+JSON.stringify({before_url:before,url:page.url(),title:await page.title(),visible_text:text,available_actions:safe,acted,screenshot:input.screenshot})); + }finally{await browser.close()}})().catch(error=>{console.error(error);process.exit(1)});""" + output = ctx.exec( + ["node", "-e", script, module, json.dumps(payload)], + timeout=120, + target_log_source="browser-persona", + target_log_exclude_prefixes=("__ORBIT_PERSONA_RESULT__",), + ) + result = next( + ( + line.removeprefix("__ORBIT_PERSONA_RESULT__") + for line in reversed(output.splitlines()) + if line.startswith("__ORBIT_PERSONA_RESULT__") + ), + "", + ) + if not result: + raise RuntimeError("browser persona did not return structured evidence") + return json.loads(result) + + +@graph.step( + "validate-browser-runtime", + title="Validate persona browser runtime", + phase="before_all", + outputs=["browser_target"], +) +@runner.phase("before_all", step_id="validate-browser-runtime") +def validate_browser_runtime(ctx): + """Validate the browser base URL before the persona journey starts.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("An autonomous persona needs a browser base URL") + + +@graph.step( + "validate", + title="Validate autonomous persona contract", + phase="before_all", + inputs=["browser_target"], + outputs=["persona_contract"], +) +@runner.phase("before_all", step_id="validate") +def before_all(ctx): + """Require one fixed persona journey case.""" + if not ctx.test_cases: + raise ValueError("An autonomous persona needs one persona contract") + ctx.log("Validated the autonomous persona contract and safe browser boundary") + + +@graph.step( + "observe", + title="Observe current page", + phase="before_each", + inputs=["persona_contract"], + outputs=["page_choices"], +) +@runner.phase("before_each", step_id="observe") +def before_each(ctx): + """Observe the current page and persist its safe action choices.""" + state = load_state(ctx) + url = str(state.get("last_url") or ctx.build["browser_base_url"]) + observation = browser(ctx, url=url, screenshot=f"persona-observation-{ctx.loop_index}.png") + state["observation"] = observation + save_state(ctx, state) + ctx.emit_result( + { + "persona": { + "iteration": ctx.loop_index, + "state": { + key: state[key] + for key in ( + "persona", + "current_goal", + "feeling", + "next_intent", + "learnings", + "reported_issues", + ) + }, + "observation": observation, + } + } + ) + ctx.log(f"Observed {len(observation['available_actions'])} safe visible action(s)") + + +@graph.step( + "decide", + title="Plan next persona action", + phase="execute", + inputs=["page_choices"], + outputs=["persona_plan"], +) +@runner.phase("execute", step_id="decide") +def execute(ctx): + """Ask the persona model to choose one bounded next action.""" + state = load_state(ctx) + actions = state["observation"]["available_actions"] + prompt = ( + """You are a persistent product user. Decide one next action, not a test verdict. Return JSON only: +{"intent":"short first-person purpose","action_id":"one listed id or empty","rationale":"observable reason","expected_signal":"what would change my understanding"}. +You may choose an empty action_id to observe again. Never choose actions outside listed IDs. Avoid repeating a reported issue unless new evidence exists. + +Persona state:\n""" + + json.dumps( + { + key: state[key] + for key in ( + "persona", + "current_goal", + "feeling", + "next_intent", + "learnings", + "reported_issues", + "history", + ) + }, + ensure_ascii=False, + ) + + "\nVisible safe actions:\n" + + json.dumps(actions, ensure_ascii=False) + ) + plan = model_json(ctx, prompt) + action_id = str(plan.get("action_id") or "") + selected = next((item for item in actions if item["id"] == action_id), None) + if action_id and selected is None: + raise ValueError("persona selected an action outside the visible safe action list") + state["plan"] = { + "intent": str(plan.get("intent") or state["next_intent"]), + "rationale": str(plan.get("rationale") or ""), + "expected_signal": str(plan.get("expected_signal") or ""), + "action": selected, + } + save_state(ctx, state) + ctx.emit_result({"persona": {"iteration": ctx.loop_index, "plan": state["plan"]}}) + ctx.log("Planned one persona action from rendered, safe choices") + + +@graph.step( + "act", + title="Take one safe persona action", + phase="verify", + inputs=["persona_plan"], + outputs=["action_evidence"], +) +@runner.phase("verify", step_id="act") +def verify(ctx): + """Execute the selected safe action and retain rendered evidence.""" + state = load_state(ctx) + action = state["plan"].get("action") + evidence = browser( + ctx, + url=state["observation"]["before_url"], + allowed_href=action["href"] if action else None, + screenshot=f"persona-action-{ctx.loop_index}.png", + ) + state["evidence"] = evidence + save_state(ctx, state) + ctx.emit_result({"persona": {"iteration": ctx.loop_index, "action_evidence": evidence}}) + ctx.log("Completed one bounded persona action with rendered evidence") + + +@graph.step( + "reflect", + title="Reflect persona session", + phase="after_each", + inputs=["action_evidence"], + outputs=["persona_handoff"], +) +@runner.phase("after_each", step_id="reflect") +def after_each(ctx): + """Reflect on the action and persist the persona's next intent.""" + state = load_state(ctx) + prompt = """You are reflecting as a persistent product user after one safe browser action. Return JSON only: +{"feeling":"brief feeling","learning":"specific observation","issue":{"title":"short or empty","evidence":"observable evidence","severity":"low|medium|high"},"next_intent":"one concrete next action to investigate"}. +Do not repeat a reported issue title unless the new evidence materially differs. + +Current state:\n""" + json.dumps(state, ensure_ascii=False) + reflection = model_json(ctx, prompt) + issue = reflection.get("issue") if isinstance(reflection.get("issue"), dict) else {} + title = str(issue.get("title") or "").strip() + known = {str(item.get("title", "")).casefold() for item in state["reported_issues"]} + if title and title.casefold() not in known: + state["reported_issues"].append( + { + "title": title, + "evidence": str(issue.get("evidence") or ""), + "severity": str(issue.get("severity") or "low"), + "iteration": ctx.loop_index, + } + ) + learning = str(reflection.get("learning") or "").strip() + if learning and learning not in state["learnings"]: + state["learnings"].append(learning) + state["feeling"] = str(reflection.get("feeling") or state["feeling"]) + state["next_intent"] = str(reflection.get("next_intent") or state["plan"]["intent"]) + state["last_url"] = state["evidence"]["url"] + state["history"].append( + { + "iteration": ctx.loop_index, + "intent": state["plan"]["intent"], + "action": state["plan"].get("action"), + "feeling": state["feeling"], + "learning": learning, + "next_intent": state["next_intent"], + } + ) + save_state(ctx, state) + summary = {key: state[key] for key in ("feeling", "next_intent", "learnings", "reported_issues")} + ctx.emit_result( + {"persona": {"iteration": ctx.loop_index, "reflection": reflection, "next_state": summary}} + ) + ctx.save_data_file( + f"autonomous-persona/iteration-{ctx.loop_index}.json", + json.dumps( + { + "plan": state["plan"], + "evidence": state["evidence"], + "reflection": reflection, + "next_state": summary, + }, + ensure_ascii=False, + indent=2, + ), + label="Autonomous persona handoff", + content_type="application/json", + ) + ctx.log("Retained persona feeling, learning, deduplicated issues, and next intent") + + +@graph.step( + "finalize", + title="Finalize autonomous persona", + phase="after_all", + inputs=["persona_handoff"], + outputs=["final_status"], +) +@runner.phase("after_all", step_id="finalize") +def after_all(ctx): + """Finalize the autonomous persona journey.""" + ctx.log("Finalized the autonomous persona journey") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.critical-flow-proof/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.critical-flow-proof/runner.py new file mode 100644 index 0000000..0145fc6 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.critical-flow-proof/runner.py @@ -0,0 +1,207 @@ +"""Prove critical browser flows with an explicit recurring lifecycle. + +The runner retries failed fixed cases before rotating to the next case. Every +iteration retains rendered browser evidence and a bounded state handoff for +supervisor review. +""" + +import json + +from orbit_sdk import graph, runner + +NAMESPACE = "continuous_journey" + +graph.connect("validate-critical-flow-runtime", "validate-critical-flow") +graph.connect("validate-critical-flow", "load-critical-flow-state") +graph.connect("load-critical-flow-state", "plan-critical-flow") +graph.connect("plan-critical-flow", "run-critical-flow", label="focused cases") +graph.connect("run-critical-flow", "review-critical-flow", kind="data", label="browser evidence") +graph.connect("review-critical-flow", "retain-critical-flow") +graph.connect("retain-critical-flow", "plan-critical-flow", kind="loop", label="next iteration") +graph.connect("retain-critical-flow", "finalize-critical-flow", kind="condition", label="completed") + + +def state(ctx): + """Load state used to focus the next critical-flow visit. + + Args: + ctx: The active Orbit runner context. + + Returns: + Persisted critical-flow state, initialized on the first iteration. + """ + return ctx.load_state(NAMESPACE, {"next_case_index": 0, "failed_case_ids": [], "history": []}) + + +@graph.step( + "validate-critical-flow-runtime", + title="Validate critical flow runtime", + phase="before_all", + outputs=["browser_runtime"], +) +@runner.phase("before_all", step_id="validate-critical-flow-runtime") +def validate_runtime(ctx): + """Validate the configured browser target.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("Set required build field(s): browser_base_url") + ctx.log("Validated the browser runtime target") + + +@graph.step( + "validate-critical-flow", + title="Validate critical flow contract", + phase="before_all", + inputs=["browser_runtime"], + outputs=["journey_contract"], +) +@runner.phase("before_all", step_id="validate-critical-flow") +def validate_contract(ctx): + """Require a fixed critical-flow case set.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + ctx.log("Validated the critical flow contract") + + +@graph.step( + "load-critical-flow-state", + title="Load critical flow state", + phase="before_each", + inputs=["journey_contract"], + outputs=["journey_state"], +) +@runner.phase("before_each", step_id="load-critical-flow-state") +def load_state(ctx): + """Ensure a state document exists before planning.""" + ctx.save_state(NAMESPACE, state(ctx)) + + +@graph.step( + "plan-critical-flow", + title="Plan focused critical flow", + phase="before_each", + inputs=["journey_state"], + outputs=["journey_plan"], +) +@runner.phase("before_each", step_id="plan-critical-flow") +def plan(ctx): + """Retry failed flows before rotating through fixed flows.""" + current = state(ctx) + failed = bool(current.get("failed_case_ids")) + failed_ids = {str(case_id) for case_id in current.get("failed_case_ids", [])} + cases = [case for case in ctx.test_cases if str(case.get("id")) in failed_ids] + if not cases: + cases = [ctx.test_cases[int(current.get("next_case_index", 0)) % len(ctx.test_cases)]] + rules = [ + "Preserve observable evidence for every browser action.", + "Do not infer a result that the page did not expose.", + ] + if failed: + rules.insert(0, "Revisit previously failed journeys before exploring a new route.") + if ctx.previous_supervisor_feedback.get("reported_issues"): + rules.insert(0, "Prioritize the supervisor's previously reported issues.") + plan_data = { + "case_ids": [str(case.get("id")) for case in cases], + "reason": "Revisit previously failed journeys." + if failed + else "Rotate one fixed journey to retain bounded coverage.", + "rules": rules, + "supervisor_feedback": ctx.previous_supervisor_feedback, + } + current["plan"] = plan_data + ctx.save_state(NAMESPACE, current) + ctx.emit_result( + {NAMESPACE: {"iteration": ctx.loop_index, "case_count": len(ctx.test_cases), "plan": plan_data}} + ) + + +@graph.step( + "run-critical-flow", + title="Run critical flow", + phase="execute", + inputs=["journey_plan"], + outputs=["journey_evidence"], +) +@runner.phase("execute", step_id="run-critical-flow") +def execute(ctx): + """Run focused critical-flow cases and save their evidence.""" + current = state(ctx) + ids = set(current["plan"]["case_ids"]) + evidence = ctx.playwright_journey([case for case in ctx.test_cases if str(case.get("id")) in ids]) + results = list(evidence.get("results", [])) + current["failed_case_ids"] = [str(item.get("id")) for item in results if not item.get("passed")] + current["next_case_index"] = (int(current.get("next_case_index", 0)) + 1) % len(ctx.test_cases) + handoff = { + "iteration": ctx.loop_index, + "reason": current["plan"]["reason"], + "rules": current["plan"]["rules"], + "passed": len(results) - len(current["failed_case_ids"]), + "failed": len(current["failed_case_ids"]), + "failed_case_ids": current["failed_case_ids"], + } + current["history"] = [*current.get("history", []), handoff][-24:] + current["handoff"] = handoff + ctx.save_state(NAMESPACE, current) + artifact = ctx.save_data_file( + f"{NAMESPACE}/iteration-{ctx.loop_index}.json", + json.dumps( + {"plan": current["plan"], "evidence": evidence, "handoff": handoff}, ensure_ascii=False, indent=2 + ), + label="Critical flow iteration evidence", + content_type="application/json", + ) + ctx.emit_result( + { + NAMESPACE: { + "iteration": ctx.loop_index, + "plan": current["plan"], + "results": results, + "evidence": evidence, + "handoff": handoff, + "artifact": artifact, + } + } + ) + + +@graph.step( + "review-critical-flow", + title="Review critical flow evidence", + phase="verify", + inputs=["journey_evidence"], + outputs=["journey_handoff"], +) +@runner.phase("verify", step_id="review-critical-flow") +def verify(ctx): + """Publish the handoff used by the next iteration.""" + ctx.emit_result({NAMESPACE: {"next_iteration": state(ctx).get("handoff", {})}}) + ctx.log("Stored the journey summary and next-iteration handoff") + + +@graph.step( + "retain-critical-flow", + title="Retain critical flow result", + phase="after_each", + inputs=["journey_handoff"], + outputs=["iteration_complete"], +) +@runner.phase("after_each", step_id="retain-critical-flow") +def after_each(ctx): + """Close one bounded critical-flow iteration.""" + ctx.log("Closed this bounded critical flow") + + +@graph.step( + "finalize-critical-flow", + title="Finalize critical flow evaluation", + phase="after_all", + inputs=["iteration_complete"], + outputs=["final_status"], +) +@runner.phase("after_all", step_id="finalize-critical-flow") +def after_all(ctx): + """Finalize the critical-flow evaluation.""" + ctx.log("Finalized the critical flow evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.site-exploration-review/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.site-exploration-review/runner.py new file mode 100644 index 0000000..d99b004 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.site-exploration-review/runner.py @@ -0,0 +1,116 @@ +"""Explore a site safely and retain rendered product-review evidence. + +The embedded browser script follows only bounded same-origin links and excludes +destructive actions. Its structured output is evidence for supervision, not an +automatic product change. +""" + +import json +from pathlib import Path + +import orbit_sdk +from orbit_sdk import graph, runner + +graph.connect("validate-site", "explore-site") +graph.connect("explore-site", "review-evidence", kind="data", label="rendered pages") +graph.connect("review-evidence", "finalize-review") + + +@graph.step("validate-site", title="Validate site", phase="before_all", outputs=["site_target"]) +@runner.phase("before_all", step_id="validate-site") +def validate_site(ctx): + """Validate the browser target before safely exploring it.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("Set required build field(s): browser_base_url") + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + + +@graph.step( + "explore-site", + title="Explore rendered site", + phase="execute", + inputs=["site_target"], + outputs=["rendered_pages"], +) +@runner.phase("execute", step_id="explore-site") +def explore_site(ctx): + """Visit a bounded number of safe same-origin links. + + The embedded Playwright script excludes destructive paths and emits one + machine-readable evidence record after collecting rendered page content. + """ + module = str(Path(orbit_sdk.__file__).resolve().parents[2] / "frontend" / "node_modules" / "playwright") + screenshot = ( + ctx.app_data + / "artifacts" + / ctx.environment.get("ORBIT_RUN_ID", "manual") + / f"loop-{ctx.loop_index}" + / "site-exploration.png" + ) + screenshot.parent.mkdir(parents=True, exist_ok=True) + script = """const {chromium}=require(process.argv[1]),i=JSON.parse(process.argv[2]),blocked=/(logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe)/i;(async()=>{const b=await chromium.launch({headless:true}),p=await b.newPage(),v=[],seen=new Set(),o=new URL(i.baseUrl).origin;try{await p.goto(i.baseUrl,{waitUntil:'domcontentloaded',timeout:30000});for(let n=0;n<=i.maxClicks;n++){seen.add(p.url());v.push({url:p.url(),title:await p.title(),text:(await p.locator('body').innerText().catch(()=>'' )).replace(/\\s+/g,' ').slice(0,1200)});if(n===i.maxClicks)break;const a=await p.locator('a[href]').evaluateAll(x=>x.map((e,k)=>({k,href:e.href,text:(e.textContent||'').trim()})));const c=a.filter(x=>{try{const u=new URL(x.href);return u.origin===o&&!seen.has(u.href)&&!blocked.test(u.pathname+' '+x.text)}catch{return false}});if(!c.length)break;await p.locator('a[href]').nth(c[0].k).click({timeout:5000});await p.waitForLoadState('domcontentloaded',{timeout:10000}).catch(()=>{})}await p.screenshot({path:i.screenshot,fullPage:true});console.log('__ORBIT_SITE_EXPLORATION_RESULT__'+JSON.stringify({visited:v,screenshot:i.screenshot}))}finally{await b.close()}})().catch(e=>{console.error(e);process.exit(1)})""" + output = ctx.exec( + [ + "node", + "-e", + script, + module, + json.dumps( + {"baseUrl": ctx.build["browser_base_url"], "maxClicks": 3, "screenshot": str(screenshot)} + ), + ], + timeout=120, + target_log_source="site-exploration", + target_log_exclude_prefixes=("__ORBIT_SITE_EXPLORATION_RESULT__",), + ) + payload = next( + ( + line.removeprefix("__ORBIT_SITE_EXPLORATION_RESULT__") + for line in reversed(output.splitlines()) + if line.startswith("__ORBIT_SITE_EXPLORATION_RESULT__") + ), + "", + ) + if not payload: + raise RuntimeError("site exploration did not return structured evidence") + evidence = json.loads(payload) + titles = [str(page.get("title") or page.get("url")) for page in evidence.get("visited", [])] + ctx.emit_result( + { + "site_exploration": { + "evidence": evidence, + "opinion": f"Explored {len(titles)} rendered page(s): {'; '.join(titles[:3])}. Review the captured pages for clarity, usefulness, and friction.", + } + } + ) + + +@graph.step( + "review-evidence", + title="Review exploration evidence", + phase="verify", + inputs=["rendered_pages"], + outputs=["product_review"], +) +@runner.phase("verify", step_id="review-evidence") +def verify(ctx): + """Keep the rendered exploration evidence available to the supervisor.""" + ctx.log("Retained rendered exploration evidence for review") + + +@graph.step( + "finalize-review", + title="Finalize site review", + phase="after_all", + inputs=["product_review"], + outputs=["completed_review"], +) +@runner.phase("after_all", step_id="finalize-review") +def after_all(ctx): + """Finalize the bounded site exploration review.""" + ctx.log("Finalized the bounded site exploration review") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.user-journey-smoke-test/runner.py b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.user-journey-smoke-test/runner.py new file mode 100644 index 0000000..39fefb4 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/quick-starts/openorbit.user-journey-smoke-test/runner.py @@ -0,0 +1,85 @@ +"""Run fixed browser smoke tests through explicit lifecycle phases. + +The runner validates the declared browser target and journey cases, retains the +rendered evidence, and fails the run if any declared journey does not pass. +""" + +from orbit_sdk import graph, runner + +graph.connect("validate-browser-runtime", "validate-browser-journey") +graph.connect("validate-browser-journey", "run-browser-journey") +graph.connect("run-browser-journey", "verify-browser-evidence", kind="data", label="journey evidence") +graph.connect("verify-browser-evidence", "finalize-browser-evaluation") + + +@graph.step( + "validate-browser-runtime", + title="Validate browser runtime", + phase="before_all", + outputs=["browser_target"], +) +@runner.phase("before_all", step_id="validate-browser-runtime") +def validate_runtime(ctx): + """Validate the browser target before running the smoke test.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("Set required build field(s): browser_base_url") + + +@graph.step( + "validate-browser-journey", + title="Validate browser journey contract", + phase="before_all", + inputs=["browser_target"], + outputs=["journey_contract"], +) +@runner.phase("before_all", step_id="validate-browser-journey") +def validate_contract(ctx): + """Require fixed browser journey cases.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + + +@graph.step( + "run-browser-journey", + title="Run browser journey", + phase="execute", + inputs=["journey_contract"], + outputs=["journey_evidence"], +) +@runner.phase("execute", step_id="run-browser-journey") +def execute(ctx): + """Run the declared journeys and fail when any journey does not pass.""" + evidence = ctx.playwright_journey() + ctx.emit_result({"browser_smoke": {"iteration": ctx.loop_index, "evidence": evidence}}) + if not all(item.get("passed") for item in evidence.get("results", [])): + raise SystemExit("A browser journey failed") + + +@graph.step( + "verify-browser-evidence", + title="Verify journey evidence", + phase="verify", + inputs=["journey_evidence"], + outputs=["journey_verdict"], +) +@runner.phase("verify", step_id="verify-browser-evidence") +def verify(ctx): + """Publish the retained browser evidence for review.""" + ctx.log("Quick start browser evaluation completed") + + +@graph.step( + "finalize-browser-evaluation", + title="Finalize browser evaluation", + phase="after_all", + inputs=["journey_verdict"], + outputs=["completed_evaluation"], +) +@runner.phase("after_all", step_id="finalize-browser-evaluation") +def after_all(ctx): + """Record completion of the one-shot smoke evaluation.""" + ctx.log("Finalized the one-shot browser evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/runner-templates/autonomous-persona-journey.py b/backend/orbit_sdk/visual/canonical/runner-templates/autonomous-persona-journey/runner.py similarity index 93% rename from runner-templates/autonomous-persona-journey.py rename to backend/orbit_sdk/visual/canonical/runner-templates/autonomous-persona-journey/runner.py index bde34b5..ee28f6b 100644 --- a/runner-templates/autonomous-persona-journey.py +++ b/backend/orbit_sdk/visual/canonical/runner-templates/autonomous-persona-journey/runner.py @@ -2,6 +2,7 @@ import json import re +import subprocess from pathlib import Path import orbit_sdk @@ -17,6 +18,7 @@ graph.connect("decide", "act", kind="data", label="one safe action") graph.connect("act", "reflect", kind="data", label="action evidence") graph.connect("reflect", "observe", kind="loop", label="next visit") +graph.connect("reflect", "finalize-persona-journey", kind="condition", label="completed") def state_path(ctx): @@ -82,25 +84,18 @@ def browser(ctx, *, url, screenshot, allowed_href=None): const safe=actions.filter(item=>{try{const u=new URL(item.href), origin=new URL(before).origin;return u.origin===origin&&!blocked.test(u.pathname+' '+item.label)}catch{return false}}).slice(0,30); let acted=false;if(input.allowedHref){const candidate=safe.find(item=>item.href===input.allowedHref);if(!candidate)throw new Error('planned action is no longer an allowed visible link');await page.goto(candidate.href,{waitUntil:'domcontentloaded',timeout:30000});acted=true;} const text=(await page.locator('body').innerText().catch(()=>'' )).replace(/\s+/g,' ').slice(0,1800);await page.screenshot({path:input.screenshot,fullPage:true}); - console.log(`browser navigation completed: ${before} -> ${page.url()}`);console.log('__ORBIT_PERSONA_RESULT__'+JSON.stringify({before_url:before,url:page.url(),title:await page.title(),visible_text:text,available_actions:safe,acted,screenshot:input.screenshot})); + console.log(JSON.stringify({before_url:before,url:page.url(),title:await page.title(),visible_text:text,available_actions:safe,acted,screenshot:input.screenshot})); }finally{await browser.close()}})().catch(error=>{console.error(error);process.exit(1)});""" - output = ctx.exec( + result = subprocess.run( ["node", "-e", script, module, json.dumps(payload)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=120, - target_log_source="browser-persona", - target_log_exclude_prefixes=("__ORBIT_PERSONA_RESULT__",), ) - result = next( - ( - line.removeprefix("__ORBIT_PERSONA_RESULT__") - for line in reversed(output.splitlines()) - if line.startswith("__ORBIT_PERSONA_RESULT__") - ), - "", - ) - if not result: - raise RuntimeError("browser persona did not return structured evidence") - return json.loads(result) + if result.returncode: + raise RuntimeError(result.stdout[-4000:] or "browser persona action failed") + return json.loads(result.stdout.strip().splitlines()[-1]) @graph.step("validate", title="Validate autonomous persona", phase="before_all", outputs=["persona_contract"]) diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/evidence-gated-probe-cycle/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/evidence-gated-probe-cycle/runner.py new file mode 100644 index 0000000..3c58371 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/evidence-gated-probe-cycle/runner.py @@ -0,0 +1,122 @@ +"""Run an external structured evaluator through explicit runner phases. + +The command receives a fixed probe matrix in ``ORBIT_CYCLE_INPUT`` and returns +one JSON object for each action. This template validates and retains evidence; +the supervisor, not the runner, interprets the result. +""" + +from orbit_sdk import graph, runner + +COMMAND_ENV = "ORBIT_PROBE_COMMAND" +NAMESPACE = "probe_gate" + + +def invoke(ctx, action): + """Run one structured evaluator action through the SDK command contract.""" + return ctx.run_json_action( + command_env=COMMAND_ENV, + action=action, + input_env="ORBIT_CYCLE_INPUT", + input_data={"iteration": ctx.loop_index, "build": ctx.build, "probes": ctx.test_cases}, + timeout=3600, + log_source="probe-gate", + ) + + +graph.connect("validate-probe-gate-contract", "preflight-probe-gate") +graph.connect("preflight-probe-gate", "prepare-probe-gate", label="prepared inputs") +graph.connect("prepare-probe-gate", "run-probe-gate", kind="data", label="action result") +graph.connect("run-probe-gate", "collect-probe-gate") +graph.connect("collect-probe-gate", "close-probe-gate") +graph.connect("close-probe-gate", "prepare-probe-gate", kind="loop", label="next cycle") +graph.connect("close-probe-gate", "finalize-probe-gate", kind="condition", label="completed") + + +@graph.step( + "validate-probe-gate-contract", + title="Validate probe matrix contract", + phase="before_all", + outputs=["contract"], +) +@runner.phase("before_all", step_id="validate-probe-gate-contract") +def validate_contract(ctx): + """Require a fixed probe matrix before any external command runs.""" + ctx.require_test_cases(label="fixed probe case") + + +@graph.step( + "preflight-probe-gate", + title="Check probe matrix readiness", + phase="before_all", + inputs=["contract"], + outputs=["preflight"], +) +@runner.phase("before_all", step_id="preflight-probe-gate") +def preflight(ctx): + """Record evaluator readiness once for the complete run.""" + ctx.emit_result({NAMESPACE: {"preflight": invoke(ctx, "preflight")}}) + + +@graph.step( + "prepare-probe-gate", + title="Prepare probe matrix", + phase="before_each", + inputs=["preflight"], + outputs=["prepared"], +) +@runner.phase("before_each", step_id="prepare-probe-gate") +def prepare(ctx): + """Ask the evaluator to prepare inputs for this iteration.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "prepared": invoke(ctx, "prepare")}}) + + +@graph.step( + "run-probe-gate", title="Run probe matrix", phase="execute", inputs=["prepared"], outputs=["result"] +) +@runner.phase("execute", step_id="run-probe-gate") +def execute(ctx): + """Run the probe matrix and retain its structured result.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "result": invoke(ctx, "run-probes")}}) + + +@graph.step( + "collect-probe-gate", + title="Collect probe matrix evidence", + phase="verify", + inputs=["result"], + outputs=["evidence"], +) +@runner.phase("verify", step_id="collect-probe-gate") +def verify(ctx): + """Collect evidence that the supervisor can assess.""" + ctx.emit_result({NAMESPACE: {"iteration": ctx.loop_index, "evidence": invoke(ctx, "collect-evidence")}}) + + +@graph.step( + "close-probe-gate", + title="Close probe matrix cycle", + phase="after_each", + inputs=["evidence"], + outputs=["complete"], +) +@runner.phase("after_each", step_id="close-probe-gate") +def after_each(ctx): + """Mark the bounded probe iteration as complete.""" + ctx.log("Completed one bounded probe matrix cycle") + + +@graph.step( + "finalize-probe-gate", + title="Finalize probe matrix", + phase="after_all", + inputs=["complete"], + outputs=["final"], +) +@runner.phase("after_all", step_id="finalize-probe-gate") +def after_all(ctx): + """Record completion after the final probe iteration.""" + ctx.log("Finalized the probe matrix") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/external-command-adapter/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/external-command-adapter/runner.py new file mode 100644 index 0000000..e5fc970 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/external-command-adapter/runner.py @@ -0,0 +1,120 @@ +"""Run an external command adapter through explicit runner phases. + +The adapter may return ordinary text rather than JSON. Its lifecycle output is +retained under ``external_adapter`` so a supervisor can assess evidence without +the runner imposing product-specific semantics. +""" + +from orbit_sdk import graph, runner + + +def invoke(ctx, action): + """Run one external command action through the SDK command contract.""" + return ctx.run_command_action( + command_env="ORBIT_ADAPTER_COMMAND", action=action, timeout=3600, log_source="external-adapter" + ) + + +graph.connect("validate-adapter-contract", "check-adapter") +graph.connect("check-adapter", "prepare-adapter", label="prepared target") +graph.connect("prepare-adapter", "run-adapter", kind="data", label="adapter output") +graph.connect("run-adapter", "collect-adapter-evidence") +graph.connect("collect-adapter-evidence", "close-adapter-cycle") +graph.connect("close-adapter-cycle", "prepare-adapter", kind="loop", label="next cycle") +graph.connect("close-adapter-cycle", "finalize-adapter", kind="condition", label="completed") + + +@graph.step( + "validate-adapter-contract", + title="Validate adapter command", + phase="before_all", + outputs=["adapter_contract"], +) +@runner.phase("before_all", step_id="validate-adapter-contract") +def validate_contract(ctx): + """Validate the adapter command before invoking it.""" + ctx.command_from_env("ORBIT_ADAPTER_COMMAND") + + +@graph.step( + "check-adapter", + title="Check adapter readiness", + phase="before_all", + inputs=["adapter_contract"], + outputs=["adapter_status"], +) +@runner.phase("before_all", step_id="check-adapter") +def check_adapter(ctx): + """Record adapter readiness once for the run.""" + ctx.emit_result({"external_adapter": {"status": invoke(ctx, "status"), "iteration": ctx.loop_index}}) + + +@graph.step( + "prepare-adapter", + title="Prepare adapter cycle", + phase="before_each", + inputs=["adapter_status"], + outputs=["prepared_target"], +) +@runner.phase("before_each", step_id="prepare-adapter") +def prepare(ctx): + """Prepare the external adapter for this iteration.""" + ctx.emit_result({"external_adapter": {"prepared": invoke(ctx, "prepare"), "iteration": ctx.loop_index}}) + + +@graph.step( + "run-adapter", + title="Run bounded adapter task", + phase="execute", + inputs=["prepared_target"], + outputs=["adapter_result"], +) +@runner.phase("execute", step_id="run-adapter") +def execute(ctx): + """Execute the adapter's bounded unit of work.""" + ctx.emit_result({"external_adapter": {"result": invoke(ctx, "run-once"), "iteration": ctx.loop_index}}) + + +@graph.step( + "collect-adapter-evidence", + title="Collect adapter evidence", + phase="verify", + inputs=["adapter_result"], + outputs=["adapter_evidence"], +) +@runner.phase("verify", step_id="collect-adapter-evidence") +def verify(ctx): + """Collect adapter evidence for supervisor review.""" + ctx.emit_result( + {"external_adapter": {"evidence": invoke(ctx, "collect-evidence"), "iteration": ctx.loop_index}} + ) + + +@graph.step( + "close-adapter-cycle", + title="Close adapter cycle", + phase="after_each", + inputs=["adapter_evidence"], + outputs=["cycle_complete"], +) +@runner.phase("after_each", step_id="close-adapter-cycle") +def after_each(ctx): + """Close one adapter lifecycle iteration.""" + ctx.log("Completed one bounded external adapter cycle") + + +@graph.step( + "finalize-adapter", + title="Finalize external automation", + phase="after_all", + inputs=["cycle_complete"], + outputs=["final_status"], +) +@runner.phase("after_all", step_id="finalize-adapter") +def after_all(ctx): + """Finalize the external automation evaluation.""" + ctx.log("Finalized the external automation evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/json-agent-cycle/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/json-agent-cycle/runner.py new file mode 100644 index 0000000..9105141 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/json-agent-cycle/runner.py @@ -0,0 +1,121 @@ +"""Run an external JSON agent through explicit runner phases. + +The configured command receives fixed cases in ``ORBIT_CYCLE_INPUT`` and must +emit one JSON object for each action. Orbit preserves those responses as +evidence while keeping the lifecycle graph visible in this source file. +""" + +from orbit_sdk import graph, runner + + +def invoke(ctx, action): + """Run one external agent action through the SDK command contract.""" + return ctx.run_json_action( + command_env="ORBIT_AGENT_COMMAND", + action=action, + input_env="ORBIT_CYCLE_INPUT", + input_data={"iteration": ctx.loop_index, "build": ctx.build, "test_cases": ctx.test_cases}, + timeout=3600, + log_source="agent-cycle", + ) + + +graph.connect("validate-agent-cycle-contract", "preflight-agent-cycle") +graph.connect("preflight-agent-cycle", "prepare-agent-cycle", label="prepared inputs") +graph.connect("prepare-agent-cycle", "run-agent-cycle", kind="data", label="action result") +graph.connect("run-agent-cycle", "collect-agent-cycle") +graph.connect("collect-agent-cycle", "close-agent-cycle") +graph.connect("close-agent-cycle", "prepare-agent-cycle", kind="loop", label="next cycle") +graph.connect("close-agent-cycle", "finalize-agent-cycle", kind="condition", label="completed") + + +@graph.step( + "validate-agent-cycle-contract", + title="Validate external agent contract", + phase="before_all", + outputs=["contract"], +) +@runner.phase("before_all", step_id="validate-agent-cycle-contract") +def validate_contract(ctx): + """Require fixed test cases before starting the external agent.""" + ctx.require_test_cases() + + +@graph.step( + "preflight-agent-cycle", + title="Check external agent readiness", + phase="before_all", + inputs=["contract"], + outputs=["preflight"], +) +@runner.phase("before_all", step_id="preflight-agent-cycle") +def preflight(ctx): + """Capture the agent's one-time readiness result.""" + ctx.emit_result({"agent_cycle": {"preflight": invoke(ctx, "status")}}) + + +@graph.step( + "prepare-agent-cycle", + title="Prepare external agent", + phase="before_each", + inputs=["preflight"], + outputs=["prepared"], +) +@runner.phase("before_each", step_id="prepare-agent-cycle") +def prepare(ctx): + """Prepare the agent for the current bounded iteration.""" + ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "prepared": invoke(ctx, "prepare")}}) + + +@graph.step( + "run-agent-cycle", title="Run external agent", phase="execute", inputs=["prepared"], outputs=["result"] +) +@runner.phase("execute", step_id="run-agent-cycle") +def execute(ctx): + """Run the agent once and preserve its structured output.""" + ctx.emit_result({"agent_cycle": {"iteration": ctx.loop_index, "result": invoke(ctx, "run-once")}}) + + +@graph.step( + "collect-agent-cycle", + title="Collect external agent evidence", + phase="verify", + inputs=["result"], + outputs=["evidence"], +) +@runner.phase("verify", step_id="collect-agent-cycle") +def verify(ctx): + """Collect the agent's evidence payload for supervision.""" + ctx.emit_result( + {"agent_cycle": {"iteration": ctx.loop_index, "evidence": invoke(ctx, "collect-evidence")}} + ) + + +@graph.step( + "close-agent-cycle", + title="Close external agent cycle", + phase="after_each", + inputs=["evidence"], + outputs=["complete"], +) +@runner.phase("after_each", step_id="close-agent-cycle") +def after_each(ctx): + """Close one external-agent iteration.""" + ctx.log("Completed one bounded external agent cycle") + + +@graph.step( + "finalize-agent-cycle", + title="Finalize external agent", + phase="after_all", + inputs=["complete"], + outputs=["final"], +) +@runner.phase("after_all", step_id="finalize-agent-cycle") +def after_all(ctx): + """Finalize the external-agent evaluation.""" + ctx.log("Finalized the external agent") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/native-improvement-cycle/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/native-improvement-cycle/runner.py new file mode 100644 index 0000000..6e95a57 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/native-improvement-cycle/runner.py @@ -0,0 +1,350 @@ +"""Iteratively validate and reversibly improve a managed prompt in a Git repository. + +Requires fixed target-AI cases, a model profile, and a readable managed prompt. +Only accepted feedback is applied, every mutation has a rollback snapshot, and +the runner never creates a Git commit in the evaluated repository. +""" + +import hashlib +import json +import re + +from orbit_sdk import graph, runner + +REQUIRED_SUFFICIENT_EVALUATIONS = 3 + +graph.connect("validate-target", "validate-evaluation-inputs") +graph.connect("validate-evaluation-inputs", "snapshot-baseline") +graph.connect("snapshot-baseline", "prepare-prompt") +graph.connect("prepare-prompt", "exercise-target", label="managed prompt") +graph.connect("exercise-target", "assess-candidate", kind="data", label="responses") +graph.connect("assess-candidate", "retain-iteration") +graph.connect("retain-iteration", "prepare-prompt", kind="loop", label="next evaluation") +graph.connect("retain-iteration", "restore-baseline", kind="condition", label="completed") +# Marker comments make replacement idempotent and preserve the surrounding +# target prompt content that OpenOrbit does not own. +PROMPT_BLOCK_START = "" +PROMPT_BLOCK_END = "" + + +def state_path(ctx): + """Return the per-build state file outside the target repository. + + Args: + ctx: The active Orbit runner context. + + Returns: + The persistent state path for this build. + """ + build_id = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(ctx.build.get("id") or "manual")) + directory = ctx.app_data / "improvement-cycles" + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{build_id}.json" + + +def load_state(ctx): + """Load the previous verdict state, or start a fresh candidate baseline.""" + path = state_path(ctx) + if not path.exists(): + return {"candidate_fingerprint": None, "sufficient_evaluations": 0, "history": []} + return json.loads(path.read_text(encoding="utf-8")) + + +def save_state(ctx, state): + """Persist only bounded history so recurring evaluations do not grow unbounded.""" + state["history"] = state.get("history", [])[-24:] + state_path(ctx).write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8") + + +def git(ctx, *args): + """Run Git in the configured project root without invoking a shell. + + Args: + ctx: The active Orbit runner context. + *args: Git arguments excluding the executable name. + + Returns: + Standard output from the Git command. + """ + return ctx.exec(["git", *args], cwd=ctx.project_root, timeout=300) + + +def candidate(ctx): + """Fingerprint the current working-tree diff and retain its changed paths.""" + patch = git(ctx, "diff", "--binary", "--") + changed = [line for line in git(ctx, "diff", "--name-only").splitlines() if line] + return (hashlib.sha256(patch.encode("utf-8")).hexdigest() if patch else None), changed + + +def update_prompt_from_accepted_proposals(ctx, proposals): + """Replace only OpenOrbit's managed prompt block and retain a rollback version.""" + prompt_path = str(ctx.build.get("managed_prompt_path") or ctx.build.get("prompt_bundle") or "").strip() + if not prompt_path: + raise ValueError("native improvement cycle requires target_environment.managed_prompt_path") + target = ctx.project_path(prompt_path) + current = target.read_text(encoding="utf-8") + if not proposals: + # Do not manufacture a changing candidate when the supervisor has not + # accepted a change. A stable candidate must retain the same fingerprint + # across repeated validations before it can be promoted. + return {"path": prompt_path, "changed": False, "reason": "no_accepted_proposals"} + lines = ["## Accepted improvement proposals", "", f"Iteration: {ctx.loop_index}", ""] + for proposal in proposals: + lines.extend( + ( + f"### {proposal.get('title') or 'Accepted proposal'}", + str(proposal.get("rationale") or ""), + f"Acceptance evidence: {proposal.get('acceptanceEvidence') or ''}", + "", + ) + ) + block = "\n".join((PROMPT_BLOCK_START, "\n".join(lines).rstrip(), PROMPT_BLOCK_END)) + start, end = current.find(PROMPT_BLOCK_START), current.find(PROMPT_BLOCK_END) + if start >= 0 and end > start: + updated = current[:start] + block + current[end + len(PROMPT_BLOCK_END) :] + elif start >= 0 or end >= 0: + raise ValueError("prompt has an incomplete OpenOrbit accepted-proposals block") + else: + updated = current.rstrip() + "\n\n" + block + "\n" + return ctx.update_file(prompt_path, updated) + + +def managed_prompt_evidence(ctx): + """Expose the current managed prompt beside the target-AI response evidence.""" + prompt_path = str(ctx.build.get("managed_prompt_path") or ctx.build.get("prompt_bundle") or "").strip() + if not prompt_path: + raise ValueError("native improvement cycle requires target_environment.managed_prompt_path") + content = ctx.project_path(prompt_path).read_text(encoding="utf-8") + return { + "path": prompt_path, + "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "content": content, + } + + +@graph.step( + "validate-target", title="Validate target repository", phase="before_all", outputs=["repository_target"] +) +@runner.phase("before_all", step_id="validate-target") +def before_all(ctx): + """Validate that the configured target is a Git repository.""" + # Repository validation runs once before the iteration loop begins. + git(ctx, "rev-parse", "--show-toplevel") + + +@graph.step( + "validate-evaluation-inputs", + title="Validate evaluation inputs", + phase="before_all", + inputs=["repository_target"], + outputs=["evaluation_contract"], +) +@runner.phase("before_all", step_id="validate-evaluation-inputs") +def validate_evaluation_inputs(ctx): + """Validate fixed target-AI cases and the selected model profile.""" + """Validate the fixed evidence contract independently of the repository.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed target-AI prompt for a native improvement cycle") + if not isinstance(ctx.resource("model_profile", {}), dict) or not ctx.resource("model_profile", {}).get( + "model" + ): + raise ValueError("Select a configured model profile for a native improvement cycle") + ctx.log("Validated an OpenOrbit-native target-AI prompt improvement cycle") + + +@graph.step( + "snapshot-baseline", + title="Snapshot iteration baseline", + phase="before_each", + inputs=["evaluation_contract"], + outputs=["baseline_snapshot"], +) +@runner.phase("before_each", step_id="snapshot-baseline") +def snapshot_baseline(ctx): + """Save a reversible checkpoint before mutating the target prompt.""" + """Capture the reversible target state before preparing this iteration.""" + ctx.save_before_each_snapshot() + + +@graph.step( + "prepare-prompt", + title="Prepare prompt candidate", + phase="before_each", + inputs=["baseline_snapshot"], + outputs=["managed_prompt"], +) +@runner.phase("before_each", step_id="prepare-prompt") +def before_each(ctx): + """Apply eligible prompt feedback and emit the candidate evidence.""" + # Apply only feedback accepted by a human before the next validation. + feedback = ctx.previous_supervisor_feedback + accepted = [ + proposal + for proposal in feedback.get("improvements", []) + if isinstance(proposal, dict) and str(proposal.get("status") or "").lower() == "accepted" + ] + requires_human_approval = bool(ctx.build.get("require_human_approval_before_apply", False)) + if requires_human_approval and accepted: + # A supervisor's adoption is a recommendation, not an operator + # authorization. Keep it as evidence until an operator approves it. + prompt_update = { + "changed": False, + "reason": "awaiting_human_approval", + "proposal_count": len(accepted), + } + accepted = [] + else: + prompt_update = update_prompt_from_accepted_proposals(ctx, accepted) + accepted_ids = [str(value) for value in feedback.get("_orbit_proposal_ids", [])] + proposal_applications = ctx.record_proposal_application(accepted_ids, prompt_update) + fingerprint, changed = candidate(ctx) + ctx.emit_result( + { + "improvement_cycle": { + "iteration": ctx.loop_index, + "candidate_fingerprint": fingerprint, + "changed_paths": changed, + "prompt_update": prompt_update, + "requires_human_approval": requires_human_approval, + "managed_prompt": managed_prompt_evidence(ctx), + "proposal_applications": proposal_applications, + } + } + ) + ctx.log("Refreshed the rollback-protected prompt from accepted supervisor feedback") + + +@graph.step( + "exercise-target", + title="Exercise target AI", + phase="execute", + inputs=["managed_prompt"], + outputs=["target_responses"], +) +@runner.phase("execute", step_id="exercise-target") +def execute(ctx): + """Exercise the managed prompt with every fixed target-AI case.""" + # Exercise the evaluated AI with the current managed prompt. The raw reply + # is retained as supervisor evidence instead of treating a browser page as + # proof that a prompt instruction was followed. + managed_prompt = managed_prompt_evidence(ctx) + responses = [] + for case in ctx.test_cases: + request = str(case.get("prompt") or "").strip() + if not request: + raise ValueError("each target-AI test case requires a prompt") + turn = ctx.complete_model( + "# Managed agent instructions\\n" + + managed_prompt["content"] + + "\\n\\n# User request\\n" + + request + + "\\n\\nRespond as the managed agent." + ) + responses.append( + { + "id": case.get("id"), + "name": case.get("name"), + "request": request, + "acceptance": str(case.get("acceptance") or ""), + "response": turn["response"], + "model": turn["model"], + } + ) + artifact = ctx.save_data_file( + "target-ai-responses.json", + json.dumps(responses, ensure_ascii=False, indent=2), + label="Target AI responses", + content_type="application/json", + ) + fingerprint, changed = candidate(ctx) + ctx.emit_result( + { + "improvement_cycle": { + "iteration": ctx.loop_index, + "candidate_fingerprint": fingerprint, + "changed_paths": changed, + "evidence": {"target_ai_responses": responses, "artifact": artifact}, + } + } + ) + + +@graph.step( + "assess-candidate", + title="Assess candidate evidence", + phase="verify", + inputs=["target_responses"], + outputs=["candidate_verdict"], +) +@runner.phase("verify", step_id="assess-candidate") +def verify(ctx): + """Track whether the current candidate is stable enough for approval.""" + # Promote a candidate only after the required number of stable evaluations. + state = load_state(ctx) + fingerprint, changed = candidate(ctx) + if not fingerprint: + state["candidate_fingerprint"] = None + state["sufficient_evaluations"] = 0 + verdict = "no_candidate" + elif state.get("candidate_fingerprint") == fingerprint: + state["sufficient_evaluations"] = int(state.get("sufficient_evaluations", 0)) + 1 + verdict = ( + "ready_for_approval" + if state["sufficient_evaluations"] >= REQUIRED_SUFFICIENT_EVALUATIONS + else "continue_validation" + ) + else: + state["candidate_fingerprint"] = fingerprint + state["sufficient_evaluations"] = 1 + verdict = "continue_validation" + state.setdefault("history", []).append( + {"iteration": ctx.loop_index, "fingerprint": fingerprint, "paths": changed, "verdict": verdict} + ) + save_state(ctx, state) + ctx.emit_result( + { + "improvement_cycle": { + "candidate_fingerprint": fingerprint, + "changed_paths": changed, + "sufficient_evaluations": state["sufficient_evaluations"], + "required_evaluations": REQUIRED_SUFFICIENT_EVALUATIONS, + "verdict": verdict, + } + } + ) + ctx.log(f"Candidate verdict: {verdict}") + + +@graph.step( + "retain-iteration", + title="Retain iteration evidence", + phase="after_each", + inputs=["candidate_verdict"], + outputs=["iteration_snapshot"], +) +@runner.phase("after_each", step_id="retain-iteration") +def after_each(ctx): + """Retain a recovery checkpoint and iteration evidence.""" + # Preserve the first evaluated state as a named recovery checkpoint. + ctx.save_first_after_each_snapshot() + # Per-iteration evidence remains available for supervisor review. + ctx.log("Retained prompt versions, decisions, and validation evidence") + + +@graph.step( + "restore-baseline", + title="Restore baseline", + phase="after_all", + inputs=["iteration_snapshot"], + outputs=["restored_target"], +) +@runner.phase("after_all", step_id="restore-baseline") +def after_all(ctx): + """Restore the original target after the recurring evaluation completes.""" + # Return the target to its exact baseline without creating a Git commit. + ctx.restore_before_each_snapshot() + ctx.log("Restored the native improvement target without committing changes") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/playwright-continuous-journey/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/playwright-continuous-journey/runner.py new file mode 100644 index 0000000..8d3487e --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/playwright-continuous-journey/runner.py @@ -0,0 +1,151 @@ +"""A reference runner for recurring, evidence-led browser journeys.""" + +import json +import re + +from orbit_sdk import graph, runner + +graph.connect("validate", "plan") +graph.connect("plan", "exercise", kind="data", label="focused journey") +graph.connect("exercise", "retain") +graph.connect("retain", "close-journey-cycle") +graph.connect("close-journey-cycle", "plan", kind="loop", label="next iteration") +graph.connect("close-journey-cycle", "finalize-browser-journey", kind="condition", label="completed") + + +def state_path(ctx): + """Keep durable journey state in OpenOrbit AppData, never in the target repo.""" + build_id = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(ctx.build.get("id") or "journey")) + directory = ctx.app_data / "continuous-journeys" + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{build_id}.json" + + +def load_state(ctx): + path = state_path(ctx) + return ( + json.loads(path.read_text(encoding="utf-8")) + if path.exists() + else {"next_case": 0, "failed": [], "history": []} + ) + + +def save_state(ctx, state): + state["history"] = state.get("history", [])[-24:] + state_path(ctx).write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8") + + +@graph.step( + "validate", title="Validate recurring browser journey", phase="before_all", outputs=["journey_contract"] +) +@runner.phase("before_all") +def before_all(ctx): + # The SDK owns browser execution; callers supply only a URL and fixed cases. + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one journey case are required") + ctx.log("Validated the continuous Playwright journey contract") + + +@graph.step( + "plan", + title="Choose next user journey", + phase="before_each", + inputs=["journey_contract"], + outputs=["journey_plan"], +) +@runner.phase("before_each") +def before_each(ctx): + state = load_state(ctx) + # Failed paths are retried first; otherwise rotate to retain broad coverage. + focused = [case for case in ctx.test_cases if str(case.get("id")) in set(state["failed"])] + if not focused: + focused = [ctx.test_cases[int(state["next_case"]) % len(ctx.test_cases)]] + state["plan"] = { + "case_ids": [str(case.get("id")) for case in focused], + "prior_feedback": ctx.previous_supervisor_feedback, + } + save_state(ctx, state) + ctx.emit_result({"continuous_journey": {"iteration": ctx.loop_index, "plan": state["plan"]}}) + ctx.log(f"Planned journey case(s): {', '.join(state['plan']['case_ids'])}") + + +@graph.step( + "exercise", + title="Exercise rendered journey", + phase="execute", + inputs=["journey_plan"], + outputs=["browser_evidence"], +) +@runner.phase("execute") +def execute(ctx): + state = load_state(ctx) + chosen = set(state["plan"]["case_ids"]) + evidence = ctx.playwright_journey([case for case in ctx.test_cases if str(case.get("id")) in chosen]) + state["failed"] = [str(item.get("id")) for item in evidence["results"] if not item["passed"]] + state["next_case"] = int(state["next_case"]) + 1 + state["history"].append( + {"iteration": ctx.loop_index, "case_ids": state["plan"]["case_ids"], "failed": state["failed"]} + ) + save_state(ctx, state) + ctx.emit_result( + { + "continuous_journey": { + "iteration": ctx.loop_index, + "evidence": evidence, + "handoff": state["history"][-1], + } + } + ) + # Preserve a small, downloadable iteration summary alongside the richer + # Playwright evidence retained by the SDK. + ctx.save_data_file( + f"continuous-journey/iteration-{ctx.loop_index}.json", + json.dumps(state["history"][-1], ensure_ascii=False, indent=2), + label="Continuous journey handoff", + content_type="application/json", + ) + ctx.log(f"Executed {len(evidence['results'])} browser case(s); failed: {len(state['failed'])}") + + +@graph.step( + "retain", + title="Retain journey handoff", + phase="verify", + inputs=["browser_evidence"], + outputs=["next_iteration"], +) +@runner.phase("verify") +def verify(ctx): + state = load_state(ctx) + ctx.emit_result( + {"continuous_journey": {"next_iteration": state["history"][-1], "state_path": str(state_path(ctx))}} + ) + ctx.log("Retained browser evidence and the next-iteration handoff") + + +@graph.step( + "close-journey-cycle", + title="Close browser journey cycle", + phase="after_each", + inputs=["next_iteration"], + outputs=["cycle_complete"], +) +@runner.phase("after_each") +def after_each(ctx): + ctx.log("Completed one bounded continuous browser journey") + + +@graph.step( + "finalize-browser-journey", + title="Finalize browser journey", + phase="after_all", + inputs=["cycle_complete"], + outputs=["journey_complete"], +) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the continuous browser journey") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/selenium-external-journey/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/selenium-external-journey/runner.py new file mode 100644 index 0000000..100b15c --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/selenium-external-journey/runner.py @@ -0,0 +1,125 @@ +"""Reference adapter runner for teams that already operate Selenium journeys.""" + +import json +import os +import shlex + +from orbit_sdk import graph, runner + +graph.connect("ready", "prepare") +graph.connect("prepare", "run", kind="data", label="one Selenium cycle") +graph.connect("run", "collect") +graph.connect("collect", "close-selenium-cycle") +graph.connect("close-selenium-cycle", "prepare", kind="loop", label="next cycle") +graph.connect("close-selenium-cycle", "finalize-selenium-journey", kind="condition", label="completed") + + +def command(): + """Read an explicit adapter command without embedding target credentials.""" + configured = os.environ.get("ORBIT_SELENIUM_COMMAND", "").strip() + if not configured: + raise ValueError( + "Set ORBIT_SELENIUM_COMMAND to an adapter supporting status, prepare, run-once, and collect-evidence" + ) + value = json.loads(configured) if configured.startswith("[") else shlex.split(configured) + if not isinstance(value, list) or not value or not all(isinstance(item, str) for item in value): + raise ValueError("ORBIT_SELENIUM_COMMAND must resolve to a non-empty string command list") + return value + + +def invoke(ctx, action): + # One bounded call per phase: the adapter must never start its own daemon. + return ctx.exec( + [*command(), action], + cwd=ctx.project_root, + timeout=3600, + target_log_source="selenium-adapter", + ) + + +def evidence(value): + """Bound adapter output before it becomes retained OpenOrbit evidence.""" + text = str(value).strip() + return text[:4000] if text else "(adapter returned no output)" + + +@graph.step("ready", title="Check Selenium adapter", phase="before_all", outputs=["adapter_status"]) +@runner.phase("before_all") +def before_all(ctx): + status = evidence(invoke(ctx, "status")) + ctx.emit_result({"selenium_journey": {"status": status}}) + ctx.log("Validated the Selenium adapter contract") + + +@graph.step( + "prepare", + title="Prepare Selenium cycle", + phase="before_each", + inputs=["adapter_status"], + outputs=["prepared_cycle"], +) +@runner.phase("before_each") +def before_each(ctx): + prepared = evidence(invoke(ctx, "prepare")) + ctx.emit_result({"selenium_journey": {"iteration": ctx.loop_index, "prepared": prepared}}) + ctx.log("Prepared one Selenium adapter cycle") + + +@graph.step( + "run", + title="Run Selenium journey", + phase="execute", + inputs=["prepared_cycle"], + outputs=["journey_result"], +) +@runner.phase("execute") +def execute(ctx): + result = evidence(invoke(ctx, "run-once")) + ctx.emit_result({"selenium_journey": {"iteration": ctx.loop_index, "result": result}}) + + +@graph.step( + "collect", + title="Collect Selenium evidence", + phase="verify", + inputs=["journey_result"], + outputs=["journey_evidence"], +) +@runner.phase("verify") +def verify(ctx): + collected = evidence(invoke(ctx, "collect-evidence")) + ctx.emit_result({"selenium_journey": {"iteration": ctx.loop_index, "evidence": collected}}) + ctx.save_data_file( + f"selenium-journey/iteration-{ctx.loop_index}.txt", + collected, + label="Selenium adapter evidence", + content_type="text/plain", + ) + + +@graph.step( + "close-selenium-cycle", + title="Close Selenium adapter cycle", + phase="after_each", + inputs=["browser_evidence"], + outputs=["cycle_complete"], +) +@runner.phase("after_each") +def after_each(ctx): + ctx.log("Completed one bounded Selenium adapter cycle") + + +@graph.step( + "finalize-selenium-journey", + title="Finalize Selenium adapter evaluation", + phase="after_all", + inputs=["cycle_complete"], + outputs=["journey_complete"], +) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the Selenium adapter evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/site-exploration/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/site-exploration/runner.py new file mode 100644 index 0000000..76e8464 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/site-exploration/runner.py @@ -0,0 +1,116 @@ +"""Safely explore same-origin pages through explicit runner phases. + +The browser script excludes destructive actions and returns bounded rendered +page evidence. The template retains that evidence for product review without +claiming that the pages themselves were changed. +""" + +import json +from pathlib import Path + +import orbit_sdk +from orbit_sdk import graph, runner + +graph.connect("validate-site", "explore-site") +graph.connect("explore-site", "review-evidence", kind="data", label="rendered pages") +graph.connect("review-evidence", "finalize-review") + + +@graph.step("validate-site", title="Validate site", phase="before_all", outputs=["site_target"]) +@runner.phase("before_all", step_id="validate-site") +def validate_site(ctx): + """Validate the browser target before safely exploring it.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("Set required build field(s): browser_base_url") + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + + +@graph.step( + "explore-site", + title="Explore rendered site", + phase="execute", + inputs=["site_target"], + outputs=["rendered_pages"], +) +@runner.phase("execute", step_id="explore-site") +def explore_site(ctx): + """Visit a bounded number of safe same-origin links. + + The embedded Playwright script excludes destructive paths and emits one + machine-readable evidence record after collecting rendered page content. + """ + module = str(Path(orbit_sdk.__file__).resolve().parents[2] / "frontend" / "node_modules" / "playwright") + screenshot = ( + ctx.app_data + / "artifacts" + / ctx.environment.get("ORBIT_RUN_ID", "manual") + / f"loop-{ctx.loop_index}" + / "site-exploration.png" + ) + screenshot.parent.mkdir(parents=True, exist_ok=True) + script = """const {chromium}=require(process.argv[1]),i=JSON.parse(process.argv[2]),blocked=/(logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe)/i;(async()=>{const b=await chromium.launch({headless:true}),p=await b.newPage(),v=[],seen=new Set(),o=new URL(i.baseUrl).origin;try{await p.goto(i.baseUrl,{waitUntil:'domcontentloaded',timeout:30000});for(let n=0;n<=i.maxClicks;n++){seen.add(p.url());v.push({url:p.url(),title:await p.title(),text:(await p.locator('body').innerText().catch(()=>'' )).replace(/\\s+/g,' ').slice(0,1200)});if(n===i.maxClicks)break;const a=await p.locator('a[href]').evaluateAll(x=>x.map((e,k)=>({k,href:e.href,text:(e.textContent||'').trim()})));const c=a.filter(x=>{try{const u=new URL(x.href);return u.origin===o&&!seen.has(u.href)&&!blocked.test(u.pathname+' '+x.text)}catch{return false}});if(!c.length)break;await p.locator('a[href]').nth(c[0].k).click({timeout:5000});await p.waitForLoadState('domcontentloaded',{timeout:10000}).catch(()=>{})}await p.screenshot({path:i.screenshot,fullPage:true});console.log('__ORBIT_SITE_EXPLORATION_RESULT__'+JSON.stringify({visited:v,screenshot:i.screenshot}))}finally{await b.close()}})().catch(e=>{console.error(e);process.exit(1)})""" + output = ctx.exec( + [ + "node", + "-e", + script, + module, + json.dumps( + {"baseUrl": ctx.build["browser_base_url"], "maxClicks": 3, "screenshot": str(screenshot)} + ), + ], + timeout=120, + target_log_source="site-exploration", + target_log_exclude_prefixes=("__ORBIT_SITE_EXPLORATION_RESULT__",), + ) + payload = next( + ( + line.removeprefix("__ORBIT_SITE_EXPLORATION_RESULT__") + for line in reversed(output.splitlines()) + if line.startswith("__ORBIT_SITE_EXPLORATION_RESULT__") + ), + "", + ) + if not payload: + raise RuntimeError("site exploration did not return structured evidence") + evidence = json.loads(payload) + titles = [str(page.get("title") or page.get("url")) for page in evidence.get("visited", [])] + ctx.emit_result( + { + "site_exploration": { + "evidence": evidence, + "opinion": f"Explored {len(titles)} rendered page(s): {'; '.join(titles[:3])}. Review the captured pages for clarity, usefulness, and friction.", + } + } + ) + + +@graph.step( + "review-evidence", + title="Review exploration evidence", + phase="verify", + inputs=["rendered_pages"], + outputs=["product_review"], +) +@runner.phase("verify", step_id="review-evidence") +def verify(ctx): + """Keep the rendered exploration evidence available to the supervisor.""" + ctx.log("Retained rendered exploration evidence for review") + + +@graph.step( + "finalize-review", + title="Finalize site review", + phase="after_all", + inputs=["product_review"], + outputs=["completed_review"], +) +@runner.phase("after_all", step_id="finalize-review") +def after_all(ctx): + """Finalize the bounded site exploration review.""" + ctx.log("Finalized the bounded site exploration review") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/source-aware-browser-journey/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/source-aware-browser-journey/runner.py new file mode 100644 index 0000000..9160839 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/source-aware-browser-journey/runner.py @@ -0,0 +1,126 @@ +"""Validate a declared source contract before running rendered browser journeys. + +Source matches provide implementation context only; rendered browser evidence +remains the proof of user-visible behavior. The lifecycle keeps those two kinds +of evidence distinct for supervisor review. +""" + +import json +import os + +from orbit_sdk import graph, runner + + +def source_contract(ctx): + """Return bounded source evidence without treating source as UI proof. + + Args: + ctx: The active Orbit runner context. + + Returns: + The configured expression and a bounded list of matching files. + + Raises: + ValueError: If the source contract is absent or has no matching files. + """ + pattern = os.environ.get("ORBIT_SOURCE_CONTRACT_PATTERN", "").strip() + if not pattern: + raise ValueError("Set ORBIT_SOURCE_CONTRACT_PATTERN to a required source expression") + glob = os.environ.get("ORBIT_SOURCE_CONTRACT_GLOB", "").strip() + command = ["rg", "-l", pattern] + if glob: + command.extend(["--glob", glob]) + command.append(".") + output = ctx.exec(command, cwd=ctx.project_root, timeout=30) + files = [path for path in output.splitlines() if path][:40] + if not files: + raise ValueError("The configured source contract was not found in the target project") + return {"pattern": pattern, "glob": glob or None, "files": files} + + +graph.connect("validate-source-contract", "run-browser-journey") +graph.connect("run-browser-journey", "publish-rendered-evidence", kind="data", label="browser evidence") +graph.connect("publish-rendered-evidence", "close-source-aware-cycle") +graph.connect("close-source-aware-cycle", "run-browser-journey", kind="loop", label="next iteration") +graph.connect( + "close-source-aware-cycle", "finalize-source-aware-journey", kind="condition", label="completed" +) + + +@graph.step( + "validate-source-contract", + title="Validate source and browser contract", + phase="before_all", + outputs=["source_contract"], +) +@runner.phase("before_all", step_id="validate-source-contract") +def validate_source_contract(ctx): + """Validate browser inputs and publish the source-side contract.""" + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one fixed journey case are required") + contract = source_contract(ctx) + ctx.emit_result({"source_aware_journey": {"source_contract": contract}}) + ctx.log(f"Validated source contract in {len(contract['files'])} file(s)") + + +@graph.step( + "run-browser-journey", + title="Run rendered browser journey", + phase="execute", + inputs=["source_contract"], + outputs=["browser_evidence"], +) +@runner.phase("execute", step_id="run-browser-journey") +def run_browser_journey(ctx): + """Run the rendered browser journey independently of source evidence.""" + evidence = ctx.playwright_journey() + ctx.emit_result({"source_aware_journey": {"iteration": ctx.loop_index, "evidence": evidence}}) + ctx.save_data_file( + f"source-aware-journey/iteration-{ctx.loop_index}.json", + json.dumps(evidence, ensure_ascii=False, indent=2), + label="Source-aware rendered journey evidence", + content_type="application/json", + ) + + +@graph.step( + "publish-rendered-evidence", + title="Publish rendered evidence", + phase="verify", + inputs=["browser_evidence"], + outputs=["review_ready"], +) +@runner.phase("verify", step_id="publish-rendered-evidence") +def publish_rendered_evidence(ctx): + """Mark the combined source and browser evidence ready for review.""" + ctx.log("Published source context with rendered browser evidence") + + +@graph.step( + "close-source-aware-cycle", + title="Close source-aware browser cycle", + phase="after_each", + inputs=["review_ready"], + outputs=["cycle_complete"], +) +@runner.phase("after_each", step_id="close-source-aware-cycle") +def close_source_aware_cycle(ctx): + """Close one bounded source-aware browser iteration.""" + ctx.log("Completed one bounded source-aware browser journey") + + +@graph.step( + "finalize-source-aware-journey", + title="Finalize source-aware browser journey", + phase="after_all", + inputs=["cycle_complete"], + outputs=["journey_complete"], +) +@runner.phase("after_all", step_id="finalize-source-aware-journey") +def finalize_source_aware_journey(ctx): + """Finalize the source-aware browser journey.""" + ctx.log("Finalized the source-aware browser evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/tailwind-source-aware/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/tailwind-source-aware/runner.py new file mode 100644 index 0000000..a0b2f2d --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/tailwind-source-aware/runner.py @@ -0,0 +1,103 @@ +"""Reference runner for Tailwind applications with rendered browser evidence.""" + +from orbit_sdk import graph, runner + +graph.connect("inspect-source", "run-browser") +graph.connect("run-browser", "publish-evidence") +graph.connect("publish-evidence", "close-tailwind-cycle") +graph.connect("close-tailwind-cycle", "run-browser", kind="loop", label="next iteration") +graph.connect("close-tailwind-cycle", "finalize-tailwind-journey", kind="condition", label="completed") + + +@graph.step( + "inspect-source", + title="Inspect Tailwind source contract", + phase="before_all", + outputs=["source_contract"], +) +@runner.phase("before_all") +def before_all(ctx): + # Inspect stylesheet content as well as config paths: Tailwind v4 commonly + # uses `@import "tailwindcss"` in an otherwise generic `index.css` name. + files = ctx.exec( + ["sh", "-lc", "rg -l 'tailwindcss|@tailwind' --glob '*.css' . || true"], + cwd=ctx.project_root, + timeout=30, + ) + config = ctx.exec( + ["sh", "-lc", "rg --files -g 'tailwind.config.*' . || true"], cwd=ctx.project_root, timeout=30 + ) + if not files.strip() and not config.strip(): + raise ValueError("No Tailwind configuration or stylesheet was found in the target project") + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one journey case are required") + ctx.emit_result( + { + "tailwind_journey": { + "stylesheet_files": files.splitlines()[:40], + "config_files": config.splitlines()[:20], + } + } + ) + ctx.log("Validated the Tailwind source contract") + + +@graph.step( + "run-browser", + title="Run rendered Tailwind journey", + phase="execute", + inputs=["source_contract"], + outputs=["browser_evidence"], +) +@runner.phase("execute") +def execute(ctx): + # The SDK captures screenshots and visible-page evidence; Tailwind class names + # are never treated as proof that a user-visible interaction succeeded. + evidence = ctx.playwright_journey() + ctx.emit_result({"tailwind_journey": {"iteration": ctx.loop_index, "evidence": evidence}}) + ctx.save_data_file( + f"tailwind-journey/iteration-{ctx.loop_index}.json", + __import__("json").dumps(evidence, ensure_ascii=False, indent=2), + label="Tailwind rendered journey evidence", + content_type="application/json", + ) + + +@graph.step( + "publish-evidence", + title="Publish rendered evidence", + phase="verify", + inputs=["browser_evidence"], + outputs=["review_ready"], +) +@runner.phase("verify") +def verify(ctx): + ctx.log("Published Tailwind source context with rendered browser evidence") + + +@graph.step( + "close-tailwind-cycle", + title="Close Tailwind browser cycle", + phase="after_each", + inputs=["review_ready"], + outputs=["cycle_complete"], +) +@runner.phase("after_each") +def after_each(ctx): + ctx.log("Completed one bounded Tailwind browser journey") + + +@graph.step( + "finalize-tailwind-journey", + title="Finalize Tailwind browser evaluation", + phase="after_all", + inputs=["cycle_complete"], + outputs=["journey_complete"], +) +@runner.phase("after_all") +def after_all(ctx): + ctx.log("Finalized the Tailwind browser evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/canonical/runner-templates/user-journey-cycle/runner.py b/backend/orbit_sdk/visual/canonical/runner-templates/user-journey-cycle/runner.py new file mode 100644 index 0000000..429d584 --- /dev/null +++ b/backend/orbit_sdk/visual/canonical/runner-templates/user-journey-cycle/runner.py @@ -0,0 +1,207 @@ +"""Run a recurring browser journey with phases declared in this runner. + +The runner retries failed fixed cases before rotating through the journey set. +It persists only bounded state and rendered evidence, so the supervisor can +review what happened without relying on hidden lifecycle helpers. +""" + +import json + +from orbit_sdk import graph, runner + +NAMESPACE = "user_journey" + +graph.connect("validate-user-journey-runtime", "validate-user-journey") +graph.connect("validate-user-journey", "load-user-journey-state") +graph.connect("load-user-journey-state", "plan-user-journey") +graph.connect("plan-user-journey", "run-user-journey", label="focused cases") +graph.connect("run-user-journey", "review-user-journey", kind="data", label="browser evidence") +graph.connect("review-user-journey", "retain-user-journey") +graph.connect("retain-user-journey", "plan-user-journey", kind="loop", label="next iteration") +graph.connect("retain-user-journey", "finalize-user-journey", kind="condition", label="completed") + + +def state(ctx): + """Load the bounded state used to choose the next browser journey. + + Args: + ctx: The active Orbit runner context. + + Returns: + Persisted journey state, initialized for a new build when absent. + """ + return ctx.load_state(NAMESPACE, {"next_case_index": 0, "failed_case_ids": [], "history": []}) + + +@graph.step( + "validate-user-journey-runtime", + title="Validate user journey runtime", + phase="before_all", + outputs=["browser_runtime"], +) +@runner.phase("before_all", step_id="validate-user-journey-runtime") +def validate_runtime(ctx): + """Validate the browser base URL before the recurring journey starts.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("Set required build field(s): browser_base_url") + ctx.log("Validated the browser runtime target") + + +@graph.step( + "validate-user-journey", + title="Validate user journey contract", + phase="before_all", + inputs=["browser_runtime"], + outputs=["journey_contract"], +) +@runner.phase("before_all", step_id="validate-user-journey") +def validate_contract(ctx): + """Require fixed journey cases for reproducible coverage.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + ctx.log("Validated the user journey contract") + + +@graph.step( + "load-user-journey-state", + title="Load user journey state", + phase="before_each", + inputs=["journey_contract"], + outputs=["journey_state"], +) +@runner.phase("before_each", step_id="load-user-journey-state") +def load_state(ctx): + """Persist an initialized state document for this iteration.""" + ctx.save_state(NAMESPACE, state(ctx)) + + +@graph.step( + "plan-user-journey", + title="Plan focused user journey", + phase="before_each", + inputs=["journey_state"], + outputs=["journey_plan"], +) +@runner.phase("before_each", step_id="plan-user-journey") +def plan(ctx): + """Choose failed cases first, then rotate through the fixed case set.""" + current = state(ctx) + failed = bool(current.get("failed_case_ids")) + failed_ids = {str(case_id) for case_id in current.get("failed_case_ids", [])} + cases = [case for case in ctx.test_cases if str(case.get("id")) in failed_ids] + if not cases: + cases = [ctx.test_cases[int(current.get("next_case_index", 0)) % len(ctx.test_cases)]] + rules = [ + "Preserve observable evidence for every browser action.", + "Do not infer a result that the page did not expose.", + ] + if failed: + rules.insert(0, "Revisit previously failed journeys before exploring a new route.") + if ctx.previous_supervisor_feedback.get("reported_issues"): + rules.insert(0, "Prioritize the supervisor's previously reported issues.") + plan_data = { + "case_ids": [str(case.get("id")) for case in cases], + "reason": "Revisit previously failed journeys." + if failed + else "Rotate one fixed journey to retain bounded coverage.", + "rules": rules, + "supervisor_feedback": ctx.previous_supervisor_feedback, + } + current["plan"] = plan_data + ctx.save_state(NAMESPACE, current) + ctx.emit_result( + {NAMESPACE: {"iteration": ctx.loop_index, "case_count": len(ctx.test_cases), "plan": plan_data}} + ) + + +@graph.step( + "run-user-journey", + title="Run user journey", + phase="execute", + inputs=["journey_plan"], + outputs=["journey_evidence"], +) +@runner.phase("execute", step_id="run-user-journey") +def execute(ctx): + """Run the focused browser cases and retain evidence plus next state.""" + current = state(ctx) + ids = set(current["plan"]["case_ids"]) + evidence = ctx.playwright_journey([case for case in ctx.test_cases if str(case.get("id")) in ids]) + results = list(evidence.get("results", [])) + current["failed_case_ids"] = [str(item.get("id")) for item in results if not item.get("passed")] + current["next_case_index"] = (int(current.get("next_case_index", 0)) + 1) % len(ctx.test_cases) + handoff = { + "iteration": ctx.loop_index, + "reason": current["plan"]["reason"], + "rules": current["plan"]["rules"], + "passed": len(results) - len(current["failed_case_ids"]), + "failed": len(current["failed_case_ids"]), + "failed_case_ids": current["failed_case_ids"], + } + current["history"] = [*current.get("history", []), handoff][-24:] + current["handoff"] = handoff + ctx.save_state(NAMESPACE, current) + artifact = ctx.save_data_file( + f"{NAMESPACE}/iteration-{ctx.loop_index}.json", + json.dumps( + {"plan": current["plan"], "evidence": evidence, "handoff": handoff}, ensure_ascii=False, indent=2 + ), + label="User journey iteration evidence", + content_type="application/json", + ) + ctx.emit_result( + { + NAMESPACE: { + "iteration": ctx.loop_index, + "plan": current["plan"], + "results": results, + "evidence": evidence, + "handoff": handoff, + "artifact": artifact, + } + } + ) + + +@graph.step( + "review-user-journey", + title="Review user journey evidence", + phase="verify", + inputs=["journey_evidence"], + outputs=["journey_handoff"], +) +@runner.phase("verify", step_id="review-user-journey") +def verify(ctx): + """Expose the next-iteration handoff to the supervisor.""" + ctx.emit_result({NAMESPACE: {"next_iteration": state(ctx).get("handoff", {})}}) + ctx.log("Stored the journey summary and next-iteration handoff") + + +@graph.step( + "retain-user-journey", + title="Retain user journey result", + phase="after_each", + inputs=["journey_handoff"], + outputs=["iteration_complete"], +) +@runner.phase("after_each", step_id="retain-user-journey") +def after_each(ctx): + """Close the current bounded journey iteration.""" + ctx.log("Closed this bounded user journey") + + +@graph.step( + "finalize-user-journey", + title="Finalize user journey evaluation", + phase="after_all", + inputs=["iteration_complete"], + outputs=["final_status"], +) +@runner.phase("after_all", step_id="finalize-user-journey") +def after_all(ctx): + """Finalize the recurring journey after its last iteration.""" + ctx.log("Finalized the user journey evaluation") + + +if __name__ == "__main__": + runner.main() diff --git a/backend/orbit_sdk/visual/improvement.py b/backend/orbit_sdk/visual/improvement.py new file mode 100644 index 0000000..614232f --- /dev/null +++ b/backend/orbit_sdk/visual/improvement.py @@ -0,0 +1,349 @@ +"""Reusable, evidence-backed prompt-improvement operations for Visual Mode.""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Mapping + +from ..decorators.visual import visual_node + +_REQUIRED_SUFFICIENT_EVALUATIONS = 3 +_PROMPT_BLOCK_START = "" +_PROMPT_BLOCK_END = "" + + +def _git(ctx: Any, *args: str) -> str: + return ctx.exec(["git", *args], cwd=ctx.project_root, timeout=300) + + +def _prompt_path(ctx: Any) -> str: + value = str(ctx.build.get("managed_prompt_path") or ctx.build.get("prompt_bundle") or "").strip() + if not value: + raise ValueError("native improvement cycle requires target_environment.managed_prompt_path") + return value + + +def _prompt_evidence(ctx: Any) -> dict[str, str]: + path = _prompt_path(ctx) + content = ctx.project_path(path).read_text(encoding="utf-8") + return {"path": path, "sha256": hashlib.sha256(content.encode()).hexdigest(), "content": content} + + +def _candidate(ctx: Any) -> tuple[str | None, list[str]]: + patch = _git(ctx, "diff", "--binary", "--") + changed = [line for line in _git(ctx, "diff", "--name-only").splitlines() if line] + return (hashlib.sha256(patch.encode()).hexdigest() if patch else None), changed + + +def _state_path(ctx: Any) -> Any: + build_id = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(ctx.build.get("id") or "manual")) + directory = ctx.app_data / "improvement-cycles" + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{build_id}.json" + + +def _load_state(ctx: Any) -> dict[str, Any]: + path = _state_path(ctx) + if not path.exists(): + return {"candidate_fingerprint": None, "sufficient_evaluations": 0, "history": []} + value = json.loads(path.read_text(encoding="utf-8")) + return ( + value + if isinstance(value, dict) + else {"candidate_fingerprint": None, "sufficient_evaluations": 0, "history": []} + ) + + +def _save_state(ctx: Any, state: dict[str, Any]) -> None: + state["history"] = list(state.get("history", []))[-24:] + _state_path(ctx).write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _accepted_proposals(ctx: Any) -> list[dict[str, Any]]: + feedback = ctx.previous_supervisor_feedback + return [ + proposal + for proposal in feedback.get("improvements", []) + if isinstance(proposal, dict) and str(proposal.get("status") or "").lower() == "accepted" + ] + + +def _apply_proposals(ctx: Any, proposals: list[dict[str, Any]]) -> dict[str, object]: + path = _prompt_path(ctx) + current = ctx.project_path(path).read_text(encoding="utf-8") + if not proposals: + return {"path": path, "changed": False, "reason": "no_accepted_proposals"} + lines = ["## Accepted improvement proposals", "", f"Iteration: {ctx.loop_index}", ""] + for proposal in proposals: + lines.extend( + ( + f"### {proposal.get('title') or 'Accepted proposal'}", + str(proposal.get("rationale") or ""), + f"Acceptance evidence: {proposal.get('acceptanceEvidence') or ''}", + "", + ) + ) + block = "\n".join((_PROMPT_BLOCK_START, "\n".join(lines).rstrip(), _PROMPT_BLOCK_END)) + start, end = current.find(_PROMPT_BLOCK_START), current.find(_PROMPT_BLOCK_END) + if start >= 0 and end > start: + updated = current[:start] + block + current[end + len(_PROMPT_BLOCK_END) :] + elif start >= 0 or end >= 0: + raise ValueError("prompt has an incomplete OpenOrbit accepted-proposals block") + else: + updated = current.rstrip() + "\n\n" + block + "\n" + return ctx.update_file(path, updated) + + +@visual_node( + kind="validate_git_repository", + group_key="improvement", + display_name="Validate Git repository", + title_key="visual.nodes.validateGitRepository.title", + description_key="visual.nodes.validateGitRepository.description", + default_outputs=("repository_target",), +) +def validate_git_repository(ctx: Any, _: Mapping[str, Any], __: Mapping[str, object]) -> dict[str, object]: + """Require a Git worktree before an improvement operation begins.""" + _git(ctx, "rev-parse", "--show-toplevel") + return {"repository_target": True} + + +@visual_node( + kind="validate_improvement_inputs", + group_key="improvement", + display_name="Validate improvement inputs", + title_key="visual.nodes.validateImprovementInputs.title", + description_key="visual.nodes.validateImprovementInputs.description", + default_outputs=("evaluation_contract",), +) +def validate_improvement_inputs( + ctx: Any, _: Mapping[str, Any], __: Mapping[str, object] +) -> dict[str, object]: + """Require fixed target-AI cases and the selected model profile.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed target-AI prompt for a native improvement cycle") + profile = ctx.resource("model_profile", {}) + if not isinstance(profile, dict) or not profile.get("model"): + raise ValueError("Select a configured model profile for a native improvement cycle") + ctx.log("Validated an OpenOrbit-native target-AI prompt improvement cycle") + return {"evaluation_contract": True} + + +@visual_node( + kind="prepare_managed_prompt", + group_key="improvement", + display_name="Prepare managed prompt", + title_key="visual.nodes.prepareManagedPrompt.title", + description_key="visual.nodes.prepareManagedPrompt.description", + default_config={"mode": "apply_accepted"}, + default_outputs=("managed_prompt",), +) +def prepare_managed_prompt(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Apply approved changes or expose a prompt as isolated-worktree evidence.""" + mode = str(config.get("mode", "apply_accepted")) + if mode == "isolated_worktree": + value = {"iteration": ctx.loop_index, "proposal_mode": mode, "managed_prompt": _prompt_evidence(ctx)} + elif mode == "apply_accepted": + accepted = _accepted_proposals(ctx) + required = bool(ctx.build.get("require_human_approval_before_apply", False)) + update = ( + {"changed": False, "reason": "awaiting_human_approval", "proposal_count": len(accepted)} + if required and accepted + else _apply_proposals(ctx, accepted if not required else []) + ) + ids = [str(value) for value in ctx.previous_supervisor_feedback.get("_orbit_proposal_ids", [])] + fingerprint, changed = _candidate(ctx) + value = { + "iteration": ctx.loop_index, + "candidate_fingerprint": fingerprint, + "changed_paths": changed, + "prompt_update": update, + "requires_human_approval": required, + "managed_prompt": _prompt_evidence(ctx), + "proposal_applications": ctx.record_proposal_application(ids, update), + } + ctx.log("Refreshed the rollback-protected prompt from accepted supervisor feedback") + else: + raise ValueError("has an unsupported managed prompt mode") + ctx.emit_result({"improvement_cycle": value}) + return {"managed_prompt": value["managed_prompt"]} + + +@visual_node( + kind="exercise_managed_prompt", + group_key="improvement", + display_name="Exercise managed prompt", + title_key="visual.nodes.exerciseManagedPrompt.title", + description_key="visual.nodes.exerciseManagedPrompt.description", + default_config={"include_candidate": False}, + default_outputs=("target_responses",), +) +def exercise_managed_prompt( + ctx: Any, config: Mapping[str, Any], __: Mapping[str, object] +) -> dict[str, object]: + """Run every fixed request against the managed prompt and retain raw evidence.""" + prompt = _prompt_evidence(ctx) + responses: list[dict[str, object]] = [] + for case in ctx.test_cases: + request = str(case.get("prompt") or "").strip() + if not request: + raise ValueError("each target-AI test case requires a prompt") + turn = ctx.complete_model( + "# Managed agent instructions\n" + + prompt["content"] + + "\n\n# User request\n" + + request + + "\n\nRespond as the managed agent." + ) + responses.append( + { + "id": case.get("id"), + "name": case.get("name"), + "request": request, + "acceptance": str(case.get("acceptance") or ""), + "response": turn["response"], + "model": turn["model"], + } + ) + artifact = ctx.save_data_file( + "target-ai-responses.json", + json.dumps(responses, ensure_ascii=False, indent=2), + label="Target AI responses", + content_type="application/json", + ) + fingerprint, changed = _candidate(ctx) + value: dict[str, object] = { + "iteration": ctx.loop_index, + "evidence": {"target_ai_responses": responses, "artifact": artifact}, + } + if config.get("include_candidate", False) or fingerprint or changed: + value.update({"candidate_fingerprint": fingerprint, "changed_paths": changed}) + ctx.emit_result({"improvement_cycle": value}) + return {"target_responses": responses} + + +@visual_node( + kind="assess_prompt_candidate", + group_key="improvement", + display_name="Assess prompt candidate", + title_key="visual.nodes.assessPromptCandidate.title", + description_key="visual.nodes.assessPromptCandidate.description", + default_outputs=("candidate_verdict",), +) +def assess_prompt_candidate(ctx: Any, _: Mapping[str, Any], __: Mapping[str, object]) -> dict[str, object]: + """Promote only a stable candidate after repeated sufficient evaluations.""" + state = _load_state(ctx) + fingerprint, changed = _candidate(ctx) + if not fingerprint: + state["candidate_fingerprint"], state["sufficient_evaluations"], verdict = None, 0, "no_candidate" + elif state.get("candidate_fingerprint") == fingerprint: + state["sufficient_evaluations"] = int(state.get("sufficient_evaluations", 0)) + 1 + verdict = ( + "ready_for_approval" + if state["sufficient_evaluations"] >= _REQUIRED_SUFFICIENT_EVALUATIONS + else "continue_validation" + ) + else: + state["candidate_fingerprint"], state["sufficient_evaluations"], verdict = ( + fingerprint, + 1, + "continue_validation", + ) + state.setdefault("history", []).append( + {"iteration": ctx.loop_index, "fingerprint": fingerprint, "paths": changed, "verdict": verdict} + ) + _save_state(ctx, state) + value = { + "candidate_fingerprint": fingerprint, + "changed_paths": changed, + "sufficient_evaluations": state["sufficient_evaluations"], + "required_evaluations": _REQUIRED_SUFFICIENT_EVALUATIONS, + "verdict": verdict, + } + ctx.emit_result({"improvement_cycle": value}) + ctx.log(f"Candidate verdict: {verdict}") + return {"candidate_verdict": value} + + +@visual_node( + kind="retain_improvement_iteration", + group_key="improvement", + display_name="Retain improvement iteration", + title_key="visual.nodes.retainImprovementIteration.title", + description_key="visual.nodes.retainImprovementIteration.description", + default_outputs=("iteration_snapshot",), +) +def retain_improvement_iteration( + ctx: Any, _: Mapping[str, Any], __: Mapping[str, object] +) -> dict[str, object]: + """Keep a recovery checkpoint and the current improvement evidence.""" + value = ctx.save_first_after_each_snapshot() + ctx.log("Retained prompt versions, decisions, and validation evidence") + return {"iteration_snapshot": value} + + +@visual_node( + kind="restore_improvement_baseline", + group_key="improvement", + display_name="Restore improvement baseline", + title_key="visual.nodes.restoreImprovementBaseline.title", + description_key="visual.nodes.restoreImprovementBaseline.description", + default_outputs=("restored_target",), +) +def restore_improvement_baseline( + ctx: Any, _: Mapping[str, Any], __: Mapping[str, object] +) -> dict[str, object]: + """Restore the exact pre-iteration target without creating a commit.""" + value = ctx.restore_before_each_snapshot() + ctx.log("Restored the native improvement target without committing changes") + return {"restored_target": value} + + +@visual_node( + kind="create_agent_proposal", + group_key="improvement", + display_name="Create agent proposal", + title_key="visual.nodes.createAgentProposal.title", + description_key="visual.nodes.createAgentProposal.description", + default_config={"provider": "codex", "options": ""}, + default_outputs=("agent_proposal",), + required_config=("provider",), +) +def create_agent_proposal(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Ask an isolated coding agent for one supervisor-reviewed proposal.""" + issue = ctx.current_issue_assessment + assessment = issue.get("evaluation") if isinstance(issue, dict) else None + if not issue or (isinstance(assessment, dict) and assessment.get("approval") == "rejected"): + reason = "no_reported_issue" if not issue else "issue_rejected" + value = {"skipped": True, "reason": reason, **({"issue": issue} if issue else {})} + ctx.emit_result({"agent_proposal": value}) + if issue: + ctx.log("Skipped AI agent work because the supervisor rejected the issue") + return {"agent_proposal": value} + task = "\n".join( + ( + "You are the autonomous improvement agent for this repository.", + "Do not ask questions or wait for approval. First inspect the repository and the managed prompt, then make exactly one atomic, evidence-backed improvement, validate it, and finish the task completely.", + "You are working in an isolated proposal worktree. Make the complete change there; it will be captured as a reviewable diff and will not be applied to the source repository automatically. Do not commit, reset, or discard unrelated user changes.", + f"Managed prompt path: {ctx.build.get('managed_prompt_path') or ctx.build.get('prompt_bundle')}", + f"Build purpose: {ctx.build.get('purpose', '')}", + f"Fixed acceptance criteria: {json.dumps([case.get('acceptance', '') for case in ctx.test_cases], ensure_ascii=False)}", + f"Supervisor-assessed issue to resolve: {json.dumps(issue, ensure_ascii=False)}", + "If you completed meaningful feedback or a change, print one final line exactly in this format: ORBIT_AGENT_FEEDBACK: . If there is no meaningful feedback, do not print that marker.", + ) + ) + agent = ctx.run_ai_agent(task, provider=config["provider"], options=config.get("options", "")) + proposal = agent.get("proposal", {}) + if agent["feedback"] and agent["changed_files"] and proposal.get("fingerprint"): + ctx.register_evaluation( + str(agent["feedback"]), + changed_files=[str(path) for path in agent["changed_files"]], + validation="Agent completed its autonomous repository task.", + improvement_fingerprint=str(proposal["fingerprint"]), + ) + value = {"issue": issue, "agent": {key: value for key, value in agent.items() if key != "output"}} + ctx.emit_result({"agent_proposal": value}) + ctx.log("Retained an isolated agent change proposal for review") + return {"agent_proposal": value} diff --git a/backend/orbit_sdk/visual/journeys.py b/backend/orbit_sdk/visual/journeys.py new file mode 100644 index 0000000..b55e943 --- /dev/null +++ b/backend/orbit_sdk/visual/journeys.py @@ -0,0 +1,458 @@ +"""SDK-owned stateful browser-journey operations used by shipped templates.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from ..decorators.visual import visual_node + +_PERSONA_SCRIPT = r"""const { chromium } = require(process.argv[1]); const input = JSON.parse(process.argv[2]); +const blocked=/(logout|signout|delete|remove|destroy|payment|checkout|purchase|upgrade|unsubscribe)/i;(async()=>{const options={headless:true};if(input.executablePath)options.executablePath=input.executablePath;const browser=await chromium.launch(options);const page=await browser.newPage();try{await page.goto(input.url,{waitUntil:'domcontentloaded',timeout:30000});const before=page.url();const actions=await page.locator('a[href]').evaluateAll(items=>items.map((item,index)=>({id:`link-${index}`,href:item.href,label:(item.textContent||'').trim().replace(/\s+/g,' ').slice(0,140)})).filter(item=>item.href));const safe=actions.filter(item=>{try{const u=new URL(item.href),origin=new URL(before).origin;return u.origin===origin&&!blocked.test(u.pathname+' '+item.label)}catch{return false}}).slice(0,30);let acted=false;if(input.allowedHref){const candidate=safe.find(item=>item.href===input.allowedHref);if(!candidate)throw new Error('planned action is no longer an allowed visible link');await page.goto(candidate.href,{waitUntil:'domcontentloaded',timeout:30000});acted=true}const text=(await page.locator('body').innerText().catch(()=>'' )).replace(/\s+/g,' ').slice(0,1800);await page.screenshot({path:input.screenshot,fullPage:true});console.log('__ORBIT_PERSONA_RESULT__'+JSON.stringify({before_url:before,url:page.url(),title:await page.title(),visible_text:text,available_actions:safe,acted,screenshot:input.screenshot}))}finally{await browser.close()}})().catch(error=>{console.error(error);process.exit(1)});""" + + +def _persona_state(ctx: Any, namespace: str) -> dict[str, Any]: + value = ctx.load_state(namespace, None) + if isinstance(value, dict): + return value + case = ctx.test_cases[0] + return { + "persona": str(case.get("name") or "A careful product user"), + "current_goal": str(case.get("prompt") or "Understand the product through safe visits."), + "feeling": "curious", + "next_intent": "Orient myself on the first visible page.", + "learnings": [], + "reported_issues": [], + "history": [], + } + + +def _save_persona_state(ctx: Any, namespace: str, state: dict[str, Any]) -> None: + state["learnings"] = list(state.get("learnings", []))[-12:] + state["reported_issues"] = list(state.get("reported_issues", []))[-8:] + state["history"] = list(state.get("history", []))[-24:] + ctx.save_state(namespace, state) + + +def _persona_browser( + ctx: Any, *, url: str, screenshot: str, allowed_href: str | None = None +) -> dict[str, Any]: + import orbit_sdk + + module = str(Path(orbit_sdk.__file__).resolve().parents[2] / "frontend" / "node_modules" / "playwright") + artifacts = ( + ctx.app_data / "artifacts" / ctx.environment.get("ORBIT_RUN_ID", "manual") / f"loop-{ctx.loop_index}" + ) + artifacts.mkdir(parents=True, exist_ok=True) + payload = { + "url": url, + "screenshot": str(artifacts / screenshot), + "allowedHref": allowed_href, + "executablePath": str(ctx.build.get("browser_executable_path") or "").strip(), + } + output = ctx.exec( + ["node", "-e", _PERSONA_SCRIPT, module, json.dumps(payload)], + timeout=120, + target_log_source="browser-persona", + target_log_exclude_prefixes=("__ORBIT_PERSONA_RESULT__",), + ) + result = next( + ( + line.removeprefix("__ORBIT_PERSONA_RESULT__") + for line in reversed(output.splitlines()) + if line.startswith("__ORBIT_PERSONA_RESULT__") + ), + "", + ) + if not result: + raise RuntimeError("browser persona did not return structured evidence") + value = json.loads(result) + if not isinstance(value, dict): + raise RuntimeError("browser persona did not return an object") + return value + + +def _journey_state(ctx: Any, namespace: str) -> dict[str, Any]: + value = ctx.load_state(namespace, {"next_case_index": 0, "failed_case_ids": [], "history": []}) + return value if isinstance(value, dict) else {"next_case_index": 0, "failed_case_ids": [], "history": []} + + +@visual_node( + kind="validate_browser_runtime", + group_key="journeys", + display_name="Validate browser runtime", + title_key="visual.nodes.validateBrowserRuntime.title", + description_key="visual.nodes.validateBrowserRuntime.description", + default_config={"field": "browser_base_url"}, + default_outputs=("browser_runtime",), +) +def validate_browser_runtime( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Require the browser target required by user-journey templates.""" + field = str(config.get("field", "browser_base_url")) + if not ctx.build.get(field): + raise ValueError(f"Set required build field(s): {field}") + return {"browser_runtime": True} + + +@visual_node( + kind="validate_browser_journey_contract", + group_key="contracts", + display_name="Validate browser journey contract", + title_key="visual.nodes.validateBrowserRuntime.title", + description_key="visual.nodes.validateBrowserRuntime.description", + default_outputs=("journey_contract",), +) +def validate_browser_journey_contract( + ctx: Any, _: Mapping[str, Any], __: Mapping[str, object] +) -> dict[str, object]: + """Require the browser URL and fixed cases used by a recurring journey.""" + if not ctx.build.get("browser_base_url") or not ctx.test_cases: + raise ValueError("A browser base URL and at least one journey case are required") + ctx.log("Validated the continuous Playwright journey contract") + return {"journey_contract": True} + + +@visual_node( + kind="initialize_user_journey_state", + group_key="journeys", + display_name="Initialize user journey state", + title_key="visual.nodes.initializeUserJourneyState.title", + description_key="visual.nodes.initializeUserJourneyState.description", + default_config={"namespace": "user_journey"}, + default_outputs=("journey_state",), + required_config=("namespace",), +) +def initialize_user_journey_state( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Persist a bounded initial state before planning a journey iteration.""" + state = _journey_state(ctx, config["namespace"]) + ctx.save_state(config["namespace"], state) + return {"journey_state": state} + + +@visual_node( + kind="plan_user_journey", + group_key="journeys", + display_name="Plan user journey", + title_key="visual.nodes.planUserJourney.title", + description_key="visual.nodes.planUserJourney.description", + default_config={"namespace": "user_journey"}, + default_outputs=("journey_plan",), + required_config=("namespace",), +) +def plan_user_journey(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Retry failed cases first, otherwise rotate one fixed browser journey.""" + if not ctx.test_cases: + raise ValueError("Select at least one fixed journey case before running this runner") + namespace = config["namespace"] + current = _journey_state(ctx, namespace) + failed_ids = {str(case_id) for case_id in current.get("failed_case_ids", [])} + cases = [case for case in ctx.test_cases if str(case.get("id")) in failed_ids] + failed = bool(cases) + if not cases: + cases = [ctx.test_cases[int(current.get("next_case_index", 0)) % len(ctx.test_cases)]] + rules = [ + "Preserve observable evidence for every browser action.", + "Do not infer a result that the page did not expose.", + ] + if failed: + rules.insert(0, "Revisit previously failed journeys before exploring a new route.") + if ctx.previous_supervisor_feedback.get("reported_issues"): + rules.insert(0, "Prioritize the supervisor's previously reported issues.") + plan = { + "case_ids": [str(case.get("id")) for case in cases], + "reason": "Revisit previously failed journeys." + if failed + else "Rotate one fixed journey to retain bounded coverage.", + "rules": rules, + "supervisor_feedback": ctx.previous_supervisor_feedback, + } + current["plan"] = plan + ctx.save_state(namespace, current) + ctx.emit_result( + {namespace: {"iteration": ctx.loop_index, "case_count": len(ctx.test_cases), "plan": plan}} + ) + return {"journey_plan": plan} + + +@visual_node( + kind="run_user_journey", + group_key="journeys", + display_name="Run user journey", + title_key="visual.nodes.runUserJourney.title", + description_key="visual.nodes.runUserJourney.description", + default_config={"namespace": "user_journey", "history_limit": 24}, + default_outputs=("journey_evidence",), + required_config=("namespace",), +) +def run_user_journey(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Run selected browser cases and retain bounded next-iteration state.""" + namespace = config["namespace"] + current = _journey_state(ctx, namespace) + plan = current.get("plan") if isinstance(current.get("plan"), dict) else {} + selected = {str(value) for value in plan.get("case_ids", [])} + evidence = ctx.playwright_journey([case for case in ctx.test_cases if str(case.get("id")) in selected]) + results = list(evidence.get("results", [])) if isinstance(evidence, dict) else [] + failed_ids = [ + str(item.get("id")) for item in results if isinstance(item, dict) and not item.get("passed") + ] + current["failed_case_ids"] = failed_ids + current["next_case_index"] = (int(current.get("next_case_index", 0)) + 1) % len(ctx.test_cases) + handoff = { + "iteration": ctx.loop_index, + "reason": plan.get("reason", ""), + "rules": plan.get("rules", []), + "passed": len(results) - len(failed_ids), + "failed": len(failed_ids), + "failed_case_ids": failed_ids, + } + limit = int(config.get("history_limit", 24)) + current["history"] = [*current.get("history", []), handoff][-max(1, limit) :] + current["handoff"] = handoff + ctx.save_state(namespace, current) + artifact = ctx.save_data_file( + f"{namespace}/iteration-{ctx.loop_index}.json", + json.dumps({"plan": plan, "evidence": evidence, "handoff": handoff}, ensure_ascii=False, indent=2), + label="User journey iteration evidence", + content_type="application/json", + ) + value = { + "iteration": ctx.loop_index, + "plan": plan, + "results": results, + "evidence": evidence, + "handoff": handoff, + "artifact": artifact, + } + ctx.emit_result({namespace: value}) + return {"journey_evidence": value} + + +@visual_node( + kind="publish_user_journey_handoff", + group_key="journeys", + display_name="Publish user journey handoff", + title_key="visual.nodes.publishUserJourneyHandoff.title", + description_key="visual.nodes.publishUserJourneyHandoff.description", + default_config={"namespace": "user_journey"}, + default_outputs=("journey_handoff",), + required_config=("namespace",), +) +def publish_user_journey_handoff( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Expose retained handoff evidence to the supervisor.""" + namespace = config["namespace"] + handoff = _journey_state(ctx, namespace).get("handoff", {}) + ctx.emit_result({namespace: {"next_iteration": handoff}}) + return {"journey_handoff": handoff} + + +@visual_node( + kind="validate_persona_contract", + group_key="journeys", + display_name="Validate persona contract", + title_key="visual.nodes.validatePersonaContract.title", + description_key="visual.nodes.validatePersonaContract.description", + default_outputs=("persona_contract",), +) +def validate_persona_contract(ctx: Any, _: Mapping[str, Any], __: Mapping[str, object]) -> dict[str, object]: + """Require a browser target and one fixed persona contract.""" + if not ctx.build.get("browser_base_url"): + raise ValueError("An autonomous persona needs a browser base URL") + if not ctx.test_cases: + raise ValueError("An autonomous persona needs one persona contract") + ctx.log("Validated the autonomous persona contract and safe browser boundary") + return {"persona_contract": True} + + +@visual_node( + kind="observe_persona_page", + group_key="journeys", + display_name="Observe persona page", + title_key="visual.nodes.observePersonaPage.title", + description_key="visual.nodes.observePersonaPage.description", + default_config={"namespace": "autonomous_persona"}, + default_outputs=("page_choices",), + required_config=("namespace",), +) +def observe_persona_page(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Capture safe, rendered choices and retain the persona observation.""" + namespace = config["namespace"] + state = _persona_state(ctx, namespace) + observation = _persona_browser( + ctx, + url=str(state.get("last_url") or ctx.build["browser_base_url"]), + screenshot=f"persona-observation-{ctx.loop_index}.png", + ) + state["observation"] = observation + _save_persona_state(ctx, namespace, state) + summary = { + key: state[key] + for key in ("persona", "current_goal", "feeling", "next_intent", "learnings", "reported_issues") + } + ctx.emit_result({"persona": {"iteration": ctx.loop_index, "state": summary, "observation": observation}}) + ctx.log(f"Observed {len(observation['available_actions'])} safe visible action(s)") + return {"page_choices": observation} + + +@visual_node( + kind="plan_persona_action", + group_key="journeys", + display_name="Plan persona action", + title_key="visual.nodes.planPersonaAction.title", + description_key="visual.nodes.planPersonaAction.description", + default_config={"namespace": "autonomous_persona"}, + default_outputs=("persona_plan",), + required_config=("namespace",), +) +def plan_persona_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Let a persona select exactly one action from rendered safe choices.""" + namespace = config["namespace"] + state = _persona_state(ctx, namespace) + actions = state["observation"]["available_actions"] + prompt = ( + """You are a persistent product user. Decide one next action, not a test verdict. Return JSON only: +{"intent":"short first-person purpose","action_id":"one listed id or empty","rationale":"observable reason","expected_signal":"what would change my understanding"}. +You may choose an empty action_id to observe again. Never choose actions outside listed IDs. Avoid repeating a reported issue unless new evidence exists. + +Persona state:\n""" + + json.dumps( + { + key: state[key] + for key in ( + "persona", + "current_goal", + "feeling", + "next_intent", + "learnings", + "reported_issues", + "history", + ) + }, + ensure_ascii=False, + ) + + "\nVisible safe actions:\n" + + json.dumps(actions, ensure_ascii=False) + ) + plan = ctx.complete_model_json(prompt, description="persona model response") + action_id = str(plan.get("action_id") or "") + selected = next((item for item in actions if item["id"] == action_id), None) + if action_id and selected is None: + raise ValueError("persona selected an action outside the visible safe action list") + state["plan"] = { + "intent": str(plan.get("intent") or state["next_intent"]), + "rationale": str(plan.get("rationale") or ""), + "expected_signal": str(plan.get("expected_signal") or ""), + "action": selected, + } + _save_persona_state(ctx, namespace, state) + ctx.emit_result({"persona": {"iteration": ctx.loop_index, "plan": state["plan"]}}) + ctx.log("Planned one persona action from rendered, safe choices") + return {"persona_plan": state["plan"]} + + +@visual_node( + kind="run_persona_action", + group_key="journeys", + display_name="Run persona action", + title_key="visual.nodes.runPersonaAction.title", + description_key="visual.nodes.runPersonaAction.description", + default_config={"namespace": "autonomous_persona"}, + default_outputs=("action_evidence",), + required_config=("namespace",), +) +def run_persona_action(ctx: Any, config: Mapping[str, Any], _: Mapping[str, object]) -> dict[str, object]: + """Follow the selected safe link and retain the resulting rendered page.""" + namespace = config["namespace"] + state = _persona_state(ctx, namespace) + action = state["plan"].get("action") + evidence = _persona_browser( + ctx, + url=state["observation"]["before_url"], + allowed_href=action["href"] if action else None, + screenshot=f"persona-action-{ctx.loop_index}.png", + ) + state["evidence"] = evidence + _save_persona_state(ctx, namespace, state) + ctx.emit_result({"persona": {"iteration": ctx.loop_index, "action_evidence": evidence}}) + ctx.log("Completed one bounded persona action with rendered evidence") + return {"action_evidence": evidence} + + +@visual_node( + kind="reflect_persona_session", + group_key="journeys", + display_name="Reflect persona session", + title_key="visual.nodes.reflectPersonaSession.title", + description_key="visual.nodes.reflectPersonaSession.description", + default_config={"namespace": "autonomous_persona"}, + default_outputs=("persona_handoff",), + required_config=("namespace",), +) +def reflect_persona_session( + ctx: Any, config: Mapping[str, Any], _: Mapping[str, object] +) -> dict[str, object]: + """Persist feelings, learnings, unique issues, and the next safe intent.""" + namespace = config["namespace"] + state = _persona_state(ctx, namespace) + prompt = """You are reflecting as a persistent product user after one safe browser action. Return JSON only: +{"feeling":"brief feeling","learning":"specific observation","issue":{"title":"short or empty","evidence":"observable evidence","severity":"low|medium|high"},"next_intent":"one concrete next action to investigate"}. +Do not repeat a reported issue title unless the new evidence materially differs. + +Current state:\n""" + json.dumps(state, ensure_ascii=False) + reflection = ctx.complete_model_json(prompt, description="persona model response") + issue = reflection.get("issue") if isinstance(reflection.get("issue"), dict) else {} + title = str(issue.get("title") or "").strip() + known = {str(item.get("title", "")).casefold() for item in state["reported_issues"]} + if title and title.casefold() not in known: + state["reported_issues"].append( + { + "title": title, + "evidence": str(issue.get("evidence") or ""), + "severity": str(issue.get("severity") or "low"), + "iteration": ctx.loop_index, + } + ) + learning = str(reflection.get("learning") or "").strip() + if learning and learning not in state["learnings"]: + state["learnings"].append(learning) + state["feeling"] = str(reflection.get("feeling") or state["feeling"]) + state["next_intent"] = str(reflection.get("next_intent") or state["plan"]["intent"]) + state["last_url"] = state["evidence"]["url"] + state["history"].append( + { + "iteration": ctx.loop_index, + "intent": state["plan"]["intent"], + "action": state["plan"].get("action"), + "feeling": state["feeling"], + "learning": learning, + "next_intent": state["next_intent"], + } + ) + _save_persona_state(ctx, namespace, state) + summary = {key: state[key] for key in ("feeling", "next_intent", "learnings", "reported_issues")} + ctx.emit_result( + {"persona": {"iteration": ctx.loop_index, "reflection": reflection, "next_state": summary}} + ) + artifact = ctx.save_data_file( + f"autonomous-persona/iteration-{ctx.loop_index}.json", + json.dumps( + { + "plan": state["plan"], + "evidence": state["evidence"], + "reflection": reflection, + "next_state": summary, + }, + ensure_ascii=False, + indent=2, + ), + label="Autonomous persona handoff", + content_type="application/json", + ) + ctx.log("Retained persona feeling, learning, deduplicated issues, and next intent") + return {"persona_handoff": {"reflection": reflection, "next_state": summary, "artifact": artifact}} diff --git a/backend/orbit_sdk/visual/layout.py b/backend/orbit_sdk/visual/layout.py new file mode 100644 index 0000000..72f0404 --- /dev/null +++ b/backend/orbit_sdk/visual/layout.py @@ -0,0 +1,51 @@ +"""Initial canvas layout shared by Visual Mode's shipped blueprints. + +The read-only workflow preview places lifecycle phases in columns and stacks +steps in each phase. Persist the same starting positions in blueprints, while +leaving later user-driven node movement entirely untouched. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +LIFECYCLE_PHASES = ( + "before_all", + "before_each", + "execute", + "verify", + "after_supervision", + "after_each", + "after_all", +) + +# These match the preview's 340px phase columns and its 48px vertical gap. +# Visual Mode nodes are 140px tall for initial-layout purposes. +_COLUMN_WIDTH = 340 +_LEFT_INSET = 30 +_TOP_INSET = 72 +_NODE_HEIGHT = 140 +_ROW_GAP = 48 + + +def initial_positions(phases: Iterable[str]) -> list[dict[str, int]]: + """Return preview-style initial positions in the supplied node order.""" + values = list(phases) + present = list(dict.fromkeys(values)) + ordered = [ + *[phase for phase in LIFECYCLE_PHASES if phase in present], + *[phase for phase in present if phase not in LIFECYCLE_PHASES], + ] + columns = {phase: index for index, phase in enumerate(ordered)} + rows = {phase: 0 for phase in ordered} + positions: list[dict[str, int]] = [] + for phase in values: + row = rows[phase] + positions.append( + { + "x": _LEFT_INSET + columns[phase] * _COLUMN_WIDTH, + "y": _TOP_INSET + row * (_NODE_HEIGHT + _ROW_GAP), + } + ) + rows[phase] += 1 + return positions diff --git a/backend/orbit_sdk/visual/registry.py b/backend/orbit_sdk/visual/registry.py new file mode 100644 index 0000000..31ee7ac --- /dev/null +++ b/backend/orbit_sdk/visual/registry.py @@ -0,0 +1,123 @@ +"""Declarative registry for SDK-backed visual runner nodes. + +This module deliberately has no FastAPI or frontend dependency. The API server +serializes its catalog, the runner compiler validates against it, and the SDK +runtime dispatches through it. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +from . import bindings + +VisualNodeHandler = Callable[[Any, Mapping[str, Any], Mapping[str, object]], dict[str, object]] +VisualNodeValidator = Callable[[Mapping[str, Any]], None] + + +@dataclass(frozen=True) +class VisualNodeDefinition: + """One SDK-owned operation that may be placed in a visual runner.""" + + kind: str + group_key: str + title_key: str + description_key: str + display_name: str + default_inputs: tuple[str, ...] = () + default_outputs: tuple[str, ...] = () + required_config: tuple[str, ...] = () + config_schema: dict[str, object] | None = None + default_config: dict[str, object] | None = None + handler: VisualNodeHandler | None = None + is_custom_script: bool = False + palette_visible: bool = True + config_validator: VisualNodeValidator | None = None + + def validate_config(self, config: Mapping[str, Any]) -> None: + """Validate the minimum portable configuration for this operation.""" + for name in self.required_config: + value = config.get(name) + if bindings.is_binding(value): + continue + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"requires {name}") + bindings.validate(config) + if self.config_validator: + self.config_validator(config) + + def catalog(self) -> dict[str, object]: + """Return frontend-safe metadata without exposing executable code.""" + return { + "kind": self.kind, + "group_key": self.group_key, + "title_key": self.title_key, + "description_key": self.description_key, + "display_name": self.display_name, + "default_inputs": list(self.default_inputs), + "default_outputs": list(self.default_outputs), + "config_schema": self.config_schema or {"type": "object"}, + "default_config": self.default_config or {}, + "is_custom_script": self.is_custom_script, + "palette_visible": self.palette_visible, + } + + +class VisualNodeRegistry: + """Register and dispatch SDK-owned visual node definitions.""" + + def __init__(self) -> None: + self._definitions: dict[str, VisualNodeDefinition] = {} + + def node(self, **metadata: Any) -> Callable[[VisualNodeHandler], VisualNodeHandler]: + """Register an executable visual node through a decorator.""" + + def register(handler: VisualNodeHandler) -> VisualNodeHandler: + definition = VisualNodeDefinition(handler=handler, **metadata) + if definition.kind in self._definitions: + raise ValueError(f"visual node kind must be unique: {definition.kind}") + self._definitions[definition.kind] = definition + return handler + + return register + + def register(self, definition: VisualNodeDefinition) -> None: + """Register metadata-only nodes such as the Custom Script escape hatch.""" + if definition.kind in self._definitions: + raise ValueError(f"visual node kind must be unique: {definition.kind}") + self._definitions[definition.kind] = definition + + def get(self, kind: str) -> VisualNodeDefinition: + """Return one registered definition or a clear validation error.""" + try: + return self._definitions[kind] + except KeyError as error: + raise ValueError(f"unsupported visual node kind: {kind}") from error + + def validate(self, kind: str, config: Mapping[str, Any]) -> None: + """Validate one node configuration.""" + self.get(kind).validate_config(config) + + def execute( + self, kind: str, ctx: Any, *, config: Mapping[str, Any], inputs: Mapping[str, object] + ) -> dict[str, object]: + """Run one registered node and require its JSON-object output.""" + definition = self.get(kind) + definition.validate_config(config) + handler = definition.handler + if handler is None: + raise ValueError(f"visual node kind {kind!r} has no SDK runtime handler") + result = handler(ctx, config, inputs) + if not isinstance(result, dict): + raise RuntimeError(f"visual node {kind!r} must return a JSON object") + return result + + def catalog(self) -> list[dict[str, object]]: + """Return definitions in stable registration order for clients.""" + return [definition.catalog() for definition in self._definitions.values()] + + +visual_nodes = VisualNodeRegistry() +visual_node = visual_nodes.node diff --git a/backend/orbit_sdk/visual/starters.py b/backend/orbit_sdk/visual/starters.py new file mode 100644 index 0000000..fc49edb --- /dev/null +++ b/backend/orbit_sdk/visual/starters.py @@ -0,0 +1,529 @@ +"""SDK-owned starter blueprints for the visual runner editor.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from .layout import initial_positions + + +def _node( + suffix: str, + kind: str, + title_key: str, + phase: str, + *, + inputs: list[str] | None = None, + outputs: list[str] | None = None, + config: dict[str, Any] | None = None, + x: int = 100, + y: int = 120, +) -> dict[str, Any]: + return { + "id": suffix, + "kind": kind, + "title": suffix.replace("-", " ").replace("_", " ").title(), + "title_key": title_key, + "phase": phase, + "inputs": inputs or [], + "outputs": outputs or [], + "config": config or {}, + "script": "", + "position": {"x": x, "y": y}, + } + + +def _cycle_starter( + starter_id: str, + *, + title_key: str, + description_key: str, + action_kind: str, + command_env: str, + namespace: str, + action_names: tuple[str, str, str, str], + contract_kind: str, + contract_config: dict[str, Any], + payload: dict[str, Any] | None = None, + log_source: str | None = None, + close_message: str, + final_message: str, +) -> dict[str, Any]: + """Build the seven visual steps shared by the shipped external cycles.""" + names = ("validate", "preflight", "prepare", "run", "collect", "close", "finalize") + phases = ("before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all") + outputs = ("contract", "preflight", "prepared", "result", "evidence", "complete", "final") + nodes = [ + _node( + names[0], + contract_kind, + "visual.starters.validate", + phases[0], + outputs=[outputs[0]], + config=contract_config, + ) + ] + result_keys = ("preflight", "prepared", "result", "evidence") + for index, (name, phase, result_key, action) in enumerate( + zip(names[1:5], phases[1:5], result_keys, action_names), 1 + ): + config: dict[str, Any] = { + "command_env": command_env, + "action": action, + "namespace": namespace, + "result_key": result_key, + "timeout": 3600, + "log_source": log_source, + "include_iteration": name != "preflight", + } + if payload is not None: + config.update({"input_env": "ORBIT_CYCLE_INPUT", "input_data": payload}) + nodes.append( + _node( + name, + action_kind, + f"visual.starters.{name}", + phase, + inputs=[outputs[index - 1]], + outputs=[outputs[index]], + config=config, + ) + ) + nodes.extend( + [ + _node( + names[5], + "log_message", + "visual.starters.close", + phases[5], + inputs=[outputs[4]], + outputs=[outputs[5]], + config={"message": close_message}, + ), + _node( + names[6], + "log_message", + "visual.starters.finalize", + phases[6], + inputs=[outputs[5]], + outputs=[outputs[6]], + config={"message": final_message}, + ), + ] + ) + for node, position in zip(nodes, initial_positions(node["phase"] for node in nodes)): + node["position"] = position + edges = [{"source": names[index], "target": names[index + 1], "kind": "execution"} for index in range(6)] + edges.extend( + [ + {"source": "close", "target": "prepare", "kind": "loop", "label": "next cycle"}, + {"source": "close", "target": "finalize", "kind": "condition", "label": "completed"}, + ] + ) + return { + "id": starter_id, + "group_key": "starters", + "title_key": title_key, + "description_key": description_key, + "blueprint": {"schema_version": 1, "nodes": nodes, "edges": edges}, + } + + +_AGENT_PAYLOAD = { + "iteration": {"$ctx": "loop_index"}, + "build": {"$ctx": "build"}, + "test_cases": {"$ctx": "test_cases"}, +} +_PROBE_PAYLOAD = { + "iteration": {"$ctx": "loop_index"}, + "build": {"$ctx": "build"}, + "probes": {"$ctx": "test_cases"}, +} + + +STARTERS: tuple[dict[str, Any], ...] = ( + _cycle_starter( + "json_agent_cycle", + title_key="visual.starters.jsonAgentCycle.title", + description_key="visual.starters.jsonAgentCycle.description", + action_kind="json_cycle_action", + command_env="ORBIT_AGENT_COMMAND", + namespace="agent_cycle", + action_names=("status", "prepare", "run-once", "collect-evidence"), + contract_kind="require_test_cases", + contract_config={"label": "fixed test case"}, + payload=_AGENT_PAYLOAD, + log_source="agent-cycle", + close_message="Completed one bounded external agent cycle", + final_message="Finalized the external agent", + ), + _cycle_starter( + "command_adapter", + title_key="visual.starters.commandAdapter.title", + description_key="visual.starters.commandAdapter.description", + action_kind="command_cycle_action", + command_env="ORBIT_ADAPTER_COMMAND", + namespace="external_adapter", + action_names=("status", "prepare", "run-once", "collect-evidence"), + contract_kind="validate_command_environment", + contract_config={"command_env": "ORBIT_ADAPTER_COMMAND"}, + log_source="external-adapter", + close_message="Completed one bounded external adapter cycle", + final_message="Finalized the external automation evaluation", + ), + _cycle_starter( + "evidence_gated_probe_cycle", + title_key="visual.starters.evidenceGatedProbeCycle.title", + description_key="visual.starters.evidenceGatedProbeCycle.description", + action_kind="json_cycle_action", + command_env="ORBIT_PROBE_COMMAND", + namespace="probe_gate", + action_names=("preflight", "prepare", "run-probes", "collect-evidence"), + contract_kind="require_test_cases", + contract_config={"label": "fixed probe case"}, + payload=_PROBE_PAYLOAD, + log_source="probe-gate", + close_message="Completed one bounded probe matrix cycle", + final_message="Finalized the probe matrix", + ), + { + "id": "browser_journey", + "group_key": "starters", + "title_key": "visual.starters.browserJourney.title", + "description_key": "visual.starters.browserJourney.description", + "blueprint": { + "schema_version": 1, + "nodes": [ + _node( + "validate", + "require_test_cases", + "visual.starters.validateBrowser", + "before_all", + outputs=["contract"], + config={"label": "browser journey case"}, + ), + _node( + "journey", + "browser_journey", + "visual.starters.runBrowser", + "execute", + inputs=["contract"], + outputs=["evidence"], + x=390, + y=290, + ), + _node( + "publish", + "emit_evidence", + "visual.starters.publishBrowser", + "verify", + inputs=["evidence"], + config={"namespace": "browser_journey"}, + x=680, + ), + ], + "edges": [ + {"source": "validate", "target": "journey", "kind": "execution"}, + {"source": "journey", "target": "publish", "kind": "execution"}, + ], + }, + }, + { + "id": "user_journey_cycle", + "group_key": "starters", + "title_key": "visual.starters.userJourneyCycle.title", + "description_key": "visual.starters.userJourneyCycle.description", + "blueprint": { + "schema_version": 1, + "nodes": [ + _node( + "validate_runtime", + "validate_browser_runtime", + "visual.starters.validateBrowser", + "before_all", + outputs=["browser_runtime"], + ), + _node( + "validate_contract", + "require_test_cases", + "visual.starters.validate", + "before_all", + inputs=["browser_runtime"], + outputs=["journey_contract"], + config={"label": "fixed journey case"}, + x=350, + y=290, + ), + _node( + "load_state", + "initialize_user_journey_state", + "visual.starters.loadJourney", + "before_each", + inputs=["journey_contract"], + outputs=["journey_state"], + config={"namespace": "user_journey"}, + x=600, + ), + _node( + "plan", + "plan_user_journey", + "visual.starters.planJourney", + "before_each", + inputs=["journey_state"], + outputs=["journey_plan"], + config={"namespace": "user_journey"}, + x=850, + y=290, + ), + _node( + "run", + "run_user_journey", + "visual.starters.runJourney", + "execute", + inputs=["journey_plan"], + outputs=["journey_evidence"], + config={"namespace": "user_journey"}, + x=1100, + ), + _node( + "review", + "publish_user_journey_handoff", + "visual.starters.reviewJourney", + "verify", + inputs=["journey_evidence"], + outputs=["journey_handoff"], + config={"namespace": "user_journey"}, + x=1350, + y=290, + ), + _node( + "retain", + "log_message", + "visual.starters.retainJourney", + "after_each", + inputs=["journey_handoff"], + outputs=["iteration_complete"], + config={"message": "Closed this bounded user journey"}, + x=1600, + ), + _node( + "finalize", + "log_message", + "visual.starters.finalize", + "after_all", + inputs=["iteration_complete"], + outputs=["final_status"], + config={"message": "Finalized the user journey evaluation"}, + x=1850, + y=290, + ), + ], + "edges": [ + {"source": "validate_runtime", "target": "validate_contract", "kind": "execution"}, + {"source": "validate_contract", "target": "load_state", "kind": "execution"}, + {"source": "load_state", "target": "plan", "kind": "execution"}, + {"source": "plan", "target": "run", "kind": "execution"}, + {"source": "run", "target": "review", "kind": "execution"}, + {"source": "review", "target": "retain", "kind": "execution"}, + {"source": "retain", "target": "plan", "kind": "loop", "label": "next iteration"}, + {"source": "retain", "target": "finalize", "kind": "condition", "label": "completed"}, + ], + }, + }, + { + "id": "site_exploration", + "group_key": "starters", + "title_key": "visual.starters.siteExploration.title", + "description_key": "visual.starters.siteExploration.description", + "blueprint": { + "schema_version": 1, + "nodes": [ + _node( + "validate", + "validate_build_fields", + "visual.starters.validateSite", + "before_all", + outputs=["site_target"], + config={"fields": ["browser_base_url"]}, + ), + _node( + "explore", + "explore_rendered_site", + "visual.starters.exploreSite", + "execute", + inputs=["site_target"], + outputs=["rendered_pages"], + config={"namespace": "site_exploration", "max_clicks": 3}, + x=390, + y=290, + ), + _node( + "review", + "log_message", + "visual.starters.reviewEvidence", + "verify", + inputs=["rendered_pages"], + outputs=["product_review"], + config={"message": "Retained rendered exploration evidence for review"}, + x=680, + ), + _node( + "finalize", + "log_message", + "visual.starters.finalize", + "after_all", + inputs=["product_review"], + outputs=["completed_review"], + config={"message": "Finalized the bounded site exploration review"}, + x=970, + y=290, + ), + ], + "edges": [ + {"source": "validate", "target": "explore", "kind": "execution"}, + {"source": "explore", "target": "review", "kind": "execution"}, + {"source": "review", "target": "finalize", "kind": "execution"}, + ], + }, + }, + { + "id": "source_aware_browser_journey", + "group_key": "starters", + "title_key": "visual.starters.sourceAwareBrowserJourney.title", + "description_key": "visual.starters.sourceAwareBrowserJourney.description", + "blueprint": { + "schema_version": 1, + "nodes": [ + _node( + "validate", + "validate_source_contract", + "visual.starters.validateSource", + "before_all", + outputs=["source_contract"], + config={"namespace": "source_aware_journey"}, + ), + _node( + "run", + "run_source_aware_browser_journey", + "visual.starters.runBrowser", + "execute", + inputs=["source_contract"], + outputs=["browser_evidence"], + config={"namespace": "source_aware_journey"}, + x=390, + y=290, + ), + _node( + "publish", + "log_message", + "visual.starters.publishBrowser", + "verify", + inputs=["browser_evidence"], + outputs=["review_ready"], + config={"message": "Published source context with rendered browser evidence"}, + x=680, + ), + _node( + "close", + "log_message", + "visual.starters.close", + "after_each", + inputs=["review_ready"], + outputs=["cycle_complete"], + config={"message": "Completed one bounded source-aware browser journey"}, + x=970, + y=290, + ), + _node( + "finalize", + "log_message", + "visual.starters.finalize", + "after_all", + inputs=["cycle_complete"], + outputs=["journey_complete"], + config={"message": "Finalized the source-aware browser evaluation"}, + x=1260, + ), + ], + "edges": [ + {"source": "validate", "target": "run", "kind": "execution"}, + {"source": "run", "target": "publish", "kind": "execution"}, + {"source": "publish", "target": "close", "kind": "execution"}, + {"source": "close", "target": "run", "kind": "loop", "label": "next iteration"}, + {"source": "close", "target": "finalize", "kind": "condition", "label": "completed"}, + ], + }, + }, + { + "id": "user_journey_smoke_test", + "group_key": "starters", + "title_key": "visual.starters.userJourneySmokeTest.title", + "description_key": "visual.starters.userJourneySmokeTest.description", + "blueprint": { + "schema_version": 1, + "nodes": [ + _node( + "validate_runtime", + "validate_browser_runtime", + "visual.starters.validateBrowser", + "before_all", + outputs=["browser_target"], + ), + _node( + "validate_contract", + "require_test_cases", + "visual.starters.validate", + "before_all", + inputs=["browser_target"], + outputs=["journey_contract"], + config={"label": "fixed journey case"}, + x=350, + y=290, + ), + _node( + "run", + "run_browser_smoke", + "visual.starters.runBrowser", + "execute", + inputs=["journey_contract"], + outputs=["journey_evidence"], + config={"namespace": "browser_smoke", "fail_on_unpassed": True}, + x=600, + ), + _node( + "verify", + "log_message", + "visual.starters.reviewEvidence", + "verify", + inputs=["journey_evidence"], + outputs=["journey_verdict"], + config={"message": "Quick start browser evaluation completed"}, + x=850, + y=290, + ), + _node( + "finalize", + "log_message", + "visual.starters.finalize", + "after_all", + inputs=["journey_verdict"], + outputs=["completed_evaluation"], + config={"message": "Finalized the one-shot browser evaluation"}, + x=1100, + ), + ], + "edges": [ + {"source": "validate_runtime", "target": "validate_contract", "kind": "execution"}, + {"source": "validate_contract", "target": "run", "kind": "execution"}, + {"source": "run", "target": "verify", "kind": "execution"}, + {"source": "verify", "target": "finalize", "kind": "execution"}, + ], + }, + }, +) + + +def catalog() -> list[dict[str, Any]]: + """Return isolated starter definitions so API consumers cannot mutate SDK data.""" + return deepcopy(list(STARTERS)) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 0ed3f35..1399303 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -27,6 +27,79 @@ def test_health_is_available(): assert response.json() == {"status": "ok"} +def test_visual_runner_catalog_is_served_from_sdk_registry(): + response = TestClient(app).get("/api/visual-runners/catalog") + + assert response.status_code == 200 + catalog = response.json() + assert any(node["kind"] == "custom_script" for node in catalog["nodes"]) + assert catalog["starters"] == [] + assert all(node["group_key"] != "templates" for node in catalog["nodes"]) + + +def test_system_readiness_reports_missing_system_ai_and_git(monkeypatch): + monkeypatch.setattr( + main_module.store, + "application_settings", + lambda: {"chat_model_profile_name": ""}, + ) + monkeypatch.setattr(main_module.store, "profiles", lambda: []) + monkeypatch.setattr(main_module.shutil, "which", lambda _: None) + + response = TestClient(app).get("/api/system/readiness") + + assert response.status_code == 200 + assert response.json() == { + "ready": False, + "checks": [ + { + "id": "system_ai", + "status": "blocked", + "detail": "profile_not_selected", + "settings_page": "settings", + }, + {"id": "git", "status": "blocked", "detail": "not_installed", "settings_page": None}, + ], + } + + +def test_mcp_server_exposes_openapi_backed_control_room_tools(): + headers = {"Accept": "application/json, text/event-stream", "Content-Type": "application/json"} + initialize = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "openorbit-test", "version": "1"}, + }, + } + with TestClient(app, base_url="http://localhost:3000") as client: + connected = client.post("/mcp/", headers=headers, json=initialize) + assert connected.status_code == 200 + session_id = connected.headers["mcp-session-id"] + session_headers = {**headers, "mcp-session-id": session_id} + + tools = client.post( + "/mcp/", + headers=session_headers, + json={"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + ) + resources = client.post( + "/mcp/", + headers=session_headers, + json={"jsonrpc": "2.0", "id": 3, "method": "resources/list", "params": {}}, + ) + + assert tools.status_code == 200 + assert '"name":"get_status"' in tools.text + assert '"name":"start_pipeline"' in tools.text + assert '"name":"act_on_pipeline"' in tools.text + assert resources.status_code == 200 + assert "openorbit://openapi" in resources.text + + def test_request_locale_prefers_the_browser_accept_language_priority(): request = Request( { @@ -50,6 +123,32 @@ def test_generated_sdk_docs_are_served_from_the_local_app(tmp_path, monkeypatch) assert "SDK reference" in response.text +def test_diagnostic_store_does_not_recover_an_active_pipeline(tmp_path, monkeypatch): + """A second store process must not cancel the API server's live run.""" + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + timestamp = store_module.now() + owner = store_module.ConsoleStore() + owner._save( + Run( + id="active-run", + workflow_id="workflow", + workflow_name="Workflow", + status="running", + created_at=timestamp, + updated_at=timestamp, + ) + ) + + diagnostic = store_module.ConsoleStore() + + assert diagnostic.run("active-run").status == "running" + + recovering_server = store_module.ConsoleStore(recover_interrupted_runs=True) + + assert recovering_server.run("active-run").status == "cancelled" + assert recovering_server.run("active-run").step_results[-1]["step_id"] == "orbit-restart" + + def test_cancelling_a_waiting_run_clears_its_current_phase(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") store = store_module.ConsoleStore() @@ -337,83 +436,6 @@ def test_prompt_revisions_returns_immutable_prompt_diff(tmp_path, monkeypatch): assert revisions[2]["after"] == "after\n" -def test_commit_changes_returns_sdk_commit_range(tmp_path, monkeypatch): - monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") - store = store_module.ConsoleStore() - timestamp = store_module.now() - store._save( - Run( - id="commit-run", - workflow_id="workflow", - workflow_name="Workflow", - status="succeeded", - created_at=timestamp, - updated_at=timestamp, - step_results=[ - { - "phase": "execute", - "loop_index": 2, - "ended_at": timestamp.isoformat(), - "result": { - "commit_change": { - "before": "a" * 40, - "after": "b" * 40, - "changed_paths": ["src/agent.py"], - "commits": [{"sha": "b" * 40, "subject": "Improve agent"}], - "diff_artifact": {"relative_path": "commits/a..b.patch"}, - } - }, - } - ], - ) - ) - - changes = store.commit_changes("commit-run") - - assert changes[0]["before"] == "a" * 40 - assert changes[0]["commits"][0]["subject"] == "Improve agent" - assert changes[0]["changed_paths"] == ["src/agent.py"] - - -def test_commit_changes_includes_jgent_committed_source_candidate(tmp_path, monkeypatch): - monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") - store = store_module.ConsoleStore() - timestamp = store_module.now() - store._save( - Run( - id="jgent-commit-run", - workflow_id="workflow", - workflow_name="Jgent", - status="succeeded", - created_at=timestamp, - updated_at=timestamp, - step_results=[ - { - "phase": "before_each", - "loop_index": 1, - "ended_at": timestamp.isoformat(), - "result": { - "jgent_paired": { - "committed_source_candidate": { - "status": "committed_source_candidate", - "before": "a" * 40, - "after": "b" * 40, - "changed_paths": ["src/Jgent/Agent.cs"], - "commits": [{"sha": "b" * 40, "subject": "Improve Jgent"}], - } - } - }, - } - ], - ) - ) - - changes = store.commit_changes("jgent-commit-run") - - assert changes[0]["changed_paths"] == ["src/Jgent/Agent.cs"] - assert changes[0]["commits"][0]["subject"] == "Improve Jgent" - - @pytest.mark.parametrize("terminal_status", ["failed", "cancelled"]) def test_teardown_runs_after_a_failed_or_cancelled_iteration(tmp_path, monkeypatch, terminal_status): monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") @@ -509,9 +531,13 @@ def test_native_improvement_template_uses_repository_snapshot_lifecycle(): if item["id"] == "native-improvement-cycle" ) - assert "ctx.save_before_each_snapshot()" in source - assert "ctx.save_first_after_each_snapshot()" in source - assert "ctx.restore_before_each_snapshot()" in source + from orbit_sdk.visual.builtin_templates import templates + + canonical = templates()["runner-templates:native-improvement-cycle"].source.read_text(encoding="utf-8") + assert "template_runner_templates_native_improvement_cycle_" in source + assert "ctx.save_before_each_snapshot()" in canonical + assert "ctx.save_first_after_each_snapshot()" in canonical + assert "ctx.restore_before_each_snapshot()" in canonical def test_score_select_retains_candidates_and_selects_highest_supervisor_score(tmp_path, monkeypatch): @@ -768,6 +794,37 @@ def test_runner_execution_plan_stops_when_its_run_phase_fails(tmp_path, monkeypa assert workflow.steps_for("test")[0].on_failure == "stop" +def test_builtin_runner_template_creation_persists_its_visual_definition(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") + store = store_module.ConsoleStore() + + runner = store.create_runner( + { + "id": "json-cycle", + "name": "JSON cycle", + "description": "Template-backed visual runner.", + "template_id": "json-agent-cycle", + "source": "# source is replaced by the authoritative template definition\n", + } + ) + + assert runner["visual_template_id"] == "runner-templates:json-agent-cycle" + assert runner["visual_blueprint"]["nodes"] + assert "ctx.run_visual_node('json_cycle_action'" in runner["source"] + assert "ctx.run_visual_node('template_" not in runner["source"] + + detached = store.update_runner( + runner["id"], + { + "name": runner["name"], + "description": runner["description"], + "source": "from orbit_sdk import runner\n@runner.phase('execute')\ndef run(ctx): pass\n", + }, + ) + assert "visual_blueprint" not in detached + assert "visual_template_id" not in detached + + def test_runner_saves_immutable_versions_and_can_resolve_an_older_version(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") store = store_module.ConsoleStore() @@ -797,6 +854,50 @@ def test_runner_saves_immutable_versions_and_can_resolve_an_older_version(tmp_pa assert [step.phase for step in store._runner_execution_plan("versioned-runner", 2).steps] == ["verify"] +def test_direct_code_update_detaches_a_visual_runner_without_erasing_history(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") + store = store_module.ConsoleStore() + blueprint = { + "schema_version": 1, + "nodes": [ + { + "id": "collect", + "kind": "custom_script", + "title": "Collect", + "phase": "execute", + "inputs": [], + "outputs": ["result"], + "config": {}, + "script": "outputs['result'] = True", + "position": {"x": 0, "y": 0}, + } + ], + "edges": [], + } + visual = store.create_runner( + { + "id": "visual-runner", + "name": "Visual runner", + "description": "A generated visual runner.", + "source": "placeholder", + "visual_blueprint": blueprint, + } + ) + updated = store.update_runner( + "visual-runner", + { + "name": visual["name"], + "description": visual["description"], + "source": "from orbit_sdk import runner\n@runner.phase('execute')\ndef run(ctx): pass\n", + }, + ) + + assert visual["versions"][0]["visual_blueprint"]["schema_version"] == 1 + assert "visual_blueprint" not in updated + assert "visual_blueprint" not in updated["versions"][-1] + assert updated["versions"][0]["visual_blueprint"]["nodes"][0]["id"] == "collect" + + def test_bundle_runner_updates_in_place_and_keeps_immutable_versions(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") bundle = store_module.RUNNERS / "bundle-runner" @@ -919,6 +1020,27 @@ def test_deleting_issue_management_items_hides_them_without_deleting_run_evidenc assert store.delete_issue_management_items([proposal["proposal_id"]]) == {"deleted": 0} +def test_issue_management_exposes_an_agent_worktree_diff(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "ISSUE_MANAGEMENT", tmp_path / "issue-management.yaml") + store = store_module.ConsoleStore() + proposal = { + "proposal_id": "run-1:1:agent", + "proposal": { + "kind": "agent_change", + "changed_files": ["prompt.md"], + "base_revision": "abc123", + "diff": "diff --git a/prompt.md b/prompt.md\n", + }, + } + monkeypatch.setattr(store, "proposal_lifecycles", lambda: [proposal]) + + assert store.issue_management_diff(proposal["proposal_id"]) == { + "diff": "diff --git a/prompt.md b/prompt.md\n", + "changed_files": ["prompt.md"], + "base_revision": "abc123", + } + + def test_legacy_saved_runner_is_planned_with_canonical_phases(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") store_module.RUNNERS.mkdir() @@ -961,8 +1083,6 @@ def test_v1_openapi_contract_covers_control_room_assets_and_observability(): "/api/v1/dashboard", "/api/v1/logs", "/api/v1/improvements/analytics", - "/api/v1/improvements/iterations", - "/api/v1/improvements/proposals", "/api/v1/template-translations", } assert expected <= paths.keys() @@ -1076,7 +1196,6 @@ def test_v1_read_only_control_room_resources_are_available(): "/api/v1/telemetry", "/api/v1/logs", "/api/v1/improvements", - "/api/v1/improvements/proposals", "/api/v1/reported-issues", ): assert client.get(path).status_code == 200 @@ -1094,6 +1213,38 @@ def test_supervisor_result_requires_the_two_template_return_keys(): raise AssertionError("invalid supervisor result was accepted") +def test_registered_agent_result_requires_a_supervisor_evaluation(): + with pytest.raises(ValueError, match="registered agent result"): + store_module.ConsoleStore._validated_supervisor_result( + '{"improvements": [], "reported_issues": []}', require_evaluation=True + ) + + +def test_evaluation_request_is_limited_to_the_matching_iteration_and_candidate(): + run = SimpleNamespace( + step_results=[ + { + "loop_index": 1, + "candidate_id": "1-1", + "result": {"evaluation_request": {"subject": "agent_change", "feedback": "Updated retry"}}, + }, + { + "loop_index": 2, + "candidate_id": "2-1", + "result": {"evaluation_request": {"subject": "agent_change", "feedback": "Updated parser"}}, + }, + ] + ) + + assert store_module.ConsoleStore._evaluation_request_for_iteration(run, 2, "2-1") == { + "subject": "agent_change", + "feedback": "Updated parser", + "changed_files": [], + "validation": "", + } + assert store_module.ConsoleStore._evaluation_request_for_iteration(run, 2, "2-2") is None + + def test_supervisor_result_normalizes_a_numeric_string_score(): result = store_module.ConsoleStore._validated_supervisor_result( '{"evaluation":{"score":"8","approval":"pending","summary":"ok"},"improvements":[],"reported_issues":[]}' @@ -1101,6 +1252,19 @@ def test_supervisor_result_normalizes_a_numeric_string_score(): assert result["evaluation"]["score"] == 8.0 +def test_supervisor_result_accepts_supervisor_observed_persona_journeys(): + result = store_module.ConsoleStore._validated_supervisor_result( + '{"persona_journeys":[{"persona_id":"jp_nisa_beginner","behavior_trace":' + '{"persona_goal":"Understand my NISA portfolio",' + '"current_action":"I checked the rendered holdings",' + '"decision":"I did not record a trade while the values disagree",' + '"next_action":"I will verify the displayed allocation",' + '"evidence":"The visible ACWI holding is zero"}}],' + '"improvements":[],"reported_issues":[]}' + ) + assert result["persona_journeys"][0]["persona_id"] == "jp_nisa_beginner" + + def test_supervisor_result_accepts_a_persona_journey_trace(): result = store_module.ConsoleStore._validated_supervisor_result( '{"evaluation":{"score":8,"approval":"pending","summary":"ok","behavior_trace":' @@ -1234,6 +1398,84 @@ def complete(self, _settings, prompt): assert "Prompt evidence" in captured_prompts[0] +def test_supervision_reuses_known_unresolved_issue_without_creating_another_row(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + store = store_module.ConsoleStore() + timestamp = store_module.now() + run = Run( + id="known-issue-run", + workflow_id="workflow", + workflow_name="Workflow", + supervisor_profile_name="Supervisor", + prompt_snapshot="Evaluate the target.", + status="running", + created_at=timestamp, + updated_at=timestamp, + step_results=[{"phase": "execute", "loop_index": 2, "result": {"persona_cycle": {}}}], + supervisor_results=[ + { + "iteration": 1, + "stage": "issue_assessment", + "response": { + "improvements": [ + { + "title": "Portfolio calculation basis is unclear", + "rationale": "Users cannot verify the displayed value.", + "status": "proposed", + } + ], + "reported_issues": [], + }, + } + ], + ) + store._save(run) + captured_prompts = [] + + class FakeProvider: + def complete(self, _settings, prompt): + captured_prompts.append(prompt) + return json.dumps( + { + "improvements": [ + { + "known_issue_id": "known-issue-run:1:0", + "evidence": "The basis is still absent in this iteration.", + "evaluation": {"score": 7, "approval": "pending", "summary": "Known issue."}, + } + ], + "reported_issues": [], + } + ) + + monkeypatch.setattr( + store, + "profiles", + lambda: [ + { + "profile_name": "Supervisor", + "provider": "azure-openai", + "model": "test-model", + "endpoint": "https://example.test/openai/v1", + "region": "us-east-1", + "secret_env": "AZURE_OPENAI_API_KEY", + "aws_profile": "", + "created_at": "2026-01-01T00:00:00+00:00", + } + ], + ) + monkeypatch.setattr(store_module, "AzureOpenAIProvider", FakeProvider) + monkeypatch.setattr(store, "_review_cycle_improvement", lambda *_args: None) + + store._complete_supervision(run.id) + + assert "# Known unresolved issues" in captured_prompts[0] + assert "known-issue-run:1:0" in captured_prompts[0] + assert [item["title"] for item in store.proposal_lifecycles()] == [ + "Portfolio calculation basis is unclear" + ] + + def test_runner_context_uses_the_supplied_model_profile_without_exposing_its_secret(tmp_path, monkeypatch): resources = { "model_profile": { @@ -1293,31 +1535,42 @@ def test_runner_templates_separate_direct_user_journeys_from_external_commands() compile(improvement, "native-improvement-cycle.py", "exec") compile(json_agent, "json-agent-cycle.py", "exec") compile(probe_gate, "evidence-gated-probe-cycle.py", "exec") - assert "playwright_journey" in user_journey + assert "@runner.phase" in user_journey + assert "orbit_runner_kit" not in user_journey assert "ORBIT_ADAPTER_COMMAND" not in user_journey - assert "previous_supervisor_feedback" in user_journey - assert "user-journey-state" in user_journey - assert "ORBIT_ADAPTER_COMMAND" in adapter - assert "playwright_journey" not in improvement - assert "complete_model" in improvement - assert "target_ai_responses" in improvement - assert "ORBIT_CYCLE_COMMAND" not in improvement - assert "run_paired_improvement_cycle" not in improvement - assert "update_prompt_from_accepted_proposals" in improvement - assert "ctx.accept_proposal" not in improvement - assert "ctx.update_file" in improvement - assert "managed_prompt_evidence" in improvement - assert "record_proposal_application" in improvement - assert "no_accepted_proposals" in improvement - assert "ORBIT_AGENT_COMMAND" in json_agent - assert "ORBIT_PROBE_COMMAND" in probe_gate + from orbit_sdk.visual.builtin_templates import templates as canonical_templates + + canonical = canonical_templates() + adapter_source = canonical["runner-templates:external-command-adapter"].source.read_text(encoding="utf-8") + improvement_source = canonical["runner-templates:native-improvement-cycle"].source.read_text( + encoding="utf-8" + ) + assert "template_runner_templates_external_command_adapter_" in adapter + assert "ORBIT_ADAPTER_COMMAND" in adapter_source + assert "playwright_journey" not in improvement_source + assert "complete_model" in improvement_source + assert "target_ai_responses" in improvement_source + assert "ORBIT_CYCLE_COMMAND" not in improvement_source + assert "run_paired_improvement_cycle" not in improvement_source + assert "update_prompt_from_accepted_proposals" in improvement_source + assert "ctx.accept_proposal" not in improvement_source + assert "ctx.update_file" in improvement_source + assert "managed_prompt_evidence" in improvement_source + assert "record_proposal_application" in improvement_source + assert "no_accepted_proposals" in improvement_source + assert "ORBIT_AGENT_COMMAND" in canonical["runner-templates:json-agent-cycle"].source.read_text( + encoding="utf-8" + ) + assert "ORBIT_PROBE_COMMAND" in canonical["runner-templates:evidence-gated-probe-cycle"].source.read_text( + encoding="utf-8" + ) assert "Insighta" not in json_agent assert "Jgent" not in json_agent assert "Insighta" not in probe_gate assert "Jgent" not in probe_gate -def test_site_exploration_quick_start_uses_the_langgraph_runner(): +def test_site_exploration_quick_start_declares_its_lifecycle(): store = store_module.ConsoleStore() quick_start = next( item for item in store._built_in_quick_starts() if item["id"] == "openorbit.site-exploration-review" @@ -1325,8 +1578,14 @@ def test_site_exploration_quick_start_uses_the_langgraph_runner(): runner = quick_start["assets"]["runner"] assert "LangGraph" in quick_start["description"] assert runner["template_id"] == "site-exploration" - assert "StateGraph" in runner["source"] - assert "logout|signout|delete" in runner["source"] + assert "@runner.phase" in runner["source"] + assert "orbit_runner_kit" not in runner["source"] + from orbit_sdk.visual.builtin_templates import templates + + assert "template_quick_starts_openorbit_site_exploration_review_" in runner["source"] + assert "logout|signout|delete" in templates()[ + "quick-starts:openorbit.site-exploration-review" + ].source.read_text(encoding="utf-8") def test_quick_start_workflow_graph_can_be_previewed_before_creation(monkeypatch): @@ -1344,6 +1603,21 @@ def preview(source): assert "@runner.phase" in captured["source"] +def test_shipped_template_graphs_are_previewable_with_unique_nodes(): + """Every shipped template publishes a valid graph for node-based execution.""" + store = store_module.ConsoleStore() + sources = [item["source"] for item in store.runner_templates()] + sources.extend(item["assets"]["runner"]["source"] for item in store._built_in_quick_starts()) + + for source in sources: + definition = store.preview_runner_graph(source) + assert definition is not None + nodes = definition["nodes"] + assert nodes + assert len({node["id"] for node in nodes}) == len(nodes) + assert all(node["phase"] for node in nodes) + + def test_saved_runner_graph_preview_uses_the_selected_version(monkeypatch): store = store_module.ConsoleStore() expected = {"nodes": [{"id": "start"}], "edges": []} @@ -1378,15 +1652,25 @@ def preview(source): @pytest.mark.parametrize( ("quick_start_id", "phases"), [ - ("openorbit.user-journey-smoke-test", ["before_all", "execute", "verify", "after_all"]), + ( + "openorbit.user-journey-smoke-test", + ["before_all", "before_all", "execute", "verify", "after_all"], + ), ("openorbit.site-exploration-review", ["before_all", "execute", "verify", "after_all"]), ( "openorbit.agent-self-improvement", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + [ + "before_all", + "before_all", + "before_each", + "execute", + "after_each", + "after_each", + ], ), ( "openorbit.ai-slo-drift-monitor", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ], ) @@ -1408,20 +1692,48 @@ def test_quick_start_runner_graph_matches_its_execution_purpose(monkeypatch, qui @pytest.mark.parametrize( ("template_id", "phases"), [ - ("user-journey-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "user-journey-cycle", + [ + "before_all", + "before_all", + "before_each", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ], + ), ( "external-command-adapter", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ( "native-improvement-cycle", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + [ + "before_all", + "before_all", + "before_each", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ], ), ("site-exploration", ["before_all", "execute", "verify", "after_all"]), - ("json-agent-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "source-aware-browser-journey", + ["before_all", "execute", "verify", "after_each", "after_all"], + ), + ( + "json-agent-cycle", + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), ( "evidence-gated-probe-cycle", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ], ) @@ -1456,6 +1768,24 @@ def test_ai_slo_drift_quick_start_uses_a_recurring_evidence_gate(): assert "baseline" in quick_start["assets"]["prompt_template"]["content"] +def test_agent_self_improvement_quick_start_embeds_agent_parameters_in_its_runner(): + store = store_module.ConsoleStore() + quick_start = next( + item for item in store._built_in_quick_starts() if item["id"] == "openorbit.agent-self-improvement" + ) + + from orbit_sdk.visual.builtin_templates import templates + + resolved = store._substitute( + templates()["quick-starts:openorbit.agent-self-improvement"].source.read_text(encoding="utf-8"), + {"agent_provider": "claude-code", "agent_options": "--model sonnet"}, + ) + + assert 'AGENT_PROVIDER = "claude-code"' in resolved + assert 'AGENT_OPTIONS = "--model sonnet"' in resolved + assert "agent_provider" not in quick_start["build"] + + def test_ai_slo_drift_quick_start_persists_its_evaluator_command(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "CONFIG", tmp_path) monkeypatch.setattr(store_module, "SETTINGS", tmp_path / "settings.json") @@ -1478,9 +1808,12 @@ def test_ai_slo_drift_quick_start_persists_its_evaluator_command(tmp_path, monke ) execution = store._execution_environment(created["generated"]["execution_environment_id"]) + runner = store._runner(created["generated"]["runner_id"]) assert execution["environment_variables"] == {"ORBIT_PROBE_COMMAND": "uv run ai-eval"} assert created["build"]["repeat_interval_minutes"] == 1440 assert store.profiles()[-1]["endpoint"] == "" + assert runner["visual_template_id"] == "quick-starts:openorbit.ai-slo-drift-monitor" + assert runner["visual_blueprint"]["parameters"]["probe_command"] == "uv run ai-eval" def test_browser_quick_starts_create_an_internal_workspace_without_a_repository(tmp_path, monkeypatch): @@ -1829,6 +2162,25 @@ def test_target_test_case_sets_are_managed_as_assets(tmp_path, monkeypatch): assert updated["name"] == "Updated target tests" +def test_personas_are_managed_as_reusable_assets(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "PERSONAS", tmp_path / "personas.yaml") + store = store_module.ConsoleStore() + values = { + "id": "careful-investor", + "name": "Careful investor", + "locale": "en-US", + "timezone": "America/New_York", + "activity_windows": [{"days": ["mon"], "start": "08:00", "end": "18:00"}], + "definition": "# Goals\n\n- Understand the portfolio safely.\n\n# Constraints\n\n- Never place a real order.", + "context": {"plan": "free"}, + } + created = store.create_persona(values) + assert created["context"] == {"plan": "free"} + assert created["definition"].startswith("# Goals") + updated = store.update_persona("careful-investor", {**values, "name": "Cautious investor"}) + assert updated["name"] == "Cautious investor" + + def test_proposal_history_is_derived_from_evaluation_run_results(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") store = store_module.ConsoleStore() @@ -1882,24 +2234,83 @@ def test_proposal_history_is_derived_from_evaluation_run_results(tmp_path, monke "relative_path": "evidence.json", } ] - assert store.improvement_iteration_data("build-1") == [ - { - "build_id": "build-1", - "build_name": "Build 1", - "run_id": "run-1", - "iteration": 2, - "recorded_at": timestamp.isoformat(), - "data_files": [ + + +def test_issue_management_excludes_agent_assessment_feedback_from_issue_rationales(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + monkeypatch.setattr(store_module, "ISSUE_MANAGEMENT", tmp_path / "issue-management.yaml") + store = store_module.ConsoleStore() + timestamp = store_module.now() + issue = { + "title": "Observed policy claim", + "evaluation": {"approval": "approved", "score": 6, "summary": "Observed in the response."}, + } + store._save( + Run( + id="run-agent-review", + workflow_id="workflow", + workflow_name="Workflow", + status="succeeded", + created_at=timestamp, + updated_at=timestamp, + step_results=[ { - "label": "Iteration evidence", - "filename": "evidence.json", - "path": "/tmp/orbit/evidence.json", - "relative_path": "evidence.json", - "content_type": "", + "loop_index": 1, + "result": { + "agent_run": { + "feedback": "Added a policy guardrail.", + "changed_files": ["prompt.md"], + "proposal": {"fingerprint": "a" * 64}, + }, + "agent_proposal": {"issue": issue}, + }, } ], - } - ] + supervisor_results=[ + { + "iteration": 1, + "stage": "issue_assessment", + "response": { + "improvements": [ + {"title": "Issue proposal", "status": "proposed", "rationale": "Issue reason."} + ], + "reported_issues": [issue], + }, + }, + { + "iteration": 1, + "stage": "agent_proposal_assessment", + "response": { + "evaluation": { + "approval": "rejected", + "score": 2, + "summary": "Agent review summary.", + }, + "improvements": [ + { + "title": "Do not expose this as an Issue", + "status": "proposed", + "rationale": "Agent feedback.", + } + ], + "reported_issues": [], + }, + }, + ], + ) + ) + + lifecycle = store.proposal_lifecycles() + + assert [item["title"] for item in lifecycle] == ["Issue proposal"] + assessed_issue = lifecycle[0] + assert assessed_issue["proposal"]["agent_change"]["feedback"] == "Added a policy guardrail." + assert assessed_issue["proposal"]["agent_change"]["review"]["summary"] == "Agent review summary." + assert lifecycle[0]["decision_rationale"] == "Issue reason." + assert all(item["category"] == "other" for item in lifecycle) + managed_issue = next(item for item in store.issue_management_items() if item["title"] == "Issue proposal") + assert managed_issue["comments"][0]["body"] == "Agent review summary." + assert managed_issue["comments"][0]["assigner"] == "AI supervisor" def test_hello_accepts_unsaved_profile_settings(): @@ -1945,13 +2356,18 @@ def test_application_manager_prompt_is_separate_from_model_profiles(tmp_path, mo "manager_prompt_template": expected_prompt, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": { "workspace_root": str(store_module.ROOT), "file_read_enabled": True, "file_search_enabled": True, "run_process_enabled": True, + "coding_agent_enabled": True, + "ui_context_enabled": True, + "ui_interaction_enabled": True, "terminal_enabled": True, "terminal_visible": True, + "mcp_server_url": "http://127.0.0.1:3000/mcp/", }, } store.save_settings( @@ -1965,6 +2381,9 @@ def test_application_manager_prompt_is_separate_from_model_profiles(tmp_path, mo } ) assert store.application_settings()["manager_prompt_template"] == expected_prompt + assert ( + store.save_application_settings({"coding_agent_provider": "kiro"})["coding_agent_provider"] == "kiro" + ) assert ( store.save_application_settings( { @@ -1993,6 +2412,21 @@ def test_application_manager_prompt_has_a_safe_default(tmp_path, monkeypatch): ) +def test_assistant_mcp_configuration_uses_a_valid_default_and_persists_json(tmp_path, monkeypatch): + mcp_config = tmp_path / "config" / "assistant-mcp.json" + monkeypatch.setattr(store_module, "ASSISTANT_MCP_CONFIG", mcp_config) + store = store_module.ConsoleStore() + + assert store.assistant_mcp_config()["content"] == ( + '{\n "mcpServers": {\n "openorbit": {\n "url": "http://127.0.0.1:3000/mcp/"\n }\n }\n}\n' + ) + saved = store.save_assistant_mcp_config('{"mcpServers": {"orbit": {"command": "uv"}}}') + + assert saved["content"] == '{\n "mcpServers": {\n "orbit": {\n "command": "uv"\n }\n }\n}\n' + with pytest.raises(ValueError, match="valid JSON"): + store.save_assistant_mcp_config("not json") + + def test_exact_legacy_manager_prompt_is_migrated_but_custom_prompt_is_preserved(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "SETTINGS", tmp_path / "settings.json") (tmp_path / "settings.json").write_text( diff --git a/backend/tests/test_assistant_graph.py b/backend/tests/test_assistant_graph.py new file mode 100644 index 0000000..4ad16c3 --- /dev/null +++ b/backend/tests/test_assistant_graph.py @@ -0,0 +1,63 @@ +from app.assistant_graph import DEFAULT_MCP_URL, OrbitAssistantGraph, build_assistant_prompt +from app.providers import ModelSettings + + +class FakeProvider: + def complete(self, settings, prompt): + return f"plain:{prompt}" + + def complete_with_tools(self, settings, prompt, tools, execute): + assert tools == [{"name": "get_status", "description": "status", "parameters": {}}] + return f"tools:{execute('get_status', {})}" + + +class FakeLocalTools: + def definitions(self): + return [] + + def execute(self, name, arguments): + raise AssertionError("local tools should not receive an MCP tool call") + + +class FakeMcpTools: + def definitions(self): + return [{"name": "get_status", "description": "status", "parameters": {}}] + + def execute(self, name, arguments): + assert (name, arguments) == ("get_status", {}) + return '{"health":"ok"}' + + +def test_assistant_graph_routes_mcp_tools_through_the_langgraph_model_node(): + activity = [] + graph = OrbitAssistantGraph( + FakeProvider(), + ModelSettings(), + FakeLocalTools(), + FakeMcpTools(), + on_activity=lambda phase, tool=None: activity.append((phase, tool)), + ) + + assert graph.invoke("status") == 'tools:{"health":"ok"}' + assert activity == [("thinking", None), ("working", "get_status"), ("thinking", None)] + + +def test_assistant_prompt_uses_local_mcp_as_the_control_room_source_of_truth(): + prompt = build_assistant_prompt("What changed?", [("assistant", "Earlier answer")], "ko-KR") + + assert DEFAULT_MCP_URL == "http://127.0.0.1:3000/mcp/" + assert "OpenOrbit MCP tools as the source of truth" in prompt + assert "Execution environment:" in prompt + assert "- OS: " in prompt + assert "- Default shell: " in prompt + assert "- Current working directory: " in prompt + assert "- Current local time: " in prompt + assert "Respond in BCP 47 locale 'ko-KR'." in prompt + assert "Assistant: Earlier answer" in prompt + + +def test_assistant_prompt_prioritizes_ui_tools_for_browser_ui_requests(): + prompt = build_assistant_prompt("What screen is open?", [], "en", ui_enabled=True) + + assert "first call ui_get_context" in prompt + assert "Do not use file tools to infer browser UI state" in prompt diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index 88c053b..bd6fb14 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -13,8 +13,40 @@ def tool_settings(workspace, **overrides): } -def test_disabled_tools_are_not_advertised(tmp_path): - assert AssistantToolExecutor(tool_settings(tmp_path)).definitions() == [] +def test_coding_agent_is_advertised_even_without_a_selected_provider(tmp_path): + tools = AssistantToolExecutor(tool_settings(tmp_path)) + + assert [definition["name"] for definition in tools.definitions()] == ["coding_agent"] + result = json.loads(tools.execute("coding_agent", {"task": "inspect this project"})) + assert result["error_code"] == "coding_agent_not_configured" + assert result["settings_page"] == "settings" + + +def test_coding_agent_can_be_disabled_in_assistant_tool_settings(tmp_path): + tools = AssistantToolExecutor(tool_settings(tmp_path, coding_agent_enabled=False)) + + assert tools.definitions() == [] + + +def test_coding_agent_passes_workspace_environment_to_selected_cli(tmp_path, monkeypatch): + captured = {} + + def fake_run(command, **kwargs): + captured["command"] = command + captured.update(kwargs) + return type("Result", (), {"stdout": "done", "stderr": "", "returncode": 0})() + + monkeypatch.setattr("app.assistant_tools.shutil.which", lambda _: "/usr/bin/kiro-cli") + monkeypatch.setattr("app.assistant_tools.subprocess.run", fake_run) + tools = AssistantToolExecutor(tool_settings(tmp_path), coding_agent_provider="kiro") + + result = json.loads(tools.execute("coding_agent", {"task": "Update the README"})) + + assert captured["command"][:2] == ["kiro-cli", "chat"] + assert f"- Current working directory: {tmp_path}" in captured["command"][-1] + assert "- OS: " in captured["command"][-1] + assert captured["cwd"] == tmp_path + assert result["provider"] == "kiro" def test_file_read_is_bounded_to_workspace_and_line_range(tmp_path): diff --git a/backend/tests/test_assistant_ui.py b/backend/tests/test_assistant_ui.py new file mode 100644 index 0000000..a727570 --- /dev/null +++ b/backend/tests/test_assistant_ui.py @@ -0,0 +1,60 @@ +import json +import threading + +from app.assistant_ui import AssistantUiBroker, AssistantUiToolExecutor + + +def test_ui_broker_round_trips_a_tool_command_to_the_browser_session(): + broker = AssistantUiBroker() + sent = {} + sent_event = threading.Event() + + def send(command_id, command): + sent.update(id=command_id, command=command) + sent_event.set() + + broker.open("browser-1", send) + result = {} + + def request(): + result.update(broker.request("browser-1", {"type": "get_context"}, timeout=1)) + + thread = threading.Thread(target=request) + thread.start() + assert sent_event.wait(1) + assert sent["command"] == {"type": "get_context"} + assert broker.resolve("browser-1", sent["id"], {"ok": True, "revision": 3}) + thread.join(1) + assert result == {"ok": True, "revision": 3} + + +def test_ui_tool_executor_returns_browser_context_as_a_tool_result(): + broker = AssistantUiBroker() + sent = {} + sent_event = threading.Event() + + def send(command_id, command): + sent.update(id=command_id, command=command) + sent_event.set() + + broker.open("browser-1", send) + tools = AssistantUiToolExecutor(broker, "browser-1", context_enabled=True, interaction_enabled=True) + result = {} + + def execute(): + result["value"] = json.loads(tools.execute("ui_get_context", {"component_id": "runs"})) + + thread = threading.Thread(target=execute) + thread.start() + assert sent_event.wait(1) + assert sent["command"] == {"type": "get_context", "component_id": "runs"} + broker.resolve("browser-1", sent["id"], {"ok": True, "components": []}) + thread.join(1) + assert result["value"] == {"ok": True, "components": []} + + +def test_ui_context_and_interaction_tools_can_be_enabled_separately(): + broker = AssistantUiBroker() + read_only = AssistantUiToolExecutor(broker, "browser-1", context_enabled=True, interaction_enabled=False) + + assert [tool["name"] for tool in read_only.definitions()] == ["ui_get_context"] diff --git a/backend/tests/test_orbit_sdk.py b/backend/tests/test_orbit_sdk.py index 3cae02e..974458a 100644 --- a/backend/tests/test_orbit_sdk.py +++ b/backend/tests/test_orbit_sdk.py @@ -54,6 +54,26 @@ def collect() -> None: assert graph.definition()["nodes"][0]["phase"] == "before_each" +def test_graph_exports_after_supervision_only_when_enabled(): + graph = sdk.Graph() + + @graph.step("propose", phase="after_each", after_supervision=True) + def propose() -> None: + pass + + assert graph.definition()["nodes"][0]["after_supervision"] is True + + +def test_visual_node_inputs_only_returns_declared_port_bindings(tmp_path, monkeypatch): + monkeypatch.setattr(sdk, "ORBIT_APP_DATA", tmp_path / "orbit-data") + ctx = context(tmp_path, iteration=1) + ctx.publish_visual_node_outputs("collect", {"score": 100, "private": "no"}) + ctx.publish_visual_node_outputs("other", {"score": 1}) + + assert ctx.visual_node_inputs("report", {"journey_score": ("collect", "score")}) == {"journey_score": 100} + assert ctx.visual_node_inputs("report", {"missing": ("collect", "missing")}) == {} + + def test_function_trace_emits_successful_function_evidence(tmp_path, capsys): ctx = context(tmp_path, iteration=1) @@ -63,6 +83,77 @@ def test_function_trace_emits_successful_function_evidence(tmp_path, capsys): assert "collect-source-evidence" in capsys.readouterr().out +def test_register_evaluation_emits_an_agent_change_request(tmp_path, capsys): + result = context(tmp_path, iteration=2).register_evaluation( + "Updated the retry behavior after reproducing the timeout.", + changed_files=["src/retry.py", "src/retry.py", "tests/test_retry.py"], + validation="pytest tests/test_retry.py", + ) + + emitted = [ + json.loads(line.removeprefix("__ORBIT_RESULT__")) + for line in capsys.readouterr().out.splitlines() + if line.startswith("__ORBIT_RESULT__") + ] + assert result["subject"] == "agent_change" + assert result["changed_files"] == ["src/retry.py", "tests/test_retry.py"] + assert emitted[-1]["evaluation_request"] == result + + +def test_register_evaluation_requires_actionable_feedback(tmp_path): + with pytest.raises(ValueError, match="feedback"): + context(tmp_path, iteration=1).register_evaluation(" ") + + +def test_run_ai_agent_uses_the_selected_cli_and_only_marker_feedback(tmp_path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + subprocess.run(["git", "init"], cwd=project, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=project, check=True) + subprocess.run(["git", "config", "user.name", "Orbit Test"], cwd=project, check=True) + (project / "README.md").write_text("test\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run(["git", "commit", "-m", "initial"], cwd=project, check=True, capture_output=True) + monkeypatch.setattr(sdk, "ORBIT_APP_DATA", tmp_path / "orbit-data") + ctx = context(project, iteration=1) + commands: list[list[str]] = [] + + def fake_exec(command, **_kwargs): + commands.append(command) + if command[0] == "codex": + return "completed work\nORBIT_AGENT_FEEDBACK: Updated retry behavior\n" + if command[:3] == ["git", "diff", "--name-only"]: + return "src/retry.py\n" + if command[:3] == ["git", "diff", "--binary"]: + return "diff --git a/src/retry.py b/src/retry.py\n" + return "" + + monkeypatch.setattr(ctx, "exec", fake_exec) + + result = ctx.run_ai_agent("Fix the retry behavior", provider="codex", options="--model gpt-5") + + assert commands[0][:4] == ["git", "worktree", "add", "-b"] + assert commands[0][5].startswith(str(tmp_path / "orbit-data" / "agent-worktrees")) + assert commands[1:] == [ + [ + "codex", + "exec", + "--approve-for-me", + "--model", + "gpt-5", + "Fix the retry behavior", + ], + ["git", "add", "--intent-to-add", "--all"], + ["git", "diff", "--name-only", "--"], + ["git", "diff", "--binary", "--"], + ] + assert result["feedback"] == "Updated retry behavior" + assert result["changed_files"] == ["src/retry.py"] + assert result["proposal"]["diff"] == "diff --git a/src/retry.py b/src/retry.py\n" + assert result["proposal"]["base_revision"] + assert result["proposal"]["branch"].startswith("orbit/agent-proposal/run-123/") + + def test_graph_step_automatically_traces_its_execution(tmp_path, capsys): graph = sdk.Graph() @@ -324,6 +415,77 @@ def test_exec_env_override(tmp_path, monkeypatch): assert output.strip() == "set" +def test_run_json_action_uses_stdout_without_stderr_and_passes_cycle_input(tmp_path, capsys): + project = tmp_path / "project" + project.mkdir() + script = ( + "import json, os, sys; " + "print(json.dumps({'action': sys.argv[1], 'input': json.loads(os.environ['RUNNER_CYCLE_INPUT'])})); " + "print('diagnostic output', file=sys.stderr)" + ) + ctx = sdk.RunnerContext( + phase="execute", + target_repository=project, + mode="run", + loop_index=1, + environment={ + "ORBIT_RUN_ID": "run-123", + "RUNNER_COMMAND": json.dumps([sys.executable, "-c", script]), + }, + ) + + result = ctx.run_json_action( + command_env="RUNNER_COMMAND", + action="collect", + input_env="RUNNER_CYCLE_INPUT", + input_data={"iteration": 1}, + log_source="test-command", + ) + + assert result == {"action": "collect", "input": {"iteration": 1, "action": "collect"}} + assert "diagnostic output" in capsys.readouterr().out + + +@pytest.mark.parametrize("value", ["[]", '[""]']) +def test_command_from_env_rejects_an_empty_executable(tmp_path, value): + with pytest.raises(ValueError, match="non-empty"): + sdk.RunnerContext( + phase="execute", + target_repository=tmp_path, + mode="run", + loop_index=1, + environment={"RUNNER_COMMAND": value}, + ).command_from_env("RUNNER_COMMAND") + + +def test_complete_model_json_requires_a_json_object(tmp_path, monkeypatch): + ctx = context(tmp_path, iteration=1) + monkeypatch.setattr(ctx, "complete_model", lambda _prompt: {"response": '{"next": "visit"}'}) + + assert ctx.complete_model_json("Choose an action") == {"next": "visit"} + + monkeypatch.setattr(ctx, "complete_model", lambda _prompt: {"response": "[]"}) + with pytest.raises(RuntimeError, match="must return a JSON object"): + ctx.complete_model_json("Choose an action") + + +def test_require_test_case_ids_reports_missing_cases(tmp_path): + resources = {"test_cases": [{"id": "included"}]} + ctx = sdk.RunnerContext( + phase="execute", + target_repository=tmp_path, + mode="run", + loop_index=1, + environment={ + "ORBIT_RUNNER_RESOURCES": b64encode(json.dumps(resources).encode()).decode(), + }, + ) + + ctx.require_test_case_ids({"included"}) + with pytest.raises(ValueError, match="Missing required test case: absent"): + ctx.require_test_case_ids({"included", "absent"}) + + def test_repository_snapshot_restores_worktree_index_and_head_without_a_commit(tmp_path, monkeypatch): project = tmp_path / "project" project.mkdir() diff --git a/backend/tests/test_visual_runners.py b/backend/tests/test_visual_runners.py new file mode 100644 index 0000000..9797f94 --- /dev/null +++ b/backend/tests/test_visual_runners.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import ast + +import pytest +from app.visual_runners import generate_source, validate_blueprint +from orbit_sdk.visual import builtin_template_catalog, starter_catalog, visual_nodes +from orbit_sdk.visual.bindings import evaluate, resolve +from orbit_sdk.visual.builtin_templates import definitions, instantiate_definition, templates +from orbit_sdk.visual.layout import initial_positions + + +def test_initial_visual_layout_matches_the_phase_columns_in_workflow_preview(): + assert initial_positions(["before_all", "execute", "execute", "after_all"]) == [ + {"x": 30, "y": 72}, + {"x": 370, "y": 72}, + {"x": 370, "y": 260}, + {"x": 710, "y": 72}, + ] + + +def blueprint() -> dict: + return { + "schema_version": 1, + "nodes": [ + { + "id": "score_journey", + "kind": "custom_script", + "title": "Score journey", + "phase": "verify", + "outputs": ["score"], + "config": {}, + "script": "outputs['score'] = 100", + "position": {"x": 10, "y": 20}, + } + ], + "edges": [], + } + + +def test_visual_runner_source_is_explicit_and_compilable(): + source = generate_source(blueprint()) + + compile(source, "visual-runner.py", "exec") + assert "@graph.step('score_journey'" in source + assert "@runner.phase('verify')" in source + assert "@runner.phase('verify', step_id='score_journey')" in source + assert "ctx.publish_visual_node_outputs('score_journey', outputs)" in source + + +def test_visual_runner_source_uses_python_literals_for_node_config(): + value = blueprint() + value["nodes"][0].update({"kind": "browser_journey", "config": {"fail_on_unpassed": True}}) + + source = generate_source(value) + + compile(source, "visual-runner.py", "exec") + assert "'fail_on_unpassed': True" in source + + +def test_visual_runner_rejects_unknown_nodes_and_edges(): + invalid = blueprint() + invalid["nodes"][0]["kind"] = "shell" + with pytest.raises(ValueError, match="unsupported"): + validate_blueprint(invalid) + + invalid = blueprint() + invalid["edges"] = [{"source": "missing", "target": "score_journey"}] + with pytest.raises(ValueError, match="existing"): + validate_blueprint(invalid) + + +def test_visual_runner_data_edges_bind_declared_ports_only(): + value = blueprint() + value["nodes"] = [ + {**value["nodes"][0], "id": "collect", "phase": "execute", "outputs": ["score"]}, + {**value["nodes"][0], "id": "report", "inputs": ["journey_score"], "outputs": []}, + ] + value["edges"] = [ + { + "source": "collect", + "target": "report", + "kind": "data", + "source_port": "score", + "target_port": "journey_score", + } + ] + + source = generate_source(value) + + assert "inputs=['journey_score']" in source + assert "ctx.visual_node_inputs('report', {'journey_score': ('collect', 'score')})" in source + + +def test_visual_runner_rejects_invalid_data_ports_and_noncanonical_loops(): + invalid = blueprint() + invalid["nodes"][0]["inputs"] = ["score"] + invalid["edges"] = [ + { + "source": "score_journey", + "target": "score_journey", + "kind": "data", + "source_port": "missing", + "target_port": "score", + } + ] + with pytest.raises(ValueError, match="declared"): + validate_blueprint(invalid) + + invalid = blueprint() + invalid["edges"] = [{"source": "score_journey", "target": "score_journey", "kind": "loop"}] + with pytest.raises(ValueError, match="after_each"): + validate_blueprint(invalid) + + +@pytest.mark.parametrize( + ("kind", "config"), + [ + ("command_action", {"command_env": "ORBIT_TOOL", "action": "check"}), + ("browser_journey", {}), + ("state_load", {"name": "journey"}), + ("state_save", {"name": "journey"}), + ("model_prompt", {"prompt": "Review this"}), + ("model_json", {"prompt": "Return JSON"}), + ("ai_agent", {"prompt": "Fix it", "provider": "codex"}), + ("require_test_cases", {}), + ("snapshot", {"operation": "save_before_each"}), + ], +) +def test_visual_runner_generates_sdk_backed_node_kinds(kind, config): + value = blueprint() + value["nodes"][0].update({"kind": kind, "config": config}) + + source = generate_source(value) + + compile(source, "visual-runner.py", "exec") + assert f"ctx.run_visual_node({kind!r}" in source + + +def test_sdk_catalog_owns_node_metadata_and_starter_blueprints(): + nodes = {item["kind"]: item for item in visual_nodes.catalog()} + assert nodes["browser_journey"]["title_key"] == "visual.nodes.browserJourney.title" + assert nodes["custom_script"]["is_custom_script"] is True + assert {starter["id"] for starter in starter_catalog()} >= { + "json_agent_cycle", + "command_adapter", + "browser_journey", + "user_journey_cycle", + } + for starter in starter_catalog(): + source = generate_source(starter["blueprint"]) + assert "@graph.step" in source + assert "@runner.phase" in source + + json_source = generate_source( + next(item for item in starter_catalog() if item["id"] == "json_agent_cycle")["blueprint"] + ) + assert "step_id='preflight'" in json_source + assert "ctx.run_visual_node('json_cycle_action'" in json_source + + +def test_every_shipped_runner_and_quick_start_has_a_canonical_visual_starter(): + starters = builtin_template_catalog() + ids = {starter["id"] for starter in starters} + assert len(starters) == 18 + assert sum(item.startswith("builtin:runner-templates:") for item in ids) == 11 + assert sum(item.startswith("builtin:quick-starts:") for item in ids) == 7 + assert "builtin:runner-templates:native-improvement-cycle" in ids + assert "builtin:quick-starts:openorbit.continuous-user-journey" in ids + for starter in starters: + source = generate_source(starter["blueprint"]) + assert "ctx.run_visual_node('template_" not in source + assert "step_id=" in source + + +def test_every_canonical_step_has_an_explicit_sdk_owned_visual_node(): + registered = {item["kind"] for item in visual_nodes.catalog()} + for definition in definitions().values(): + for node in definition.blueprint["nodes"]: + assert node["kind"] in registered + visual_nodes.validate(node["kind"], node["config"]) + + +def test_template_definitions_preserve_canonical_lifecycle_step_ids_and_edges(): + for template in templates().values(): + definition = definitions()[template.id] + assert {node["id"] for node in definition.blueprint["nodes"]} == {step.id for step in template.steps} + assert definition.blueprint["edges"] == list(template.edges) + source = generate_source(definition.blueprint) + for step in template.steps: + assert f"@graph.step({step.id!r}" in source + assert f"step_id={step.id!r}" in source + + +def test_shipped_template_graphs_connect_each_lifecycle_step(): + for template in templates().values(): + incoming = {step.id: 0 for step in template.steps} + for edge in template.edges: + if edge["target"] in incoming: + incoming[edge["target"]] += 1 + disconnected = [ + step.id for step in template.steps if step.phase != "before_all" and incoming[step.id] == 0 + ] + assert not disconnected, f"{template.id} has disconnected steps: {disconnected}" + + +def test_template_definitions_use_only_reusable_material_nodes(): + for definition in definitions().values(): + kinds = {node["kind"] for node in definition.blueprint["nodes"]} + assert not any(kind.startswith("template_") for kind in kinds) + assert "builtin_template_step" not in kinds + + +def test_visible_catalog_contains_only_composable_material_groups(): + groups = {item["group_key"] for item in visual_nodes.catalog() if item["palette_visible"]} + assert groups == { + "actions", + "advanced", + "ai", + "browser", + "contracts", + "evidence", + "improvement", + "journeys", + "state", + } + + +def test_quick_start_parameters_bind_only_declared_blueprint_values(): + definition = definitions()["quick-starts:openorbit.agent-self-improvement"] + blueprint = instantiate_definition(definition, {"agent_provider": "claude", "agent_options": "--fast"}) + proposal = next(node for node in blueprint["nodes"] if node["id"] == "propose-agent-change") + assert proposal["config"] == {"provider": "claude", "options": "--fast"} + + +def test_template_definitions_generate_valid_runner_sources(): + for definition in definitions().values(): + ast.parse(generate_source(definition.blueprint)) + + +def test_visual_runtime_bindings_resolve_only_context_inputs_and_resources(): + class Context: + loop_index = 4 + build = {"id": "build-1", "browser_base_url": "https://example.test"} + test_cases = [{"id": "case-1"}] + previous_supervisor_feedback = {"reported_issues": ["one"]} + current_issue_assessment = {} + environment = {"ORBIT_RUN_ID": "run-1"} + phase = "execute" + + def resource(self, name, default): + return {"model_profile": {"model": "test-model"}}.get(name, default) + + def load_state(self, name, default, *, scope): + assert (name, scope) == ("journey", "runner") + return {"next_case": 2} + + context = Context() + value = resolve( + context, + { + "iteration": {"$ctx": "loop_index"}, + "base_url": {"$ctx": "build.browser_base_url"}, + "case": {"$input": "selected.id"}, + "model": {"$resource": "model_profile.model"}, + "state": {"$state": {"name": "journey", "path": "next_case"}}, + }, + {"selected": {"id": "case-1"}}, + ) + + assert value == { + "iteration": 4, + "base_url": "https://example.test", + "case": "case-1", + "model": "test-model", + "state": 2, + } + assert evaluate(context, {"$equals": [{"$ctx": "loop_index"}, 4]}, {}) is True + + +def test_visual_runner_generates_a_safe_node_condition(): + value = blueprint() + value["nodes"][0]["when"] = {"$exists": {"$ctx": "build.browser_base_url"}} + + source = generate_source(value) + + assert "ctx.visual_should_run" in source + assert "return" in source diff --git a/docs/API.md b/docs/API.md index 0c50cb0..d5e9082 100644 --- a/docs/API.md +++ b/docs/API.md @@ -6,6 +6,18 @@ each execution is a **pipeline**. The interactive OpenAPI documentation is available from a running server at `/api/docs`; the raw contract is `/api/openapi.json`. +## MCP server + +The same local process exposes a Streamable HTTP [Model Context Protocol +(MCP)](https://modelcontextprotocol.io/) endpoint at `/mcp/`. Point an MCP +client at `http://localhost:3000/mcp/`. It provides project and pipeline +inspection, pipeline start and approval actions, improvement lifecycle lookup, +and an emergency-stop tool. The `openorbit://openapi` MCP resource exposes the +generated OpenAPI contract for the full v1 HTTP API. + +MCP has the same local-first security boundary as the HTTP API: it has no +built-in authentication and must not be exposed directly to untrusted networks. + ## Local-first security The API has no built-in authentication in this release. It is intended for the diff --git a/examples/ai-slo-supportops/slo_probe_agent.py b/examples/ai-slo-supportops/slo_probe_agent.py index efda920..1beed20 100644 --- a/examples/ai-slo-supportops/slo_probe_agent.py +++ b/examples/ai-slo-supportops/slo_probe_agent.py @@ -83,18 +83,18 @@ def evidence() -> dict[str, object]: def main() -> None: action = sys.argv[1] if len(sys.argv) > 1 else "" - if action == "preflight": + if action in {"preflight", "status"}: result = {"health": request("/health"), "metrics": request("/api/metrics")} elif action == "prepare": result = {"status": "ready", "configured_probe_matrix": 2} - elif action == "run-probes": + elif action in {"run-probes", "run-once"}: result = probes() RUNTIME.mkdir(exist_ok=True) REPORT.write_text(json.dumps(result, indent=2)) elif action == "collect-evidence": result = evidence() else: - raise SystemExit("use preflight, prepare, run-probes, or collect-evidence") + raise SystemExit("use preflight/status, prepare, run-probes/run-once, or collect-evidence") print(json.dumps(result, ensure_ascii=False)) diff --git a/examples/external-command-runner.py b/examples/external-command-runner.py index ed99bb1..1b572f7 100644 --- a/examples/external-command-runner.py +++ b/examples/external-command-runner.py @@ -7,33 +7,16 @@ from __future__ import annotations -import json -import os -import shlex - from orbit_sdk import runner -def adapter_command(): - """Read explicit command configuration without coupling to a repository path.""" - configured = os.environ.get("ORBIT_ADAPTER_COMMAND", "").strip() - if not configured: - raise ValueError("Set ORBIT_ADAPTER_COMMAND to an external tool command") - if configured.startswith("["): - command = json.loads(configured) - if not isinstance(command, list) or not all(isinstance(item, str) for item in command): - raise ValueError("ORBIT_ADAPTER_COMMAND JSON must be an array of strings") - return command - return shlex.split(configured) - - def invoke(ctx, action): """Run exactly one adapter action and preserve its output as run evidence.""" - output = ctx.exec( - [*adapter_command(), action], - cwd=ctx.project_root, + output = ctx.run_command_action( + command_env="ORBIT_ADAPTER_COMMAND", + action=action, timeout=3_600, - target_log_source="external-adapter", + log_source="external-adapter", ) artifact = ctx.write_artifact( f"external-command/{action}.log", output, content_type="text/plain; charset=utf-8" diff --git a/frontend/e2e/orbit-ui.spec.ts b/frontend/e2e/orbit-ui.spec.ts index 99c6d62..07fa0c4 100644 --- a/frontend/e2e/orbit-ui.spec.ts +++ b/frontend/e2e/orbit-ui.spec.ts @@ -37,6 +37,19 @@ test('collapses the sidebar and preserves the preference after reload', async ({ await page.getByRole('button', { name: 'Expand navigation' }).click() }) +test('uses the default local MCP connection through the Orbit assistant', async ({ page }) => { + await page.goto('/dashboard') + await page.getByRole('button', { name: 'Open Orbit assistant' }).click() + const assistant = page.getByLabel('Orbit assistant') + await assistant.getByRole('textbox', { name: 'Message' }).fill( + 'Use the OpenOrbit MCP get_status tool and briefly report the local service health.', + ) + await assistant.getByRole('button', { name: 'Send message' }).click() + const response = assistant.locator('.chat-message--assistant').last() + await expect(response).not.toContainText('Chat request failed.', { timeout: 30_000 }) + await expect(response).toContainText(/health|healthy|status|ok/i, { timeout: 30_000 }) +}) + test('opens the build and profile editors with editable lifecycle fields', async ({ page }) => { await page.goto('/builds') await expect(page.getByRole('heading', { name: 'Builds' })).toBeVisible() diff --git a/frontend/package.json b/frontend/package.json index 28331b9..a038f1b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "agent-improvement-console-ui", "private": true, - "version": "0.11.0", + "version": "0.12.0", "type": "module", "scripts": { "dev": "vite", @@ -11,6 +11,8 @@ }, "dependencies": { "@codemirror/lang-html": "^6.4.12", + "@codemirror/lang-json": "^6.0.2", + "@codemirror/lang-markdown": "^6.5.2", "@codemirror/lang-python": "^6.2.1", "@codemirror/lang-yaml": "^6.1.3", "@codemirror/theme-one-dark": "^6.1.3", @@ -20,11 +22,14 @@ "@xterm/addon-fit": "^0.10.0", "@xterm/xterm": "^5.5.0", "@xyflow/react": "^12.11.6", + "locale-codes": "^2.0.0", "lucide-react": "latest", "react": "latest", "react-dom": "latest", "react-icons": "^5.7.0", "react-markdown": "^10.1.0", + "react-select": "^5.10.2", + "react-timezone-select": "^3.3.3", "recharts": "^3.10.1", "remark-gfm": "^4.0.1", "vite": "latest" diff --git a/frontend/src/app/app-shell.tsx b/frontend/src/app/app-shell.tsx index 30ee7c0..0af5d51 100644 --- a/frontend/src/app/app-shell.tsx +++ b/frontend/src/app/app-shell.tsx @@ -7,13 +7,13 @@ import { FileCode2, PanelLeftClose, Play, - CircleAlert, + TriangleAlert, Settings, Sparkles, } from "lucide-react"; import { SiGithub } from "react-icons/si"; import { useEffect, useState, type ReactNode } from "react"; -import type { Page } from "../domain/models"; +import type { Page, SystemReadiness } from "../domain/models"; import type { Locale } from "../locales"; import { localeMessages, locales } from "../locales"; import { ChatAssistant } from "../components/chat-assistant"; @@ -27,6 +27,7 @@ export function AppShell({ theme, headerAction, activeRunCount = 0, + readiness, children, }: { page: Page; @@ -35,6 +36,7 @@ export function AppShell({ theme: string; headerAction?: ReactNode; activeRunCount?: number; + readiness: SystemReadiness | null; children: ReactNode; }) { const t = locales[locale]; @@ -61,7 +63,6 @@ export function AppShell({ ["builds", , t.builds], ["runs", , t.runs], ["improvements", , t.improvements], - ["issues", , t.issues], ["settings", , t.settings], ]; const dashboardRepositoryLinks = page === "dashboard" && ( @@ -120,15 +121,22 @@ export function AppShell({ githubLabel: string; githubTitle: string; }>(locale, "navigationAccessibility"); + const readinessCopy = localeMessages<{ + title: string; + description: string; + openSettings: string; + checks: Record }>; + }>(locale, "systemReadiness"); + const blockedChecks = readiness?.checks.filter((check) => check.status === "blocked") ?? []; const sidebarLabel = collapsed ? navigationLabels.expandNavigation : navigationLabels.collapseNavigation; return (
-