diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..a5848ac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug Report +description: Report a reproducible problem or unexpected error in OpenOrbit +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report an issue! Please provide as much detail as possible to help us investigate and resolve it quickly. + - type: textarea + id: description + attributes: + label: Problem Description + description: A clear and concise description of the bug. + placeholder: Describe what happened... + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to Reproduce + description: Exact steps to reproduce the behavior. + placeholder: | + 1. Start the application with '...' + 2. Navigate to '...' + 3. Click on '...' + 4. Observe error + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + placeholder: What did you expect to happen? + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual Behavior + description: What actually occurred, including any error messages or unexpected states. + placeholder: What happened instead? + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment & Versions + description: Relevant environment information (OS, Node version, Python version, browser, OpenOrbit version). + placeholder: | + - OS: [e.g. Linux Ubuntu 22.04 / macOS 14 / Windows 11] + - Python version: [e.g. 3.13.0] + - Node.js version: [e.g. 24.0.0] + - OpenOrbit version: [e.g. v0.8.0 or commit hash] + validations: + required: true + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context, logs, terminal outputs, or screenshots about the problem here. + placeholder: Optional additional logs or screenshots... + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..fde139f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,41 @@ +name: Feature Request +description: Propose a new idea, enhancement, or capability for OpenOrbit +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thank you for suggesting a feature! Please provide details about what problem this feature solves and how it should work. + - type: textarea + id: problem + attributes: + label: Problem Statement / Motivation + description: Is your feature request related to a problem or limitation? Please describe. + placeholder: A clear and concise description of what the problem or use case is... + validations: + required: true + - type: textarea + id: proposed-solution + attributes: + label: Proposed Solution & Expected Behavior + description: A clear and concise description of what you want to happen and how it should look or behave. + placeholder: Describe your proposed design, UI behavior, or API capability... + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: Any alternative solutions or workarounds you have considered. + placeholder: Optional description of other approaches evaluated... + validations: + required: false + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context, mockup screenshots, or implementation suggestions here. + placeholder: Optional mockups, architectural notes, or links... + validations: + required: false diff --git a/README.md b/README.md index bd1665d..eaeea5e 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,7 @@ OpenOrbit exposes a local, versioned API: - Swagger UI: `http://localhost:3000/api/docs` - OpenAPI document: `http://localhost:3000/api/openapi.json` - API base: `http://localhost:3000/api/v1` +- MCP (Streamable HTTP): `http://localhost:3000/mcp/` Read the [API reference](docs/API.md) for endpoint details. To add reusable automation, create a Python runner with explicit lifecycle phases: diff --git a/backend/app/assistant_graph.py b/backend/app/assistant_graph.py new file mode 100644 index 0000000..4959182 --- /dev/null +++ b/backend/app/assistant_graph.py @@ -0,0 +1,159 @@ +"""LangGraph orchestration for the local Orbit assistant.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, Callable, TypedDict + +from coding_agents import execution_environment_context +from langgraph.graph import END, START, StateGraph +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +from .assistant_tools import DEFAULT_MCP_URL, AssistantToolExecutor +from .assistant_ui import AssistantUiToolExecutor +from .providers import AzureOpenAIProvider, BedrockProvider, ModelSettings + + +class AssistantState(TypedDict, total=False): + prompt: str + response: str + + +class LocalMcpTools: + """Expose the mounted OpenOrbit MCP server as native model function tools.""" + + def __init__(self, url: str = DEFAULT_MCP_URL): + self.url = url + + async def _list_tools(self) -> list[dict[str, Any]]: + async with streamablehttp_client(self.url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + return [ + { + "name": tool.name, + "description": tool.description or "OpenOrbit MCP tool.", + "parameters": tool.inputSchema, + } + for tool in result.tools + ] + + async def _call_tool(self, name: str, arguments: dict[str, Any]) -> str: + async with streamablehttp_client(self.url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool(name, arguments) + if result.isError: + return json.dumps({"error": "MCP tool failed", "content": result.content}, default=str) + return json.dumps({"content": result.content}, default=str, ensure_ascii=False) + + @staticmethod + def _run(coro: Any) -> Any: + return asyncio.run(coro) + + def definitions(self) -> list[dict[str, Any]]: + try: + return self._run(self._list_tools()) + except Exception: + # The assistant remains usable while the local server is starting. + return [] + + def execute(self, name: str, arguments: dict[str, Any]) -> str: + try: + return self._run(self._call_tool(name, arguments)) + except Exception as error: + return json.dumps({"error": f"MCP unavailable: {error}"}) + + +class OrbitAssistantGraph: + """A small, explicit LangGraph for one bounded assistant turn. + + The model node can use both the local bounded workspace tools and the + mounted MCP control-room tools. Provider adapters retain the native tool + call protocol, while LangGraph owns the turn state and final response. + """ + + def __init__( + self, + provider: AzureOpenAIProvider | BedrockProvider, + settings: ModelSettings, + local_tools: AssistantToolExecutor, + mcp_tools: LocalMcpTools | None = None, + ui_tools: AssistantUiToolExecutor | None = None, + on_activity: Callable[[str, str | None], None] | None = None, + ): + self.provider = provider + self.settings = settings + self.local_tools = local_tools + self.mcp_tools = mcp_tools or LocalMcpTools(local_tools.settings["mcp_server_url"]) + self.ui_tools = ui_tools + self.on_activity = on_activity + graph = StateGraph(AssistantState) + graph.add_node("model", self._model) + graph.add_edge(START, "model") + graph.add_edge("model", END) + self.graph = graph.compile() + + def _model(self, state: AssistantState) -> AssistantState: + self._activity("thinking") + local_definitions = self.local_tools.definitions() + mcp_definitions = self.mcp_tools.definitions() + ui_definitions = self.ui_tools.definitions() if self.ui_tools else [] + definitions = [*mcp_definitions, *local_definitions, *ui_definitions] + + def execute(name: str, arguments: dict[str, Any]) -> str: + self._activity("working", name) + if any(tool["name"] == name for tool in mcp_definitions): + result = self.mcp_tools.execute(name, arguments) + elif self.ui_tools and any(tool["name"] == name for tool in ui_definitions): + result = self.ui_tools.execute(name, arguments) + else: + result = self.local_tools.execute(name, arguments) + self._activity("thinking") + return result + + response = ( + self.provider.complete_with_tools(self.settings, state["prompt"], definitions, execute) + if definitions + else self.provider.complete(self.settings, state["prompt"]) + ) + return {"response": response} + + def _activity(self, phase: str, tool: str | None = None) -> None: + if self.on_activity: + self.on_activity(phase, tool) + + def invoke(self, prompt: str) -> str: + result = self.graph.invoke({"prompt": prompt}) + return str(result["response"]) + + +def build_assistant_prompt( + content: str, history: list[tuple[str, str]], locale: str | None, ui_enabled: bool = False +) -> str: + prompt = ( + "You are Orbit, a concise assistant for the local OpenOrbit control room.\n" + "Use the OpenOrbit MCP tools as the source of truth for local control-room state. " + "Read state before suggesting or taking any state-changing action.\n" + + execution_environment_context() + ) + if ui_enabled: + prompt += ( + "Browser UI tools are available for this turn. For any question about the current Orbit " + "screen, visible component, selected item, open panel, dialog, form value, or any request " + "to change the Orbit UI, first call ui_get_context. Use ui_interact only after reading the " + "current UI context. Do not use file tools to infer browser UI state. File tools remain for " + "workspace files and coding tasks only.\n" + ) + if locale: + prompt += f"Respond in BCP 47 locale '{locale}'.\n" + if history: + prompt += ( + "Conversation so far:\n" + + "\n".join(f"{role.title()}: {message}" for role, message in history) + + "\n\n" + ) + return prompt + f"User: {content}\nAssistant:" diff --git a/backend/app/assistant_tools.py b/backend/app/assistant_tools.py index e0b97a1..d12bdc4 100644 --- a/backend/app/assistant_tools.py +++ b/backend/app/assistant_tools.py @@ -5,14 +5,24 @@ import json import os import re +import shutil import subprocess from pathlib import Path from typing import Any +from coding_agents import CODING_AGENT_PROVIDERS, build_coding_agent_command, build_coding_agent_prompt + MAX_RESULT_BYTES = 16 * 1024 MAX_FILE_LINES = 120 MAX_SEARCH_RESULTS = 12 BLOCKED_NAMES = {".env", ".git", ".ssh", "id_rsa", "id_ed25519"} +DEFAULT_MCP_URL = "http://127.0.0.1:3000/mcp/" + + +def _terminal_visible() -> bool: + """Deployment flag; terminal visibility is not an operator UI setting.""" + value = os.environ.get("ORBIT_ASSISTANT_TERMINAL_VISIBLE", "true").strip().lower() + return value not in {"0", "false", "no", "off"} def defaults() -> dict[str, Any]: @@ -21,8 +31,12 @@ def defaults() -> dict[str, Any]: "file_read_enabled": True, "file_search_enabled": True, "run_process_enabled": True, + "coding_agent_enabled": True, + "ui_context_enabled": True, + "ui_interaction_enabled": True, "terminal_enabled": True, - "terminal_visible": True, + "terminal_visible": _terminal_visible(), + "mcp_server_url": DEFAULT_MCP_URL, } @@ -36,24 +50,32 @@ def normalize_settings(value: object) -> dict[str, Any]: "file_read_enabled", "file_search_enabled", "run_process_enabled", + "coding_agent_enabled", + "ui_context_enabled", + "ui_interaction_enabled", "terminal_enabled", - "terminal_visible", ): result[key] = bool(value.get(key, result[key])) + result["terminal_visible"] = _terminal_visible() + result["mcp_server_url"] = ( + str(value.get("mcp_server_url", result["mcp_server_url"])).strip() or result["mcp_server_url"] + ) return result class AssistantToolExecutor: - def __init__(self, settings: dict[str, Any]): + def __init__(self, settings: dict[str, Any], coding_agent_provider: str = "none"): self.settings = normalize_settings(settings) root = self.settings["workspace_root"] self.root = Path(root).expanduser().resolve() if root else None + self.coding_agent_provider = ( + coding_agent_provider if coding_agent_provider in CODING_AGENT_PROVIDERS else "none" + ) def definitions(self) -> list[dict[str, Any]]: - if not self.root or not self.root.is_dir(): - return [] definitions = [] - if self.settings["file_read_enabled"]: + workspace_ready = bool(self.root and self.root.is_dir()) + if workspace_ready and self.settings["file_read_enabled"]: definitions.append( { "name": "file_read", @@ -70,7 +92,7 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) - if self.settings["file_search_enabled"]: + if workspace_ready and self.settings["file_search_enabled"]: definitions.append( { "name": "file_search", @@ -89,7 +111,7 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) - if self.settings["run_process_enabled"]: + if workspace_ready and self.settings["run_process_enabled"]: definitions.append( { "name": "run_process", @@ -107,6 +129,22 @@ def definitions(self) -> list[dict[str, Any]]: }, } ) + if self.settings["coding_agent_enabled"]: + definitions.append( + { + "name": "coding_agent", + "description": "Ask the configured local coding agent to inspect, modify, or validate the configured workspace. If no coding agent is selected, returns instructions for configuring one in Settings.", + "parameters": { + "type": "object", + "properties": { + "task": {"type": "string", "minLength": 1}, + "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 1800}, + }, + "required": ["task"], + "additionalProperties": False, + }, + } + ) return definitions def execute(self, name: str, arguments: dict[str, Any]) -> str: @@ -117,6 +155,8 @@ def execute(self, name: str, arguments: dict[str, Any]) -> str: return self._file_search(arguments) if name == "run_process" and self.settings["run_process_enabled"]: return self._run_process(arguments) + if name == "coding_agent" and self.settings["coding_agent_enabled"]: + return self._coding_agent(arguments) return self._result(error="Tool is disabled or unavailable.") except (OSError, UnicodeError, ValueError, re.error, subprocess.SubprocessError) as error: return self._result(error=str(error)) @@ -230,3 +270,43 @@ def _run_process(self, arguments: dict[str, Any]) -> str: output=output, truncated=len((result.stdout + result.stderr).encode("utf-8")) > MAX_RESULT_BYTES, ) + + def _coding_agent(self, arguments: dict[str, Any]) -> str: + if self.coding_agent_provider == "none": + return self._result( + error="No Coding Agent is selected. Select Kiro, Claude Code, or Codex in Settings.", + error_code="coding_agent_not_configured", + settings_page="settings", + ) + if not self.root or not self.root.is_dir(): + return self._result( + error="The Assistant workspace is missing. Configure a valid workspace in the Assistant tool settings.", + error_code="assistant_workspace_unavailable", + ) + prompt = build_coding_agent_prompt(str(arguments["task"]), self.root) + command = build_coding_agent_command(self.coding_agent_provider, prompt=prompt) + if not shutil.which(command[0]): + return self._result( + error=f"The selected Coding Agent ({self.coding_agent_provider}) is not installed or is unavailable on PATH.", + error_code="coding_agent_not_installed", + provider=self.coding_agent_provider, + ) + result = subprocess.run( + command, + cwd=self.root, + shell=False, + capture_output=True, + text=True, + timeout=min(max(int(arguments.get("timeout_seconds", 600)), 1), 1800), + env=os.environ.copy(), + ) + combined_output = result.stdout + result.stderr + output = combined_output.encode("utf-8")[:MAX_RESULT_BYTES].decode("utf-8", errors="ignore") + return self._result( + provider=self.coding_agent_provider, + command=command[:-1], + working_directory=str(self.root), + exit_code=result.returncode, + output=output, + truncated=len(combined_output.encode("utf-8")) > MAX_RESULT_BYTES, + ) diff --git a/backend/app/assistant_ui.py b/backend/app/assistant_ui.py new file mode 100644 index 0000000..35dd561 --- /dev/null +++ b/backend/app/assistant_ui.py @@ -0,0 +1,149 @@ +"""Request-scoped bridge between Assistant tools and one browser UI session.""" + +from __future__ import annotations + +import json +import logging +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +logger = logging.getLogger(__name__) + + +@dataclass +class PendingUiCommand: + event: threading.Event = field(default_factory=threading.Event) + result: dict[str, Any] | None = None + + +@dataclass +class UiSession: + send: Callable[[str, dict[str, Any]], None] + pending: dict[str, PendingUiCommand] = field(default_factory=dict) + + +class AssistantUiBroker: + """Route synchronous model tool calls through the active chat stream.""" + + def __init__(self) -> None: + self._sessions: dict[str, UiSession] = {} + self._lock = threading.Lock() + + def open(self, session_id: str, send: Callable[[str, dict[str, Any]], None]) -> None: + with self._lock: + self._sessions[session_id] = UiSession(send=send) + + def close(self, session_id: str) -> None: + with self._lock: + session = self._sessions.pop(session_id, None) + if session: + for pending in session.pending.values(): + pending.result = {"ok": False, "error": "UI session closed."} + pending.event.set() + + def request(self, session_id: str, command: dict[str, Any], timeout: float = 30) -> dict[str, Any]: + command_id = uuid.uuid4().hex + pending = PendingUiCommand() + with self._lock: + session = self._sessions.get(session_id) + if not session: + logger.info("assistant UI command rejected: unavailable session=%s", session_id) + return {"ok": False, "error": "The browser UI session is unavailable."} + session.pending[command_id] = pending + logger.info( + "assistant UI command sent: session=%s id=%s type=%s", + session_id, + command_id, + command.get("type"), + ) + session.send(command_id, command) + if not pending.event.wait(timeout): + with self._lock: + session = self._sessions.get(session_id) + if session: + session.pending.pop(command_id, None) + return {"ok": False, "error": "The browser UI did not respond in time."} + return pending.result or {"ok": False, "error": "The browser UI returned no result."} + + def resolve(self, session_id: str, command_id: str, result: dict[str, Any]) -> bool: + with self._lock: + session = self._sessions.get(session_id) + pending = session.pending.pop(command_id, None) if session else None + if not pending: + logger.info("assistant UI result ignored: session=%s id=%s", session_id, command_id) + return False + pending.result = result + logger.info( + "assistant UI result received: session=%s id=%s ok=%s components=%s diagnostics=%s", + session_id, + command_id, + result.get("ok"), + len(result.get("components", [])) if isinstance(result.get("components"), list) else "n/a", + result.get("diagnostics"), + ) + pending.event.set() + return True + + +class AssistantUiToolExecutor: + """Expose semantic, browser-owned UI capabilities to the Assistant.""" + + def __init__( + self, broker: AssistantUiBroker, session_id: str, *, context_enabled: bool, interaction_enabled: bool + ): + self.broker = broker + self.session_id = session_id + self.context_enabled = context_enabled + self.interaction_enabled = interaction_enabled and context_enabled + + def definitions(self) -> list[dict[str, Any]]: + definitions = [] + if self.context_enabled: + definitions.append( + { + "name": "ui_get_context", + "description": "Read the current browser page, visible Assistant-enabled components, controls, and actions. Read this before interacting with the UI.", + "parameters": { + "type": "object", + "properties": {"component_id": {"type": "string"}}, + "additionalProperties": False, + }, + } + ) + if self.interaction_enabled: + definitions.append( + { + "name": "ui_interact", + "description": "Set one visible Assistant-enabled UI control or invoke one listed UI action. Use the revision returned by ui_get_context. High-impact actions only open the normal user confirmation dialog.", + "parameters": { + "type": "object", + "properties": { + "component_id": {"type": "string", "minLength": 1}, + "revision": {"type": "integer", "minimum": 1}, + "operation": {"type": "string", "enum": ["set_control_value", "invoke_action"]}, + "target_id": {"type": "string", "minLength": 1}, + "value": {}, + }, + "required": ["component_id", "revision", "operation", "target_id"], + "additionalProperties": False, + }, + } + ) + return definitions + + def execute(self, name: str, arguments: dict[str, Any]) -> str: + if name == "ui_get_context" and self.context_enabled: + result = self.broker.request( + self.session_id, + {"type": "get_context", "component_id": arguments.get("component_id")}, + ) + elif name == "ui_interact" and self.interaction_enabled: + result = self.broker.request( + self.session_id, + {"type": "interact", **arguments}, + ) + else: + result = {"ok": False, "error": "Unknown browser UI tool."} + return json.dumps(result, ensure_ascii=False) diff --git a/backend/app/main.py b/backend/app/main.py index f70be62..b710fa8 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,22 +1,40 @@ from __future__ import annotations +import asyncio import json +import os import re +import shutil +from contextlib import asynccontextmanager from pathlib import Path from typing import Literal from fastapi import FastAPI, File, HTTPException, Query, Request, Response, UploadFile, WebSocket from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, StreamingResponse from pydantic import BaseModel, Field from . import store as store_module +from .assistant_graph import OrbitAssistantGraph, build_assistant_prompt from .assistant_tools import AssistantToolExecutor +from .assistant_ui import AssistantUiBroker, AssistantUiToolExecutor from .docker import preflight_docker +from .mcp_server import create_mcp_server from .providers import AzureOpenAIProvider, BedrockProvider, ModelSettings from .store import ConsoleStore from .terminal import serve_terminal + +@asynccontextmanager +async def application_lifespan(_: FastAPI): + """Run the MCP session manager for the same lifetime as the API server.""" + async with mcp_server.session_manager.run(): + try: + yield + finally: + store.shutdown() + + app = FastAPI( title="OpenOrbit API", version="0.2.0", @@ -54,6 +72,7 @@ {"name": "Improvements", "description": "Read-only supervisor feedback and analytics."}, {"name": "System", "description": "Local service health and capabilities."}, ], + lifespan=application_lifespan, ) app.add_middleware( CORSMiddleware, @@ -62,16 +81,11 @@ allow_headers=["*"], ) store = ConsoleStore() +assistant_ui_broker = AssistantUiBroker() WEB_DIST = Path(__file__).resolve().parents[2] / "frontend" / "dist" SDK_DOCS_DIST = Path(__file__).resolve().parents[2] / "site" -@app.on_event("shutdown") -def stop_active_runner_processes() -> None: - """Prevent browsers from outliving the local API process on reload.""" - store.shutdown() - - def safely(action): try: return action() @@ -137,6 +151,56 @@ def health_v1(): return health() +@app.get("/api/system/readiness", tags=["System"], summary="Get required local system capabilities") +def system_readiness(): + """Report prerequisites required by the control room as a whole. + + Keep this intentionally separate from build-specific validation: these + checks decide whether the application can safely offer its core features, + while repository, browser, and Docker requirements vary per build. + """ + application = store.application_settings() + profiles = {profile["profile_name"]: profile for profile in store.profiles()} + selected_profile = str(application.get("chat_model_profile_name", "")).strip() + profile = profiles.get(selected_profile) + + ai_detail = "ready" + if not selected_profile: + ai_detail = "profile_not_selected" + elif profile is None: + ai_detail = "profile_not_found" + elif not profile.get("model", "").strip(): + ai_detail = "model_not_set" + elif profile.get("provider") == "azure-openai": + if not profile.get("endpoint", "").strip(): + ai_detail = "endpoint_not_set" + elif not profile.get("secret_env", "").strip() or not os.environ.get(profile["secret_env"].strip()): + ai_detail = "secret_not_available" + elif profile.get("provider") == "aws-bedrock": + has_environment_credentials = bool( + os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY") + ) + if not profile.get("aws_profile", "").strip() and not has_environment_credentials: + ai_detail = "credentials_not_available" + + git_executable = shutil.which("git") + checks = [ + { + "id": "system_ai", + "status": "ready" if ai_detail == "ready" else "blocked", + "detail": ai_detail, + "settings_page": "settings", + }, + { + "id": "git", + "status": "ready" if git_executable else "blocked", + "detail": "ready" if git_executable else "not_installed", + "settings_page": None, + }, + ] + return {"ready": all(check["status"] == "ready" for check in checks), "checks": checks} + + @app.get("/api/v1", tags=["System"], operation_id="getApiInfo", summary="Get API entry-point information") def api_info_v1(): return { @@ -204,16 +268,6 @@ def run_telemetry(run_id: str): return safely(lambda: store.run_telemetry(run_id)) -@app.get("/api/runs/{run_id}/prompt-revisions") -def prompt_revisions(run_id: str): - return safely(lambda: store.prompt_revisions(run_id)) - - -@app.get("/api/runs/{run_id}/commit-changes") -def commit_changes(run_id: str): - return safely(lambda: store.commit_changes(run_id)) - - @app.get("/api/runs/{run_id}/artifacts/{loop_index}/{artifact_path:path}") def run_artifact(run_id: str, loop_index: int, artifact_path: str): return safely(lambda: FileResponse(store.run_artifact(run_id, loop_index, artifact_path))) @@ -254,6 +308,11 @@ def target_environments(): return store.target_environments() +@app.get("/api/personas") +def personas(): + return store.personas() + + class PromptTemplateUpdate(BaseModel): name: str = Field(min_length=1, max_length=120) content: str = Field(min_length=1, max_length=100_000) @@ -313,6 +372,19 @@ class TargetEnvironmentUpdate(BaseModel): managed_prompt_path: str = "" +class PersonaUpdate(BaseModel): + name: str = Field(min_length=1, max_length=120) + locale: str = Field(min_length=2, max_length=32) + timezone: str = Field(min_length=1, max_length=64) + activity_windows: list[dict] = Field(default_factory=list) + definition: str = Field(min_length=1, max_length=20_000) + context: dict = Field(default_factory=dict) + + +class PersonaCreate(PersonaUpdate): + id: str = Field(pattern=r"^[a-z][a-z0-9-]{2,63}$") + + @app.put("/api/prompt-templates/{template_id}") def update_prompt_template(template_id: str, values: PromptTemplateUpdate): return safely(lambda: store.update_prompt_template(template_id, values.model_dump())) @@ -338,6 +410,11 @@ def create_target_environment(values: TargetEnvironmentCreate): return safely(lambda: store.create_target_environment(values.model_dump())) +@app.post("/api/personas") +def create_persona(values: PersonaCreate): + return safely(lambda: store.create_persona(values.model_dump())) + + @app.put("/api/execution-environments/{environment_id}") def update_execution_environment(environment_id: str, values: ExecutionEnvironmentUpdate): return safely(lambda: store.update_execution_environment(environment_id, values.model_dump())) @@ -348,6 +425,11 @@ def update_target_environment(environment_id: str, values: TargetEnvironmentUpda return safely(lambda: store.update_target_environment(environment_id, values.model_dump())) +@app.put("/api/personas/{persona_id}") +def update_persona(persona_id: str, values: PersonaUpdate): + return safely(lambda: store.update_persona(persona_id, values.model_dump())) + + @app.delete("/api/execution-environments/{environment_id}") def delete_execution_environment(environment_id: str): return safely(lambda: store.delete_execution_environment(environment_id)) @@ -358,6 +440,11 @@ def delete_target_environment(environment_id: str): return safely(lambda: store.delete_target_environment(environment_id)) +@app.delete("/api/personas/{persona_id}") +def delete_persona(persona_id: str): + return safely(lambda: store.delete_persona(persona_id)) + + @app.delete("/api/prompt-templates/{template_id}") def delete_prompt_template(template_id: str): return safely(lambda: store.delete_prompt_template(template_id)) @@ -425,6 +512,7 @@ class BuildCreate(BaseModel): manager_template_id: str = "manager-default-v1" model_profile_name: str = "Default" test_case_set_id: str = Field(min_length=1, max_length=64) + persona_ids: list[str] = Field(default_factory=list) browser_base_url: str = Field(default="", max_length=2_000) browser_executable_path: str = Field(default="", max_length=4_000) browser_library_path: str = Field(default="", max_length=4_000) @@ -476,6 +564,10 @@ class IssueManagementDelete(BaseModel): proposal_ids: list[str] = Field(min_length=1, max_length=200) +class AgentIssueDecision(BaseModel): + decision: Literal["approve", "reject"] + + class RunnerAssetUpdate(BaseModel): name: str = Field(min_length=1, max_length=120) description: str = Field(min_length=1, max_length=500) @@ -975,13 +1067,23 @@ def application_settings(): return store.application_settings() +@app.get("/api/assistant-mcp-config") +def assistant_mcp_config(): + return store.assistant_mcp_config() + + class ApplicationSettingsUpdate(BaseModel): manager_prompt_template: str = Field(default="", max_length=100_000) manager_output_locale: str = Field(default="", max_length=100) chat_model_profile_name: str = Field(default="", max_length=200) + coding_agent_provider: Literal["none", "kiro", "claude-code", "codex"] = "none" assistant_tools: dict | None = None +class AssistantMcpConfigUpdate(BaseModel): + content: str = Field(min_length=2, max_length=1_000_000) + + class ApplicationDataLocationUpdate(BaseModel): path: str = Field(min_length=1, max_length=4_096) @@ -1025,6 +1127,16 @@ def update_application_settings(values: ApplicationSettingsUpdate): return store.save_application_settings(values.model_dump(exclude_unset=True)) +@app.put("/api/assistant-mcp-config") +def save_assistant_mcp_config(values: AssistantMcpConfigUpdate): + return safely(lambda: store.save_assistant_mcp_config(values.content)) + + +@app.post("/api/assistant-mcp-config/open-vscode") +def open_assistant_mcp_config_in_vscode(): + return safely(store.open_assistant_mcp_config_in_vscode) + + class ChatTurn(BaseModel): role: str = Field(pattern="^(user|assistant)$") content: str = Field(min_length=1, max_length=20_000) @@ -1033,6 +1145,13 @@ class ChatTurn(BaseModel): class ChatMessage(BaseModel): content: str = Field(min_length=1, max_length=20_000) history: list[ChatTurn] = Field(default_factory=list, max_length=12) + ui_session_id: str = Field(default="", max_length=200) + + +class AssistantUiResult(BaseModel): + session_id: str = Field(min_length=1, max_length=200) + command_id: str = Field(min_length=1, max_length=200) + result: dict class TemplateTranslationRequest(BaseModel): @@ -1168,29 +1287,105 @@ def chat(values: ChatMessage, request: Request): ) try: provider = AzureOpenAIProvider() if settings.provider == "azure-openai" else BedrockProvider() - history = "\n".join(f"{turn.role.title()}: {turn.content}" for turn in values.history) - prompt = ( - "You are Orbit, a concise assistant for the local OpenOrbit control room.\n" - "The local OpenAPI contract is available at /api/openapi.json; use it as the source of truth " - "when explaining API endpoints, parameters, and response shapes.\n" + application = store.application_settings() + tool_executor = AssistantToolExecutor( + application["assistant_tools"], application["coding_agent_provider"] ) - if locale := request_locale(request): - prompt += f"Respond in BCP 47 locale '{locale}'.\n" - if history: - prompt += f"Conversation so far:\n{history}\n\n" - prompt += f"User: {values.content}\nAssistant:" - tool_executor = AssistantToolExecutor(store.application_settings()["assistant_tools"]) - definitions = tool_executor.definitions() - response = ( - provider.complete_with_tools(settings, prompt, definitions, tool_executor.execute) - if definitions - else provider.complete(settings, prompt) + prompt = build_assistant_prompt( + values.content, + [(turn.role, turn.content) for turn in values.history], + request_locale(request), ) + response = OrbitAssistantGraph(provider, settings, tool_executor).invoke(prompt) return {"response": response, "profile_name": profile_name} except RuntimeError as error: raise HTTPException(409, str(error)) +@app.post("/api/chat/stream") +async def chat_stream(values: ChatMessage, request: Request): + profile_name = store.application_settings()["chat_model_profile_name"] + if not profile_name: + raise HTTPException(409, "Select an AI model profile for the chat assistant in Settings.") + configured = profile(store.profiles(), profile_name) + settings = ModelSettings( + **{key: value for key, value in configured.items() if key in ModelSettings.__dataclass_fields__} + ) + provider = AzureOpenAIProvider() if settings.provider == "azure-openai" else BedrockProvider() + application = store.application_settings() + tool_executor = AssistantToolExecutor( + application["assistant_tools"], application["coding_agent_provider"] + ) + prompt = build_assistant_prompt( + values.content, + [(turn.role, turn.content) for turn in values.history], + request_locale(request), + ui_enabled=bool(values.ui_session_id and application["assistant_tools"]["ui_context_enabled"]), + ) + + async def events(): + queue: asyncio.Queue[dict[str, object]] = asyncio.Queue() + loop = asyncio.get_running_loop() + + def send_ui_command(command_id: str, command: dict[str, object]) -> None: + loop.call_soon_threadsafe( + queue.put_nowait, + {"type": "ui_command", "command_id": command_id, "command": command}, + ) + + ui_tools = None + if values.ui_session_id and application["assistant_tools"]["ui_context_enabled"]: + assistant_ui_broker.open(values.ui_session_id, send_ui_command) + ui_tools = AssistantUiToolExecutor( + assistant_ui_broker, + values.ui_session_id, + context_enabled=True, + interaction_enabled=application["assistant_tools"]["ui_interaction_enabled"], + ) + + def activity(phase: str, tool: str | None = None) -> None: + event = {"type": "activity", "phase": phase} + if tool: + event["tool"] = tool + loop.call_soon_threadsafe(queue.put_nowait, event) + + async def run() -> None: + try: + response = await asyncio.to_thread( + OrbitAssistantGraph( + provider, settings, tool_executor, ui_tools=ui_tools, on_activity=activity + ).invoke, + prompt, + ) + await queue.put({"type": "response", "response": response}) + except RuntimeError as error: + await queue.put({"type": "error", "message": str(error)}) + except Exception: + await queue.put({"type": "error", "message": "Chat request failed."}) + + task = asyncio.create_task(run()) + try: + while True: + event = await queue.get() + yield json.dumps(event, ensure_ascii=False) + "\n" + if event["type"] in {"response", "error"}: + break + finally: + if not task.done(): + task.cancel() + if values.ui_session_id: + assistant_ui_broker.close(values.ui_session_id) + + return StreamingResponse(events(), media_type="application/x-ndjson") + + +@app.post("/api/chat/ui-results") +def chat_ui_result(values: AssistantUiResult): + if not assistant_ui_broker.resolve(values.session_id, values.command_id, values.result): + raise HTTPException(404, "Browser UI command was not found or has expired.") + return {"ok": True} + + @app.websocket("/api/terminal") async def terminal(websocket: WebSocket): await serve_terminal(websocket, store.application_settings()["assistant_tools"]) @@ -1326,32 +1521,27 @@ def list_improvements_v1(): return store.improvements() -@app.get( - "/api/v1/improvements/proposals", - tags=["Improvements"], - operation_id="listProposalLifecycles", - summary="List proposals and decisions from evaluation-run results", -) -def list_proposal_lifecycles_v1( - build_id: str | None = None, - status: Literal["proposed", "acceptable", "accepted", "rejected", "applied"] | None = None, -): - return store.proposal_lifecycles(build_id, status) +@app.get("/api/v1/issue-management", tags=["Improvements"], operation_id="listIssueManagementItems") +def list_issue_management_items_v1(): + return store.issue_management_items() @app.get( - "/api/v1/improvements/iterations", + "/api/v1/issue-management/{proposal_id}/diff", tags=["Improvements"], - operation_id="listImprovementIterationData", - summary="List SDK-saved data files by evaluation iteration", + operation_id="getIssueManagementDiff", ) -def list_improvement_iteration_data_v1(build_id: str | None = None): - return store.improvement_iteration_data(build_id) +def issue_management_diff_v1(proposal_id: str): + return safely(lambda: store.issue_management_diff(proposal_id)) -@app.get("/api/v1/issue-management", tags=["Improvements"], operation_id="listIssueManagementItems") -def list_issue_management_items_v1(): - return store.issue_management_items() +@app.post( + "/api/v1/issue-management/{proposal_id}/decision", + tags=["Improvements"], + operation_id="decideAgentIssue", +) +def decide_agent_issue_v1(proposal_id: str, values: AgentIssueDecision): + return safely(lambda: store.decide_agent_issue(proposal_id, values.decision)) @app.patch( @@ -1451,6 +1641,12 @@ def sdk_docs(path: str): raise HTTPException(404, "SDK documentation page was not found.") +# Streamable HTTP transport for MCP clients. The mounted application's root +# is the protocol endpoint, so the public URL is /mcp/. +mcp_server = create_mcp_server(lambda: store, app.openapi) +app.mount("/mcp", mcp_server.streamable_http_app()) + + @app.get("/{path:path}", include_in_schema=False) def local_web_app(path: str): """Serve the built React application for `orbit-agent-console run`.""" diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py new file mode 100644 index 0000000..0cc4c39 --- /dev/null +++ b/backend/app/mcp_server.py @@ -0,0 +1,122 @@ +"""MCP surface for the stable OpenOrbit v1 API. + +The tools deliberately call the same store operations as the HTTP handlers. +This keeps MCP local-first and avoids making the server call itself over HTTP. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Literal + +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from mcp.types import ToolAnnotations + +from .docker import preflight_docker +from .store import ConsoleStore + + +def _json(value: Any) -> Any: + """Convert Pydantic response models without changing ordinary JSON values.""" + return value.model_dump(mode="json") if hasattr(value, "model_dump") else value + + +def create_mcp_server( + get_store: Callable[[], ConsoleStore], openapi: Callable[[], dict[str, Any]] +) -> FastMCP: + """Create the streamable-HTTP MCP server mounted by the FastAPI app.""" + mcp = FastMCP( + "OpenOrbit", + instructions=( + "Operate the local OpenOrbit control plane. Read state before starting, approving, " + "rejecting, cancelling, or stopping pipelines. The OpenAPI resource is the complete " + "contract for the corresponding HTTP API." + ), + streamable_http_path="/", + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=["127.0.0.1:*", "localhost:*", "[::1]:*"], + ), + ) + + @mcp.resource("openorbit://openapi") + def openapi_schema() -> str: + """The generated OpenAPI 3 contract for OpenOrbit's versioned HTTP API.""" + import json + + return json.dumps(openapi(), ensure_ascii=False) + + @mcp.tool( + description="Get local service health, dashboard totals, active pipelines, and Docker readiness.", + annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False), + ) + def get_status() -> dict[str, Any]: + store = get_store() + return { + "health": {"status": "ok"}, + "dashboard": _json(store.dashboard()), + "active_pipelines": _json(store.active_evaluations()), + "docker": preflight_docker(), + } + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_projects() -> list[dict[str, Any]]: + """List all configured projects (the v1 API calls builds projects).""" + return [_json(project) for project in get_store().builds()] + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def get_project(project_id: str) -> dict[str, Any]: + """Get one project and its complete evaluation configuration.""" + return _json(get_store().build(project_id)) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_pipelines(project_id: str | None = None, status: str | None = None) -> list[dict[str, Any]]: + """List retained pipeline runs, optionally filtered by project or status.""" + pipelines = get_store().runs() + if project_id: + pipelines = [pipeline for pipeline in pipelines if pipeline.build_id == project_id] + if status: + pipelines = [pipeline for pipeline in pipelines if pipeline.status == status] + return [_json(pipeline) for pipeline in pipelines] + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def get_pipeline(pipeline_id: str) -> dict[str, Any]: + """Get a pipeline's current state, phases, evidence, and supervisor results.""" + return _json(get_store().run(pipeline_id)) + + @mcp.tool( + description="Start a project pipeline. Use test mode for a one-off safe validation run.", + annotations=ToolAnnotations(destructiveHint=False, idempotentHint=False, openWorldHint=False), + ) + def start_pipeline(project_id: str, execution_mode: Literal["run", "test"] = "run") -> dict[str, Any]: + """Start a retained run or transient test pipeline for a project.""" + return _json(get_store().invoke_remote_build(project_id, execution_mode, None)) + + @mcp.tool( + description="Approve, reject, or cancel a pipeline that is awaiting an operator action.", + annotations=ToolAnnotations(destructiveHint=True, idempotentHint=False, openWorldHint=False), + ) + def act_on_pipeline(pipeline_id: str, action: Literal["approve", "reject", "cancel"]) -> dict[str, Any]: + """Perform an approval-gated pipeline action.""" + actions = { + "approve": get_store().approve, + "reject": get_store().reject, + "cancel": get_store().cancel, + } + return _json(actions[action](pipeline_id)) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True, idempotentHint=True, openWorldHint=False)) + def list_improvements(build_id: str | None = None) -> list[dict[str, Any]]: + """List supervisor proposals and their decision lifecycle, optionally for one project.""" + return _json(get_store().proposal_lifecycles(build_id)) + + @mcp.tool( + description="Cancel every active local pipeline. Use only after confirming the affected work.", + annotations=ToolAnnotations(destructiveHint=True, idempotentHint=False, openWorldHint=False), + ) + def emergency_stop_pipelines() -> dict[str, Any]: + """Cancel all active pipelines immediately.""" + return _json(get_store().emergency_stop()) + + return mcp diff --git a/backend/app/models.py b/backend/app/models.py index f6fdb70..4612f6e 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -15,12 +15,22 @@ "teardown": "after_each", "finalize": "after_all", } -LIFECYCLE_PHASES = ("before_all", "before_each", "execute", "verify", "after_each", "after_all") +LIFECYCLE_PHASES = ( + "before_all", + "before_each", + "execute", + "verify", + "after_supervision", + "after_each", + "after_all", +) class Step(BaseModel): id: str - phase: Literal["before_all", "before_each", "execute", "verify", "after_each", "after_all"] + phase: Literal[ + "before_all", "before_each", "execute", "verify", "after_supervision", "after_each", "after_all" + ] name: str command: list[str] working_directory: str @@ -28,6 +38,7 @@ class Step(BaseModel): approval: Literal["not_required", "required"] = "not_required" on_failure: Literal["stop", "continue"] = "stop" minimum_interval_seconds: int = Field(default=0, ge=0, le=86_400) + after_supervision: bool = False @field_validator("phase", mode="before") @classmethod @@ -59,6 +70,16 @@ def lifecycle_is_complete(self, execution_mode: Literal["run", "test"] = "run") return phases in ( ["before_all", "before_each", "execute", "verify", "after_each"], ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_each", "execute", "verify", "after_supervision", "after_each"], + [ + "before_all", + "before_each", + "execute", + "verify", + "after_supervision", + "after_each", + "after_all", + ], ) diff --git a/backend/app/store.py b/backend/app/store.py index ea6939a..5a0a116 100644 --- a/backend/app/store.py +++ b/backend/app/store.py @@ -29,6 +29,7 @@ from orbit import load_bundle +from .assistant_tools import DEFAULT_MCP_URL from .assistant_tools import normalize_settings as normalize_assistant_tools from .models import PHASE_ALIASES, Run, Step, Workflow from .observability import configure_telemetry @@ -82,8 +83,10 @@ def _application_data_dir() -> Path: TARGET_TEST_CASE_SETS = CONFIG / "target-ai-test-case-sets.yaml" EXECUTION_ENVIRONMENTS = CONFIG / "execution-environments.yaml" TARGET_ENVIRONMENTS = CONFIG / "target-environments.yaml" +PERSONAS = CONFIG / "personas.yaml" CYCLE_INTERVENTIONS = CONFIG / "cycle-interventions.yaml" ISSUE_MANAGEMENT = CONFIG / "issue-management.yaml" +ASSISTANT_MCP_CONFIG = CONFIG / "assistant-mcp.json" DEFAULT_OPERATIONAL_MANAGER_PROMPT = """You are an approval-first operations manager for recurring AI evaluations. Preserve the task safety boundary, collect observable evidence, and never claim success without stated acceptance evidence. Escalate required approvals @@ -96,8 +99,8 @@ def _application_data_dir() -> Path: Your final response must be exactly one JSON object: { \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_trace\": {\"persona_goal\":\"string\",\"current_action\":\"string\",\"decision\":\"string\",\"next_action\":\"string\",\"evidence\":\"string\"}}, - \"improvements\": [{\"title\":\"string\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], - \"reported_issues\": [{\"title\":\"string\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] + \"improvements\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], + \"reported_issues\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] } For an evaluated AI, include behavior_trace and fill every field. This is an evidence-backed persona journey, not the evaluator's procedure and not hidden reasoning. Keep the visible journey concise and written from the persona's perspective: persona_goal is the persona's stable wish; current_action is the one meaningful action taken in this iteration; decision is the resulting judgment or choice; next_action is the one specific, safe next action. Write each field as one to three natural sentences in the selected output language. Follow that language's normal grammar, ellipsis, and point of view; do not mechanically repeat a subject or pronoun across fields. Do not describe navigation, waits, screenshots, generic control inspection, or other repeated mechanics in any visible journey field. evidence is a concise source-backed factual record for the evidence drawer, not a visible journey item. Use the persona's wording where useful, but do not invent motives, feelings, beliefs, or facts beyond the declared persona and observed evidence. Do not reveal hidden reasoning or evaluator chain-of-thought. Do not include behavior_trace for non-AI targets. behavior_summary is deprecated and should be omitted. Always include both array keys, using empty arrays when there are no items.""" LEGACY_OPERATIONAL_MANAGER_PROMPT = """You are an approval-first operations manager for recurring AI evaluations. @@ -110,8 +113,8 @@ def _application_data_dir() -> Path: Your final response must be exactly one JSON object: { \"evaluation\": {\"score\":\"number from 0 to 10\",\"approval\":\"approved|rejected|pending\",\"summary\":\"string\",\"behavior_summary\":\"string, only when the evaluated target is an AI\"}, - \"improvements\": [{\"title\":\"string\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], - \"reported_issues\": [{\"title\":\"string\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] + \"improvements\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"status\":\"proposed|acceptable|rejected\",\"rationale\":\"string\",\"acceptanceEvidence\":\"string\"}], + \"reported_issues\": [{\"title\":\"string\",\"category\":\"accuracy_grounding|safety_privacy|task_completion|user_experience|other\",\"severity\":\"low|medium|high|critical\",\"evidence\":\"string\",\"reproduction\":\"string\",\"status\":\"open|acknowledged|resolved\"}] } Include behavior_summary only when the evaluated target is an AI. It must describe the AI's observed responses, decisions, tool use, refusals, or other behavior in plain language; do not describe pass/fail outcomes, metrics, baselines, or the evaluator's actions. Omit behavior_summary for non-AI targets. Always include both array keys, using empty arrays when there are no items.""" PROPOSAL_DECISION_POLICY = """# Improvement decision policy @@ -148,12 +151,13 @@ def configure_application_data(path: str) -> Path: global APP_DATA, CONFIG, TARGET_TEST_CASE_SETS, EXECUTION_ENVIRONMENTS, TARGET_ENVIRONMENTS global CYCLE_INTERVENTIONS, DATA, RUNS, TELEMETRY, SETTINGS, TOOL_TIMES, RUNNERS - global RUNNER_TEMPLATES, QUICK_STARTS, QUICK_START_INSTANCES, TEMPLATE_TRANSLATIONS + global RUNNER_TEMPLATES, QUICK_STARTS, QUICK_START_INSTANCES, TEMPLATE_TRANSLATIONS, PERSONAS APP_DATA = target CONFIG = APP_DATA / "config" TARGET_TEST_CASE_SETS = CONFIG / "target-ai-test-case-sets.yaml" EXECUTION_ENVIRONMENTS = CONFIG / "execution-environments.yaml" TARGET_ENVIRONMENTS = CONFIG / "target-environments.yaml" + PERSONAS = CONFIG / "personas.yaml" CYCLE_INTERVENTIONS = CONFIG / "cycle-interventions.yaml" DATA = APP_DATA / "data" RUNS = DATA / "runs" @@ -1126,6 +1130,42 @@ def open_runner_in_vscode(self, runner_id: str) -> dict[str, str]: self._open_in_vscode(self._runner_entry_path(runner_id, int(self._runner(runner_id)["version"]))) return {"status": "opened"} + def assistant_mcp_config(self) -> dict[str, str]: + if not ASSISTANT_MCP_CONFIG.exists(): + return { + "path": str(ASSISTANT_MCP_CONFIG), + "content": json.dumps( + {"mcpServers": {"openorbit": {"url": DEFAULT_MCP_URL}}}, + indent=2, + ) + + "\n", + } + return { + "path": str(ASSISTANT_MCP_CONFIG), + "content": ASSISTANT_MCP_CONFIG.read_text(encoding="utf-8"), + } + + def save_assistant_mcp_config(self, content: str) -> dict[str, str]: + try: + document = json.loads(content) + except json.JSONDecodeError as error: + raise ValueError(f"MCP configuration must be valid JSON: {error.msg}") from error + if not isinstance(document, dict): + raise ValueError("MCP configuration must be a JSON object") + ASSISTANT_MCP_CONFIG.parent.mkdir(parents=True, exist_ok=True) + temporary = ASSISTANT_MCP_CONFIG.with_suffix(".tmp") + temporary.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + temporary.replace(ASSISTANT_MCP_CONFIG) + return self.assistant_mcp_config() + + def open_assistant_mcp_config_in_vscode(self) -> dict[str, str]: + if not ASSISTANT_MCP_CONFIG.exists(): + self.save_assistant_mcp_config( + json.dumps({"mcpServers": {"openorbit": {"url": DEFAULT_MCP_URL}}}) + ) + self._open_in_vscode(ASSISTANT_MCP_CONFIG) + return {"status": "opened"} + def delete_runner(self, runner_id: str) -> None: self._runner(runner_id) if any(build.get("runner_id") == runner_id for build in self.builds()): @@ -1136,6 +1176,47 @@ def delete_runner(self, runner_id: str) -> None: def _runner_execution_plan(self, runner_id: str, runner_version: int | None = None) -> Workflow: """Build the lifecycle declared by a runner without a workflow asset.""" runner = self._runner(runner_id) + definition = self._runner_graph_definition(runner_id, None, runner_version) + graph_nodes = definition.get("nodes", []) if isinstance(definition, dict) else [] + lifecycle_phases = {"before_all", "before_each", "execute", "verify", "after_each", "after_all"} + if isinstance(graph_nodes, list) and graph_nodes: + steps = [ + Step( + id=str(node["id"]), + phase=str(node["phase"]), + name=str(node.get("title") or node["id"]), + command=[ + sys.executable, + str(self._runner_entry_path(runner_id, runner_version)), + "--phase", + str(node["phase"]), + "--step", + str(node["id"]), + ], + working_directory=str(ROOT), + timeout_seconds=86_400 if node["phase"] == "execute" else 300, + approval="not_required", + on_failure="stop", + after_supervision=bool(node.get("after_supervision", False)), + ) + for node in graph_nodes + if isinstance(node, dict) and node.get("phase") in lifecycle_phases + ] + if steps: + return Workflow( + id=runner_id, + name=runner["name"], + description=runner["description"], + kind="improvement" + if runner.get("template_id") == "native-improvement-cycle" + else "simulation", + enabled=True, + risk="medium", + runner_id=runner_id, + runner_version=runner_version or int(runner["version"]), + steps=steps, + test_steps=deepcopy(steps), + ) source = runner["source"] if runner_version is not None: selected = next( @@ -1149,7 +1230,14 @@ def _runner_execution_plan(self, runner_id: str, runner_version: int | None = No if selected is None: raise ValueError(f"runner version {runner_version} does not exist") source = str(selected["source"]) - lifecycle_order = ("before_all", "before_each", "execute", "verify", "after_each", "after_all") + lifecycle_order = ( + "before_all", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ) declared = { PHASE_ALIASES.get(phase, phase) for phase in re.findall(r'@runner\.phase\(\s*["\']([^"\']+)["\']\s*\)', source) @@ -1280,8 +1368,10 @@ def builds(self) -> list[dict[str, Any]]: self._hydrate_build_environment(build) build.update(self._repository_metadata(str(build.get("repository", "")))) build.setdefault("created_at", fallback) - dates = [run.created_at for run in runs if run.build_id == build["id"]] + build_runs = [run for run in runs if run.build_id == build["id"]] + dates = [run.created_at for run in build_runs] build["last_run_at"] = max(dates).isoformat() if dates else None + build["run_count"] = len(build_runs) return builds def _hydrate_build_environment(self, build: dict[str, Any]) -> None: @@ -1639,6 +1729,103 @@ def target_test_case_sets(self) -> list[dict[str, Any]]: else [] ) + def personas(self) -> list[dict[str, Any]]: + values = yaml.safe_load(PERSONAS.read_text(encoding="utf-8")) if PERSONAS.exists() else [] + return ( + [ + {**item, "definition": self._persona_definition(item)} + for item in values + if isinstance(item, dict) + ] + if isinstance(values, list) + else [] + ) + + @staticmethod + def _persona_definition(values: dict[str, Any]) -> str: + """Migrate legacy goal/constraint arrays into one Markdown definition.""" + definition = str(values.get("definition", "")).strip() + if definition: + return definition + sections = [] + goals = [str(goal).strip() for goal in values.get("goals", []) if str(goal).strip()] + constraints = [ + str(constraint).strip() for constraint in values.get("constraints", []) if str(constraint).strip() + ] + if goals: + sections.append("# Goals\n\n" + "\n".join(f"- {goal}" for goal in goals)) + if constraints: + sections.append("# Constraints\n\n" + "\n".join(f"- {constraint}" for constraint in constraints)) + return "\n\n".join(sections) + + @staticmethod + def _validated_persona(values: dict[str, Any], persona_id: str) -> dict[str, Any]: + name = str(values.get("name", "")).strip() + locale, timezone = str(values.get("locale", "")).strip(), str(values.get("timezone", "")).strip() + definition, windows, context = ( + str(values.get("definition", "")).strip(), + values.get("activity_windows", []), + values.get("context", {}), + ) + if not name or not locale or not timezone or not definition: + raise ValueError("persona requires a name, locale, timezone, and Markdown definition") + if not isinstance(windows, list) or not all(isinstance(window, dict) for window in windows): + raise ValueError("persona activity windows must be a list of objects") + if not isinstance(context, dict): + raise ValueError("persona context must be an object") + return { + "id": persona_id, + "name": name, + "locale": locale, + "timezone": timezone, + "activity_windows": windows, + "definition": definition, + "context": context, + } + + def create_persona(self, values: dict[str, Any]) -> dict[str, Any]: + persona_id = str(values.get("id", "")).strip() + personas = self.personas() + if not persona_id or any(item.get("id") == persona_id for item in personas): + raise ValueError("persona ID is required and must be unique") + persona = self._validated_persona(values, persona_id) + persona["created_at"] = now().isoformat() + personas.append(persona) + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text(yaml.safe_dump(personas, allow_unicode=True, sort_keys=False), encoding="utf-8") + temporary.replace(PERSONAS) + return persona + + def update_persona(self, persona_id: str, values: dict[str, Any]) -> dict[str, Any]: + personas = self.personas() + index = next((i for i, item in enumerate(personas) if item.get("id") == persona_id), None) + if index is None: + raise KeyError(persona_id) + persona = self._validated_persona(values, persona_id) + persona["created_at"] = personas[index].get("created_at", now().isoformat()) + personas[index] = persona + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text(yaml.safe_dump(personas, allow_unicode=True, sort_keys=False), encoding="utf-8") + temporary.replace(PERSONAS) + return persona + + def delete_persona(self, persona_id: str) -> None: + personas = self.personas() + if not any(item.get("id") == persona_id for item in personas): + raise KeyError(persona_id) + if any(persona_id in build.get("persona_ids", []) for build in self.builds()): + raise ValueError("persona is used by a build") + temporary = PERSONAS.with_suffix(".tmp") + temporary.write_text( + yaml.safe_dump( + [item for item in personas if item.get("id") != persona_id], + allow_unicode=True, + sort_keys=False, + ), + encoding="utf-8", + ) + temporary.replace(PERSONAS) + @staticmethod def _validated_target_test_case_set(values: dict[str, Any], set_id: str) -> dict[str, Any]: name, description = str(values.get("name", "")).strip(), str(values.get("description", "")).strip() @@ -1955,6 +2142,12 @@ def create_build(self, values: dict[str, Any]) -> dict[str, Any]: raise ValueError("AI model profile does not exist") if not any(item.get("id") == values.get("test_case_set_id") for item in self.target_test_case_sets()): raise ValueError("target-AI test case set does not exist") + persona_ids = [str(item) for item in values.get("persona_ids", [])] + if len(set(persona_ids)) != len(persona_ids) or any( + not any(persona.get("id") == persona_id for persona in self.personas()) + for persona_id in persona_ids + ): + raise ValueError("persona selection contains an unknown or duplicate persona") build = { "id": build_id, "name": values["name"], @@ -1975,6 +2168,7 @@ def create_build(self, values: dict[str, Any]) -> dict[str, Any]: "model_profile_name": values.get("model_profile_name", "Default"), "task_instruction": values.get("task_instruction", ""), "test_case_set_id": values["test_case_set_id"], + "persona_ids": persona_ids, "browser_base_url": str(target_environment.get("browser_base_url", "")).strip(), "browser_executable_path": str(execution_environment.get("browser_executable_path", "")).strip(), "browser_library_path": str(execution_environment.get("browser_library_path", "")).strip(), @@ -2041,6 +2235,12 @@ def update_build(self, build_id: str, values: dict[str, Any]) -> dict[str, Any]: raise ValueError("AI model profile does not exist") if not any(item.get("id") == values.get("test_case_set_id") for item in self.target_test_case_sets()): raise ValueError("target-AI test case set does not exist") + persona_ids = [str(item) for item in values.get("persona_ids", [])] + if len(set(persona_ids)) != len(persona_ids) or any( + not any(persona.get("id") == persona_id for persona in self.personas()) + for persona_id in persona_ids + ): + raise ValueError("persona selection contains an unknown or duplicate persona") existing = builds[index] build = { "id": build_id, @@ -2062,6 +2262,7 @@ def update_build(self, build_id: str, values: dict[str, Any]) -> dict[str, Any]: "model_profile_name": values.get("model_profile_name", "Default"), "task_instruction": existing.get("task_instruction", ""), "test_case_set_id": values["test_case_set_id"], + "persona_ids": persona_ids, "browser_base_url": str(target_environment.get("browser_base_url", "")).strip(), "browser_executable_path": str(execution_environment.get("browser_executable_path", "")).strip(), "browser_library_path": str(execution_environment.get("browser_library_path", "")).strip(), @@ -2114,6 +2315,12 @@ def improvements(self) -> list[dict[str, Any]]: path = CONFIG / "improvements.yaml" return yaml.safe_load(path.read_text(encoding="utf-8")) if path.exists() else [] + @staticmethod + def _issue_category(value: Any) -> str: + """Normalize persisted categories; legacy implementation targets are never categories.""" + categories = {"accuracy_grounding", "safety_privacy", "task_completion", "user_experience", "other"} + return str(value) if str(value) in categories else "other" + def cycle_interventions(self) -> list[dict[str, Any]]: return ( yaml.safe_load(CYCLE_INTERVENTIONS.read_text(encoding="utf-8")) @@ -2143,6 +2350,7 @@ def proposal_lifecycles( "response": run.supervisor_response, } ] + seen_agent_improvements: set[str] = set() for record in records: if not isinstance(record, dict): continue @@ -2182,13 +2390,35 @@ def proposal_lifecycles( if isinstance(persona, str) and persona not in personas: personas.append(persona) improvements = response.get("improvements", []) + reported_issues = response.get("reported_issues", []) evaluation = response.get("evaluation") score = evaluation.get("score") if isinstance(evaluation, dict) else None if isinstance(score, bool) or not isinstance(score, (int, float)): score = None - if not isinstance(improvements, list): + if not isinstance(improvements, list) or not isinstance(reported_issues, list): continue - for index, proposal in enumerate(improvements): + # The first supervision pass produces Issue proposals and their + # operator-facing rationale. A later agent-proposal assessment + # evaluates a worktree diff; its generated improvements are + # review feedback, not new persona/Issue proposals. + # Improvements are the operator-facing Issue proposals. A + # reported issue is used only when no improvement was made, + # preventing one supervisor observation from becoming two + # indistinguishable Issue Management rows. + issue_proposals = [] + if record.get("stage") != "agent_proposal_assessment": + issue_proposals = ( + [ + (f"{run.id}:{record.get('iteration', 0)}:{index}", proposal) + for index, proposal in enumerate(improvements) + ] + if improvements + else [ + (f"{run.id}:{record.get('iteration', 0)}:issue:{index}", proposal) + for index, proposal in enumerate(reported_issues) + ] + ) + for proposal_id, proposal in issue_proposals: if not isinstance(proposal, dict): continue source_status = str(proposal.get("status") or "proposed").lower() @@ -2203,9 +2433,11 @@ def proposal_lifecycles( ) values.append( { - "proposal_id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "proposal_id": proposal_id, "title": str(proposal.get("title") or "Untitled proposal"), - "target": str(proposal.get("target") or "prompt"), + "category": self._issue_category(proposal.get("category")), + # target is legacy execution metadata. Do not expose it as an Issue category. + "target": str(proposal.get("target") or ""), "proposal": proposal, "decision": decision, "score": score, @@ -2223,9 +2455,9 @@ def proposal_lifecycles( "prompt_version": None, "events": [ { - "id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "id": proposal_id, "event_type": "decision", - "proposal_id": f"{run.id}:{record.get('iteration', 0)}:{index}", + "proposal_id": proposal_id, "decision": decision, "rationale": str(proposal.get("rationale") or ""), "recorded_at": record.get("recorded_at") or run.updated_at.isoformat(), @@ -2236,6 +2468,90 @@ def proposal_lifecycles( ], } ) + # Coding-agent changes are first-class review proposals too. + # They are emitted by the SDK only when the agent supplied + # explicit feedback, and their immutable worktree diff stays + # attached to the resulting Issue. + if record.get("stage") != "agent_proposal_assessment": + continue + for step in run.step_results: + if not isinstance(step, dict) or step.get("loop_index") != iteration: + continue + result = step.get("result") + agent = result.get("agent_run") if isinstance(result, dict) else None + if not isinstance(agent, dict) or not str(agent.get("feedback") or "").strip(): + continue + proposal_data = agent.get("proposal") + proposal_data = proposal_data if isinstance(proposal_data, dict) else {} + agent_proposal = result.get("agent_proposal") if isinstance(result, dict) else {} + linked_issue = agent_proposal.get("issue") if isinstance(agent_proposal, dict) else None + fingerprint = str(proposal_data.get("fingerprint") or "") + if not fingerprint or fingerprint in seen_agent_improvements: + continue + seen_agent_improvements.add(fingerprint) + approval = ( + str(evaluation.get("approval") or "pending").lower() + if isinstance(evaluation, dict) + else "pending" + ) + decision = ( + "accepted" + if approval == "approved" + else "rejected" + if approval == "rejected" + else "pending" + ) + parent_id = ( + str(linked_issue.get("_orbit_issue_id") or "") + if isinstance(linked_issue, dict) + else "" + ) + parent = next( + (item for item in reversed(values) if item["proposal_id"] == parent_id), None + ) + # Backward-compatible attachment for evidence retained before stable IDs existed. + if parent is None and isinstance(linked_issue, dict): + linked_title = str(linked_issue.get("title") or "") + parent = next( + ( + item + for item in reversed(values) + if item.get("run_id") == run.id + and item.get("iteration") == iteration + and item.get("title") == linked_title + ), + None, + ) + if parent is None: + candidates = [ + item + for item in values + if item.get("run_id") == run.id and item.get("iteration") == iteration + ] + if len(candidates) == 1: + parent = candidates[0] + if parent is None: + continue + parent_proposal = parent.get("proposal") + if not isinstance(parent_proposal, dict): + continue + parent_proposal["agent_change"] = { + "feedback": str(agent.get("feedback")).strip(), + "changed_files": agent.get("changed_files", []), + "diff": proposal_data.get("diff", ""), + "diff_artifact": proposal_data.get("diff_artifact"), + "base_revision": proposal_data.get("base_revision"), + "branch": proposal_data.get("branch", ""), + "worktree_path": proposal_data.get("worktree_path", ""), + "fingerprint": fingerprint, + "review": { + "decision": decision, + "score": score, + "summary": (evaluation or {}).get("summary", "") + if isinstance(evaluation, dict) + else "", + }, + } if status: values = [item for item in values if item["status"] == status or item["decision"] == status] return sorted(values, key=lambda item: str(item.get("recorded_at", "")), reverse=True) @@ -2271,13 +2587,31 @@ def issue_management_items(self) -> list[dict[str, Any]]: # management-level tombstone, not deletion of the underlying run. if record.get("status") == "deleted": continue + agent_change = (proposal.get("proposal") or {}).get("agent_change") or {} + review = agent_change.get("review") if isinstance(agent_change, dict) else None + review_comment = [] + if isinstance(review, dict) and str(review.get("summary") or "").strip(): + review_comment = [ + { + "body": str(review["summary"]).strip(), + "assigner": "AI supervisor", + "recorded_at": proposal.get("recorded_at", ""), + "kind": "agent_change_review", + "decision": review.get("decision"), + } + ] values.append( { **proposal, "management_status": record.get("status", "unreviewed"), + "proposal_action": record.get("proposal_action", "pending"), + "proposal_branch": str( + ((proposal.get("proposal") or {}).get("agent_change") or {}).get("branch") or "" + ), + "proposal_commit": str(record.get("proposal_commit") or ""), "assigner": record.get("assigner", ""), - "comments": record.get("comments", []), - "management_events": record.get("events", []), + "comments": [*review_comment, *record.get("comments", [])], + "management_events": [*review_comment, *record.get("events", [])], } ) return values @@ -2340,52 +2674,113 @@ def delete_issue_management_items(self, proposal_ids: list[str]) -> dict[str, in self._save_issue_management_records(records) return {"deleted": deleted} - def improvement_iteration_data(self, build_id: str | None = None) -> list[dict[str, Any]]: - """List SDK-saved data files by persisted evaluation run and iteration. + def issue_management_diff(self, proposal_id: str) -> dict[str, Any]: + """Return the immutable worktree patch attached to an agent Issue.""" + item = next( + (value for value in self.issue_management_items() if value["proposal_id"] == proposal_id), None + ) + if item is None: + raise KeyError(proposal_id) + proposal = item.get("proposal") + agent_change = proposal.get("agent_change") if isinstance(proposal, dict) else None + if agent_change is None and isinstance(proposal, dict) and proposal.get("kind") == "agent_change": + agent_change = proposal + if not isinstance(agent_change, dict): + return {"diff": "", "changed_files": []} + changed_files = agent_change.get("changed_files") + return { + "diff": str(agent_change.get("diff") or ""), + "changed_files": changed_files if isinstance(changed_files, list) else [], + "base_revision": agent_change.get("base_revision"), + } - These entries intentionally exist even when a supervisor did not make a - proposal. The proposal-decision history uses them to expose the full - iteration record rather than hiding developer-retained evidence behind - a proposal requirement. - """ - values: list[dict[str, Any]] = [] - for run in self.runs(): - if build_id and run.build_id != build_id: - continue - grouped: dict[int, list[dict[str, Any]]] = {} - timestamps: dict[int, str] = {} - for step in run.step_results: - if not isinstance(step, dict) or not isinstance(step.get("loop_index"), int): - continue - iteration = step["loop_index"] - for data_file in step.get("data_files", []): - if not isinstance(data_file, dict): - continue - path, filename = data_file.get("path"), data_file.get("filename") - if not isinstance(path, str) or not isinstance(filename, str): - continue - grouped.setdefault(iteration, []).append( - { - "label": str(data_file.get("label") or "")[:256], - "filename": filename, - "path": path, - "relative_path": str(data_file.get("relative_path") or ""), - "content_type": str(data_file.get("content_type") or ""), - } - ) - timestamps.setdefault(iteration, str(step.get("ended_at") or run.updated_at.isoformat())) - for iteration, data_files in grouped.items(): - values.append( - { - "build_id": run.build_id, - "build_name": run.build_name, - "run_id": run.id, - "iteration": iteration, - "recorded_at": timestamps[iteration], - "data_files": data_files, - } + def decide_agent_issue(self, proposal_id: str, decision: str) -> dict[str, Any]: + """Commit or discard a pending agent worktree proposal from its Issue.""" + if decision not in {"approve", "reject"}: + raise ValueError("decision must be approve or reject") + item = next( + (value for value in self.issue_management_items() if value["proposal_id"] == proposal_id), None + ) + if item is None: + raise KeyError(proposal_id) + proposal = item.get("proposal") + agent_change = proposal.get("agent_change") if isinstance(proposal, dict) else None + if agent_change is None and isinstance(proposal, dict) and proposal.get("kind") == "agent_change": + agent_change = proposal + if not isinstance(agent_change, dict): + raise ValueError("only agent worktree proposals can be decided from an Issue") + branch = str(agent_change.get("branch") or "").strip() + worktree_value = str(agent_change.get("worktree_path") or "").strip() + if not branch or not worktree_value: + raise ValueError("this agent proposal was created before worktree review was enabled") + worktree = Path(worktree_value).resolve() + worktree_root = (APP_DATA / "agent-worktrees").resolve() + if worktree_root not in worktree.parents: + raise ValueError("agent proposal worktree is outside Orbit storage") + source_repository = Path(self._load(str(item.get("run_id") or "")).repository).resolve() + if not source_repository.is_dir(): + raise ValueError("agent proposal source repository is no longer available") + records = self._issue_management_records() + record = records.setdefault(proposal_id, {"status": "unreviewed", "comments": [], "events": []}) + previous = str(record.get("proposal_action") or "pending") + if previous != "pending": + return next( + value for value in self.issue_management_items() if value["proposal_id"] == proposal_id + ) + timestamp = now().isoformat() + if decision == "approve": + if not worktree.is_dir(): + raise ValueError("agent proposal worktree is no longer available") + changed = subprocess.run( + ["git", "diff", "--quiet", "--"], cwd=worktree, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if changed.returncode == 0: + raise ValueError("agent proposal has no uncommitted changes") + if changed.returncode > 1: + raise ValueError( + changed.stderr.decode("utf-8", "replace").strip() or "cannot inspect agent proposal" ) - return sorted(values, key=lambda item: str(item["recorded_at"]), reverse=True) + subprocess.run( + ["git", "add", "--all"], + cwd=worktree, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + message = f"Orbit agent proposal: {str(agent_change.get('feedback') or 'approved change')[:160]}" + subprocess.run( + ["git", "commit", "-m", message], + cwd=worktree, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=worktree, check=True, text=True, stdout=subprocess.PIPE + ).stdout.strip() + record.update({"proposal_action": "approved", "proposal_commit": commit, "status": "resolved"}) + record["events"].append( + {"type": "proposal_approved", "branch": branch, "commit": commit, "recorded_at": timestamp} + ) + else: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=source_repository, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "branch", "-D", branch], + cwd=source_repository, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + record.update({"proposal_action": "rejected", "status": "resolved"}) + record["events"].append({"type": "proposal_rejected", "branch": branch, "recorded_at": timestamp}) + self._save_issue_management_records(records) + return next(value for value in self.issue_management_items() if value["proposal_id"] == proposal_id) def _save_cycle_interventions(self, values: list[dict[str, Any]]) -> None: temporary = CYCLE_INTERVENTIONS.with_suffix(".tmp") @@ -2604,51 +2999,6 @@ def prompt_revisions(self, run_id: str) -> list[dict[str, Any]]: current_by_path[path] = after return [*initial_revisions, *ordered] - def commit_changes(self, run_id: str) -> list[dict[str, Any]]: - """Return commit ranges automatically retained by SDK runner phases.""" - run = self._load(run_id) - changes: list[dict[str, Any]] = [] - for step in run.step_results: - result = step.get("result") - event = result.get("commit_change") if isinstance(result, dict) else None - if not isinstance(event, dict) and isinstance(result, dict): - jgent = result.get("jgent_paired") - candidate = jgent.get("committed_source_candidate") if isinstance(jgent, dict) else None - event = candidate if isinstance(candidate, dict) else None - if not isinstance(event, dict): - continue - before, after = str(event.get("before") or ""), str(event.get("after") or "") - if not before or not after or before == after: - continue - artifact = event.get("diff_artifact") if isinstance(event.get("diff_artifact"), dict) else None - diff: str | None = None - if artifact: - raw_path = artifact.get("path") - candidate = Path(str(raw_path)).resolve() if isinstance(raw_path, str) else None - artifacts_root = (APP_DATA / "artifacts").resolve() - if candidate and artifacts_root in candidate.parents: - try: - if candidate.stat().st_size <= 500_000: - diff = candidate.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - pass - changes.append( - { - "iteration": step.get("loop_index"), - "phase": step.get("phase"), - "recorded_at": step.get("ended_at"), - "before": before, - "after": after, - "changed_paths": event.get("changed_paths") - if isinstance(event.get("changed_paths"), list) - else [], - "commits": event.get("commits") if isinstance(event.get("commits"), list) else [], - "diff_artifact": artifact, - "diff": diff, - } - ) - return sorted(changes, key=lambda item: str(item.get("recorded_at") or ""), reverse=True) - def run_artifact(self, run_id: str, loop_index: int, relative_path: str) -> Path: """Resolve one retained run artifact without permitting path traversal.""" if loop_index < 0: @@ -3227,6 +3577,7 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": normalize_assistant_tools(None), } stored = json.loads(SETTINGS.read_text(encoding="utf-8")) @@ -3236,6 +3587,7 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": normalize_assistant_tools(None), } prompt = str(values.get("manager_prompt_template", "")).strip() @@ -3248,6 +3600,12 @@ def application_settings(self) -> dict[str, Any]: "manager_prompt_template": prompt or DEFAULT_OPERATIONAL_MANAGER_PROMPT, "manager_output_locale": output_locale, "chat_model_profile_name": str(values.get("chat_model_profile_name", "")).strip(), + "coding_agent_provider": ( + str(values.get("coding_agent_provider", "none")).strip() + if str(values.get("coding_agent_provider", "none")).strip() + in {"kiro", "claude-code", "codex"} + else "none" + ), "assistant_tools": normalize_assistant_tools(values.get("assistant_tools")), } @@ -3272,6 +3630,12 @@ def save_application_settings(self, values: dict[str, Any]) -> dict[str, Any]: ).strip(), "manager_output_locale": output_locale, "chat_model_profile_name": chat_profile_name, + "coding_agent_provider": ( + str(values.get("coding_agent_provider", current["coding_agent_provider"])).strip() + if str(values.get("coding_agent_provider", current["coding_agent_provider"])).strip() + in {"none", "kiro", "claude-code", "codex"} + else "none" + ), "assistant_tools": normalize_assistant_tools( values.get("assistant_tools", current["assistant_tools"]) ), @@ -3384,6 +3748,10 @@ def _execute(self, run_id: str) -> None: None, ) resources["test_cases"] = selected.get("cases", []) if selected else build.get("test_cases", []) + selected_personas = set(build.get("persona_ids", [])) + resources["personas"] = [ + persona for persona in self.personas() if persona.get("id") in selected_personas + ] profile_name = str(build.get("model_profile_name", "")) resources["model_profile"] = next( (profile for profile in self.profiles() if profile.get("profile_name") == profile_name), @@ -3396,7 +3764,14 @@ def _execute(self, run_id: str) -> None: if workflow.runner_id: runner_path = self._runner_entry_path(workflow.runner_id, run.runner_version) for step in [*workflow.steps, *(workflow.test_steps or [])]: - step.command = [sys.executable, str(runner_path), "--phase", step.phase] + step.command = [ + sys.executable, + str(runner_path), + "--phase", + step.phase, + "--step", + step.id, + ] step.working_directory = run.repository or str(ROOT) with self.tracer.start_as_current_span( "workflow.run", @@ -3415,8 +3790,13 @@ def _execute(self, run_id: str) -> None: self._save(run) steps = workflow.steps_for(run.execution_mode) before_all = [step for step in steps if step.phase == "before_all"] - loop_steps = [step for step in steps if step.phase in {"before_each", "execute", "verify"}] - after_each = [step for step in steps if step.phase == "after_each"] + loop_steps = [ + step + for step in steps + if step.phase in {"before_each", "execute", "verify"} and not step.after_supervision + ] + post_supervision_steps = [step for step in steps if step.after_supervision] + after_each = [step for step in steps if step.phase == "after_each" and not step.after_supervision] after_all = [step for step in steps if step.phase == "after_all"] if not self._wait_for_schedule(run_id): return @@ -3513,15 +3893,23 @@ def _execute(self, run_id: str) -> None: self._execute_step(run_id, step, loop_index, resources) if self._load(run_id).status in {"failed", "cancelled"}: break - # Cleanup is part of the lifecycle contract, not merely the - # happy path. Run it after an earlier phase fails or a user - # cancels the Run; the terminal status remains unchanged. + # Retain all iteration evidence before the supervisor selects + # an Issue. The post-supervision agent work is displayed as an + # after_each step, but runs only after that selection exists. for step in after_each: self._execute_step(run_id, step, loop_index, resources, allow_terminal=True) - if self._load(run_id).status in {"failed", "cancelled"}: - break + # The supervisor first assesses discovered issues. The agent + # may then create a proposal for an issue the supervisor did + # not reject; only that concrete proposal receives the second, + # independent score/decision. if run.execution_mode == "run" and self._load(run_id).status == "running": self._complete_supervision(run_id) + for step in post_supervision_steps: + self._execute_step(run_id, step, loop_index, resources) + if post_supervision_steps and self._load(run_id).status == "running": + self._complete_supervision(run_id, stage="agent_proposal_assessment") + if self._load(run_id).status in {"failed", "cancelled"}: + break if ( loop_index < run.loop_limit and run.repeat_interval_minutes @@ -3603,7 +3991,7 @@ def _latest_cycle_has_persona_evidence(run: Run) -> bool: return False @staticmethod - def _validated_supervisor_result(text: str) -> dict[str, Any]: + def _validated_supervisor_result(text: str, *, require_evaluation: bool = False) -> dict[str, Any]: """Validate the exact structured result required by the manager template.""" try: result = json.loads(text) @@ -3611,10 +3999,8 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: raise ValueError("supervisor did not return valid JSON") from error if not isinstance(result, dict) or not {"improvements", "reported_issues"}.issubset(result): raise ValueError("supervisor JSON must contain improvements and reported_issues") - if set(result) not in ( - {"improvements", "reported_issues"}, - {"evaluation", "improvements", "reported_issues"}, - ): + allowed_keys = {"evaluation", "persona_journeys", "improvements", "reported_issues"} + if not set(result).issubset(allowed_keys): raise ValueError("supervisor JSON contains unsupported result fields") if not all( isinstance(result[key], list) and all(isinstance(item, dict) for item in result[key]) @@ -3622,6 +4008,32 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: ): raise ValueError("supervisor improvements and reported_issues must be arrays of objects") evaluation = result.get("evaluation") + if require_evaluation and evaluation is None: + raise ValueError("supervisor evaluation is required for a registered agent result") + if evaluation is not None: + ConsoleStore._validate_evaluation(evaluation) + journeys = result.get("persona_journeys", []) + if not isinstance(journeys, list) or not all(isinstance(item, dict) for item in journeys): + raise ValueError("supervisor persona_journeys must be an array of objects") + for journey in journeys: + if ( + set(journey) != {"persona_id", "behavior_trace"} + or not isinstance(journey["persona_id"], str) + or not journey["persona_id"].strip() + ): + raise ValueError("supervisor persona journey is invalid") + ConsoleStore._validate_persona_journey_trace(journey["behavior_trace"]) + for issue in result["reported_issues"]: + issue_evaluation = issue.get("evaluation") + if issue_evaluation is not None: + ConsoleStore._validate_evaluation(issue_evaluation) + for issue in [*result["reported_issues"], *result["improvements"]]: + issue["category"] = ConsoleStore._issue_category(issue.get("category")) + return result + + @staticmethod + def _validate_evaluation(evaluation: Any) -> None: + """Validate either an issue assessment or an agent-proposal assessment.""" if evaluation is not None: if ( not isinstance(evaluation, dict) @@ -3685,9 +4097,56 @@ def _validated_supervisor_result(text: str) -> dict[str, Any]: or not all(isinstance(trace[field], str) and trace[field].strip() for field in trace) ): raise ValueError("supervisor evaluation behavior_trace is invalid") - return result - def _complete_supervision(self, run_id: str) -> None: + @staticmethod + def _validate_persona_journey_trace(trace: Any) -> None: + required = {"persona_goal", "current_action", "decision", "next_action", "evidence"} + if ( + not isinstance(trace, dict) + or set(trace) != required + or not all(isinstance(trace[field], str) and trace[field].strip() for field in required) + ): + raise ValueError("supervisor persona journey behavior_trace is invalid") + + @staticmethod + def _evaluation_request_for_iteration( + run: Run, iteration: int, candidate_id: str | None + ) -> dict[str, Any] | None: + """Return the explicit SDK evaluation request for this agent result. + + A normal iteration is evidence for improvement feedback, but is not by + itself a subject to score. Only ``RunnerContext.register_evaluation`` + can opt an agent result into score/approval evaluation. + """ + for step in reversed(run.step_results): + if step.get("loop_index") != iteration: + continue + if candidate_id is not None and step.get("candidate_id") != candidate_id: + continue + result = step.get("result") + request = result.get("evaluation_request") if isinstance(result, dict) else None + if not isinstance(request, dict): + continue + feedback = request.get("feedback") + if ( + request.get("subject") != "agent_change" + or not isinstance(feedback, str) + or not feedback.strip() + ): + continue + value = { + "subject": "agent_change", + "feedback": feedback.strip(), + "changed_files": request.get("changed_files", []), + "validation": request.get("validation", ""), + } + fingerprint = str(request.get("improvement_fingerprint") or "") + if fingerprint: + value["improvement_fingerprint"] = fingerprint + return value + return None + + def _complete_supervision(self, run_id: str, *, stage: str = "issue_assessment") -> None: """Ask the configured manager model and retain its validated JSON per Run. A missing profile is observable but never turns a successfully completed @@ -3714,6 +4173,26 @@ def _complete_supervision(self, run_id: str) -> None: ), None, ) + evaluation_request = self._evaluation_request_for_iteration(run, iteration, candidate_id) + if stage == "issue_assessment": + evaluation_request = None + elif stage != "agent_proposal_assessment": + raise ValueError(f"unsupported supervision stage: {stage}") + elif evaluation_request is None: + # No Agent feedback and no retained diff means there is no proposal + # to score. Issue assessment is already retained independently. + return + if evaluation_request and evaluation_request.get("improvement_fingerprint"): + fingerprint = evaluation_request["improvement_fingerprint"] + already_evaluated = any( + isinstance(record, dict) + and record.get("stage") == stage + and isinstance(record.get("evaluation_request"), dict) + and record["evaluation_request"].get("improvement_fingerprint") == fingerprint + for record in run.supervisor_results + ) + if already_evaluated: + return configured = next( (item for item in self.profiles() if item["profile_name"] == run.supervisor_profile_name), self.settings(), @@ -3785,6 +4264,40 @@ def supervisor_result(result: object) -> object: if cycle_evidence: supervisor_prompt += "\n\n# OpenOrbit cycle evidence\n" supervisor_prompt += json.dumps(cycle_evidence, ensure_ascii=False, default=str) + supervisor_prompt += ( + "\n\n# Persona journey timeline required\n" + "Always include a top-level persona_journeys array. For every active or processed " + "persona supported by this iteration's evidence, emit one object with its exact persona_id " + "and behavior_trace: {persona_goal, current_action, decision, next_action, evidence}. " + "Write the trace as the supervisor's evidence-backed observation of that persona, not as " + "the runner procedure or hidden reasoning. Do not invent a journey for a persona without " + "evidence; return [] when there is none. persona_journeys is independent of top-level " + "evaluation: include it even when no iteration score is allowed.\n" + "The only permitted top-level keys are evaluation (when explicitly requested), " + "persona_journeys, improvements, and reported_issues." + ) + if stage == "issue_assessment": + supervisor_prompt += ( + "\n\n# Issue assessment required\n" + "Assess every reported_issues and improvements item independently. Each discovered item must include " + "evaluation: {score, approval, summary}; approval is approved, pending, or rejected. " + "This is the score/decision for the discovered issue, not for the iteration. " + "Do not include a top-level evaluation. An agent may only create a proposal for an issue " + "whose approval is not rejected." + ) + elif evaluation_request: + supervisor_prompt += "\n\n# Agent result to evaluate\n" + supervisor_prompt += json.dumps(evaluation_request, ensure_ascii=False, default=str) + supervisor_prompt += ( + "\n\nThe SDK explicitly registered this completed agent result for evaluation. " + "Include evaluation and score only this agent result; do not score the iteration itself." + ) + else: + supervisor_prompt += ( + "\n\nNo SDK agent result was registered for evaluation in this iteration. " + "Keep improvements and reported_issues as usual, but omit evaluation entirely. " + "Do not assign a score or approval to the iteration." + ) with self.tracer.start_as_current_span( "supervisor.evaluate", attributes={ @@ -3806,7 +4319,19 @@ def supervisor_result(result: object) -> object: response_text = provider.complete(settings, supervisor_prompt) span.set_attributes(_telemetry_text_metadata("gen_ai.response", response_text)) span.add_event("gen_ai.response.received") - result = self._validated_supervisor_result(response_text) + result = self._validated_supervisor_result( + response_text, require_evaluation=evaluation_request is not None + ) + if stage == "issue_assessment" and any( + not isinstance(item.get("evaluation"), dict) + for item in [*result["reported_issues"], *result["improvements"]] + ): + raise ValueError("every supervisor-discovered issue must include score and decision") + # Manager templates created before agent-result evaluation can + # still return an evaluation by default. A score is invalid + # without the SDK's explicit registration, so never retain it. + if evaluation_request is None: + result.pop("evaluation", None) evaluation = result.get("evaluation") if evaluation is not None: threshold = ( @@ -3835,9 +4360,11 @@ def supervisor_result(result: object) -> object: { "iteration": iteration, "candidate_id": candidate_id, + "stage": stage, "status": "completed", "prompt": supervisor_prompt, "response": result, + "evaluation_request": evaluation_request, "recorded_at": now().isoformat(), } ) diff --git a/backend/coding_agents.py b/backend/coding_agents.py new file mode 100644 index 0000000..6ffebe4 --- /dev/null +++ b/backend/coding_agents.py @@ -0,0 +1,59 @@ +"""Shared local coding-agent command and prompt conventions.""" + +from __future__ import annotations + +import os +import platform +import shlex +from datetime import datetime +from pathlib import Path + +CODING_AGENT_PROVIDERS = ("kiro", "claude-code", "codex") + + +def build_coding_agent_command(provider: str, *, options: str = "", prompt: str) -> list[str]: + """Build the established CLI invocation for a locally installed coding agent.""" + commands = { + # ``--approve-for-me`` already selects Codex's workspace-write sandbox. + "codex": (["codex", "exec", "--approve-for-me"], []), + "claude-code": (["claude"], ["-p"]), + "kiro": (["kiro-cli", "chat"], []), + } + if provider not in commands: + raise ValueError("AI agent provider must be codex, claude-code, or kiro") + normalized_prompt = prompt.strip() + if not normalized_prompt: + raise ValueError("AI agent prompt must not be empty") + try: + extra_arguments = shlex.split(options) + except ValueError as error: + raise ValueError("AI agent options must be a valid command argument string") from error + executable, prompt_arguments = commands[provider] + return [*executable, *extra_arguments, *prompt_arguments, normalized_prompt] + + +def execution_environment_context(working_directory: str | Path | None = None) -> str: + """Describe the host context in which a coding task will run.""" + shell_path = os.environ.get("COMSPEC") if os.name == "nt" else os.environ.get("SHELL") + shell_path = shell_path or ("cmd.exe" if os.name == "nt" else "/bin/sh") + shell_name = os.path.basename(shell_path) or shell_path + directory = Path(working_directory).expanduser().resolve() if working_directory else Path.cwd() + current_time = datetime.now().astimezone().isoformat(timespec="seconds") + return ( + "Execution environment:\n" + f"- OS: {platform.system()} {platform.release()}\n" + f"- Default shell: {shell_name} ({shell_path})\n" + f"- Current working directory: {directory}\n" + f"- Current local time: {current_time}\n" + ) + + +def build_coding_agent_prompt(task: str, working_directory: str | Path) -> str: + """Wrap an Assistant task in the shared system context understood by all CLIs.""" + return ( + "You are the coding agent used by Orbit Assistant.\n" + "Work only in the current working directory. Inspect and validate changes where appropriate.\n" + + execution_environment_context(working_directory) + + "\nUser task:\n" + + task.strip() + ) diff --git a/backend/orbit_sdk.py b/backend/orbit_sdk.py index 51cff8d..a8763e0 100644 --- a/backend/orbit_sdk.py +++ b/backend/orbit_sdk.py @@ -15,6 +15,7 @@ import subprocess import tempfile import threading +import uuid from contextlib import contextmanager from dataclasses import dataclass, field from datetime import UTC, datetime @@ -22,6 +23,8 @@ from pathlib import Path from typing import Any, Callable, Iterator, Literal +from coding_agents import build_coding_agent_command + PROJECT_ROOT = Path(os.environ.get("ORBIT_TARGET_REPOSITORY", Path.cwd())).resolve() ORBIT_APP_DATA = Path(os.environ.get("ORBIT_APP_DATA", Path.home() / ".local" / "share" / "orbit")).resolve() @@ -32,6 +35,7 @@ "setup": "before_each", "run": "execute", "eval": "verify", + "post_supervision": "after_supervision", "teardown": "after_each", "finalize": "after_all", } @@ -52,6 +56,7 @@ class GraphNode: inputs: tuple[str, ...] = () outputs: tuple[str, ...] = () description: str | None = None + after_supervision: bool = False @dataclass(frozen=True) @@ -87,6 +92,7 @@ def step( inputs: tuple[str, ...] | list[str] = (), outputs: tuple[str, ...] | list[str] = (), description: str | None = None, + after_supervision: bool = False, ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Annotate one function as a visual workflow node. @@ -106,6 +112,7 @@ def register(handler: Callable[..., Any]) -> Callable[..., Any]: inputs=tuple(inputs), outputs=tuple(outputs), description=description, + after_supervision=after_supervision, ) self._nodes[node_id] = node setattr(handler, "__orbit_graph_node__", node) @@ -153,6 +160,7 @@ def definition(self) -> dict[str, object]: "inputs": list(node.inputs), "outputs": list(node.outputs), "description": node.description, + **({"after_supervision": True} if node.after_supervision else {}), } for node in self._nodes.values() ], @@ -1566,6 +1574,56 @@ def previous_supervisor_feedback(self) -> dict[str, object]: return feedback return {} + @property + def current_issue_assessment(self) -> dict[str, object]: + """Return this iteration's first supervisor-assessed actionable issue. + + The post-supervision agent phase uses this as its sole objective. A + rejected issue is deliberately returned too, so the runner can record + an explicit skip rather than treating absence as approval. + """ + run_id = self.environment.get("ORBIT_RUN_ID", "").strip() + if not run_id: + return {} + try: + values = json.loads( + (self.app_data / "data" / "runs" / f"{run_id}.json").read_text(encoding="utf-8") + ) + except (OSError, json.JSONDecodeError): + return {} + for record in reversed(values.get("supervisor_results", [])): + if not isinstance(record, dict) or record.get("stage") != "issue_assessment": + continue + if int(record.get("iteration", 0)) != self.loop_index: + continue + response = record.get("response") + if not isinstance(response, dict): + continue + issues = response.get("reported_issues") + improvements = response.get("improvements") + # Manager templates commonly express a discovered product problem + # as an improvement proposal. It is still an Issue for this + # lifecycle once the supervisor has scored and decided it. Prefer + # that user-facing proposal so the agent changes the same Issue + # shown in Issue Management rather than creating a parallel row. + candidates = [ + {**improvement, "_orbit_issue_id": f"{run_id}:{self.loop_index}:{index}"} + for index, improvement in enumerate(improvements) + if isinstance(improvements, list) and isinstance(improvement, dict) + ] + candidates.extend( + {**issue, "_orbit_issue_id": f"{run_id}:{self.loop_index}:issue:{index}"} + for index, issue in enumerate(issues) + if isinstance(issues, list) and isinstance(issue, dict) + ) + if candidates: + for issue in candidates: + assessment = issue.get("evaluation") + if not isinstance(assessment, dict) or assessment.get("approval") != "rejected": + return issue + return candidates[0] if candidates else {} + return {} + def log(self, message: str) -> None: """Write an Orbit runner-progress message to the workflow log. @@ -1648,6 +1706,155 @@ def emit_result(self, values: dict[str, object]) -> None: """ print("__ORBIT_RESULT__" + json.dumps(values, ensure_ascii=False), flush=True) + def register_evaluation( + self, + feedback: str, + *, + subject: str = "agent_change", + changed_files: list[str] | None = None, + validation: str = "", + improvement_fingerprint: str = "", + ) -> dict[str, object]: + """Register an AI-agent result that should receive a supervisor evaluation. + + Register only after the agent has completed a meaningful change or + produced actionable feedback. Merely running an iteration does not + create an evaluation, score, or approval decision. + + Args: + feedback: Concise account of the agent's completed work and its evidence. + subject: Stable evaluation subject. ``agent_change`` is the supported + default for autonomous SDK agents. + changed_files: Repository-relative files changed by the agent, if any. + validation: Verification performed by the agent after its work. + + Returns: + The structured evaluation request emitted for the current step. + """ + normalized_feedback = feedback.strip() + if not normalized_feedback: + raise ValueError("evaluation feedback must not be empty") + if subject != "agent_change": + raise ValueError("evaluation subject must be agent_change") + normalized_files = [ + str(path).strip() for path in (changed_files or []) if isinstance(path, str) and path.strip() + ] + request = { + "subject": subject, + "feedback": normalized_feedback, + "changed_files": list(dict.fromkeys(normalized_files)), + "validation": validation.strip(), + "improvement_fingerprint": improvement_fingerprint.strip(), + } + self.emit_result({"evaluation_request": request}) + return request + + def run_ai_agent( + self, + prompt: str, + *, + provider: str, + options: str = "", + timeout: int = 1_800, + ) -> dict[str, object]: + """Run a locally installed coding agent and retain its completion evidence. + + The agent must print ``ORBIT_AGENT_FEEDBACK: `` as its final + feedback line to opt its work into evaluation. This keeps a silent or + no-op agent run from creating a score for the iteration. + """ + normalized_prompt = prompt.strip() + if not normalized_prompt: + raise ValueError("AI agent prompt must not be empty") + # Agent edits are proposals, never writes to the evaluated repository. + # A proposal branch/worktree gives the coding agent a real Git checkout while + # preserving the operator's working tree and its uncommitted changes. + self._git_repository_root() + run_id = str(self.environment.get("ORBIT_RUN_ID") or "manual") + worktree_id = f"{run_id}-{self.loop_index}-{uuid.uuid4().hex[:8]}" + worktree = ORBIT_APP_DATA / "agent-worktrees" / worktree_id + worktree.parent.mkdir(parents=True, exist_ok=True) + base_revision = self.git_head() + if not base_revision: + raise ValueError("AI agent proposals require a checked-out Git revision") + branch = f"orbit/agent-proposal/{run_id}/{self.loop_index}-{uuid.uuid4().hex[:8]}" + self.exec( + ["git", "worktree", "add", "-b", branch, str(worktree), base_revision], + cwd=self.project_root, + target_log_source="ai-agent", + ) + output = "" + changed_files: list[str] = [] + patch = "" + keep_worktree = False + try: + output = self.exec( + build_coding_agent_command(provider, options=options, prompt=normalized_prompt), + cwd=worktree, + timeout=timeout, + target_log_source="ai-agent", + ) + # Include newly-created files in the review patch without creating + # a commit. Intent-to-add is local to this disposable proposal + # worktree and is finalized only after Issue approval. + self.exec(["git", "add", "--intent-to-add", "--all"], cwd=worktree) + changed_files = [ + line + for line in self.exec(["git", "diff", "--name-only", "--"], cwd=worktree).splitlines() + if line + ] + patch = self.exec(["git", "diff", "--binary", "--"], cwd=worktree) + feedback = next( + ( + line.removeprefix("ORBIT_AGENT_FEEDBACK:").strip() + for line in reversed(output.splitlines()) + if line.startswith("ORBIT_AGENT_FEEDBACK:") + and line.removeprefix("ORBIT_AGENT_FEEDBACK:").strip() + ), + "", + ) + keep_worktree = bool(feedback and patch) + finally: + if not keep_worktree: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=self.project_root, + env=self.environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + subprocess.run( + ["git", "branch", "-D", branch], + cwd=self.project_root, + env=self.environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + artifact = ( + self.write_artifact(f"agent-proposals/{worktree_id}.patch", patch, content_type="text/x-diff") + if patch + else None + ) + result = { + "provider": provider, + "feedback": feedback, + "changed_files": changed_files, + "output": output[-12_000:], + "proposal": { + "worktree_id": worktree_id, + "base_revision": base_revision, + "branch": branch if keep_worktree else "", + "worktree_path": str(worktree) if keep_worktree else "", + "diff": patch, + "fingerprint": _sha256(patch.encode("utf-8")) if patch else "", + "diff_artifact": artifact, + }, + } + self.emit_result({"agent_run": result}) + return result + def playwright_journey(self, cases: list[dict[str, object]] | None = None) -> dict[str, object]: """Run bounded, read-only Playwright page checks for the supplied cases. @@ -1812,7 +2019,7 @@ def __init__(self) -> None: self._handlers: dict[str, Callable[[RunnerContext], None]] = {} def phase( - self, name: str + self, name: str, *, step_id: str | None = None ) -> Callable[[Callable[[RunnerContext], None]], Callable[[RunnerContext], None]]: """Register a function as a handler for one runner lifecycle phase. @@ -1827,7 +2034,8 @@ def phase( def register(handler: Callable[[RunnerContext], None]) -> Callable[[RunnerContext], None]: phase = canonical_phase(name) - self._handlers[phase] = handler + key = f"{phase}:{step_id}" if step_id else phase + self._handlers[key] = handler setattr(handler, "__orbit_phase__", phase) return handler @@ -1846,13 +2054,15 @@ def main(self) -> None: parser = argparse.ArgumentParser(description="Orbit runner phase") command = parser.add_mutually_exclusive_group(required=True) command.add_argument("--phase") + parser.add_argument("--step") command.add_argument("--graph", action="store_true") args = parser.parse_args() if args.graph: print(json.dumps(graph.definition(), ensure_ascii=False)) return phase = canonical_phase(args.phase) - handler = self._handlers.get(phase) + handler = self._handlers.get(f"{phase}:{args.step}") if args.step else None + handler = handler or self._handlers.get(phase) if handler is None: raise SystemExit(f"runner does not define phase: {args.phase}") context = RunnerContext( diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 0ed3f35..265bec7 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -27,6 +27,69 @@ def test_health_is_available(): assert response.json() == {"status": "ok"} +def test_system_readiness_reports_missing_system_ai_and_git(monkeypatch): + monkeypatch.setattr( + main_module.store, + "application_settings", + lambda: {"chat_model_profile_name": ""}, + ) + monkeypatch.setattr(main_module.store, "profiles", lambda: []) + monkeypatch.setattr(main_module.shutil, "which", lambda _: None) + + response = TestClient(app).get("/api/system/readiness") + + assert response.status_code == 200 + assert response.json() == { + "ready": False, + "checks": [ + { + "id": "system_ai", + "status": "blocked", + "detail": "profile_not_selected", + "settings_page": "settings", + }, + {"id": "git", "status": "blocked", "detail": "not_installed", "settings_page": None}, + ], + } + + +def test_mcp_server_exposes_openapi_backed_control_room_tools(): + headers = {"Accept": "application/json, text/event-stream", "Content-Type": "application/json"} + initialize = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "openorbit-test", "version": "1"}, + }, + } + with TestClient(app, base_url="http://localhost:3000") as client: + connected = client.post("/mcp/", headers=headers, json=initialize) + assert connected.status_code == 200 + session_id = connected.headers["mcp-session-id"] + session_headers = {**headers, "mcp-session-id": session_id} + + tools = client.post( + "/mcp/", + headers=session_headers, + json={"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + ) + resources = client.post( + "/mcp/", + headers=session_headers, + json={"jsonrpc": "2.0", "id": 3, "method": "resources/list", "params": {}}, + ) + + assert tools.status_code == 200 + assert '"name":"get_status"' in tools.text + assert '"name":"start_pipeline"' in tools.text + assert '"name":"act_on_pipeline"' in tools.text + assert resources.status_code == 200 + assert "openorbit://openapi" in resources.text + + def test_request_locale_prefers_the_browser_accept_language_priority(): request = Request( { @@ -337,83 +400,6 @@ def test_prompt_revisions_returns_immutable_prompt_diff(tmp_path, monkeypatch): assert revisions[2]["after"] == "after\n" -def test_commit_changes_returns_sdk_commit_range(tmp_path, monkeypatch): - monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") - store = store_module.ConsoleStore() - timestamp = store_module.now() - store._save( - Run( - id="commit-run", - workflow_id="workflow", - workflow_name="Workflow", - status="succeeded", - created_at=timestamp, - updated_at=timestamp, - step_results=[ - { - "phase": "execute", - "loop_index": 2, - "ended_at": timestamp.isoformat(), - "result": { - "commit_change": { - "before": "a" * 40, - "after": "b" * 40, - "changed_paths": ["src/agent.py"], - "commits": [{"sha": "b" * 40, "subject": "Improve agent"}], - "diff_artifact": {"relative_path": "commits/a..b.patch"}, - } - }, - } - ], - ) - ) - - changes = store.commit_changes("commit-run") - - assert changes[0]["before"] == "a" * 40 - assert changes[0]["commits"][0]["subject"] == "Improve agent" - assert changes[0]["changed_paths"] == ["src/agent.py"] - - -def test_commit_changes_includes_jgent_committed_source_candidate(tmp_path, monkeypatch): - monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") - store = store_module.ConsoleStore() - timestamp = store_module.now() - store._save( - Run( - id="jgent-commit-run", - workflow_id="workflow", - workflow_name="Jgent", - status="succeeded", - created_at=timestamp, - updated_at=timestamp, - step_results=[ - { - "phase": "before_each", - "loop_index": 1, - "ended_at": timestamp.isoformat(), - "result": { - "jgent_paired": { - "committed_source_candidate": { - "status": "committed_source_candidate", - "before": "a" * 40, - "after": "b" * 40, - "changed_paths": ["src/Jgent/Agent.cs"], - "commits": [{"sha": "b" * 40, "subject": "Improve Jgent"}], - } - } - }, - } - ], - ) - ) - - changes = store.commit_changes("jgent-commit-run") - - assert changes[0]["changed_paths"] == ["src/Jgent/Agent.cs"] - assert changes[0]["commits"][0]["subject"] == "Improve Jgent" - - @pytest.mark.parametrize("terminal_status", ["failed", "cancelled"]) def test_teardown_runs_after_a_failed_or_cancelled_iteration(tmp_path, monkeypatch, terminal_status): monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") @@ -919,6 +905,27 @@ def test_deleting_issue_management_items_hides_them_without_deleting_run_evidenc assert store.delete_issue_management_items([proposal["proposal_id"]]) == {"deleted": 0} +def test_issue_management_exposes_an_agent_worktree_diff(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "ISSUE_MANAGEMENT", tmp_path / "issue-management.yaml") + store = store_module.ConsoleStore() + proposal = { + "proposal_id": "run-1:1:agent", + "proposal": { + "kind": "agent_change", + "changed_files": ["prompt.md"], + "base_revision": "abc123", + "diff": "diff --git a/prompt.md b/prompt.md\n", + }, + } + monkeypatch.setattr(store, "proposal_lifecycles", lambda: [proposal]) + + assert store.issue_management_diff(proposal["proposal_id"]) == { + "diff": "diff --git a/prompt.md b/prompt.md\n", + "changed_files": ["prompt.md"], + "base_revision": "abc123", + } + + def test_legacy_saved_runner_is_planned_with_canonical_phases(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNNERS", tmp_path / "runners") store_module.RUNNERS.mkdir() @@ -961,8 +968,6 @@ def test_v1_openapi_contract_covers_control_room_assets_and_observability(): "/api/v1/dashboard", "/api/v1/logs", "/api/v1/improvements/analytics", - "/api/v1/improvements/iterations", - "/api/v1/improvements/proposals", "/api/v1/template-translations", } assert expected <= paths.keys() @@ -1076,7 +1081,6 @@ def test_v1_read_only_control_room_resources_are_available(): "/api/v1/telemetry", "/api/v1/logs", "/api/v1/improvements", - "/api/v1/improvements/proposals", "/api/v1/reported-issues", ): assert client.get(path).status_code == 200 @@ -1094,6 +1098,38 @@ def test_supervisor_result_requires_the_two_template_return_keys(): raise AssertionError("invalid supervisor result was accepted") +def test_registered_agent_result_requires_a_supervisor_evaluation(): + with pytest.raises(ValueError, match="registered agent result"): + store_module.ConsoleStore._validated_supervisor_result( + '{"improvements": [], "reported_issues": []}', require_evaluation=True + ) + + +def test_evaluation_request_is_limited_to_the_matching_iteration_and_candidate(): + run = SimpleNamespace( + step_results=[ + { + "loop_index": 1, + "candidate_id": "1-1", + "result": {"evaluation_request": {"subject": "agent_change", "feedback": "Updated retry"}}, + }, + { + "loop_index": 2, + "candidate_id": "2-1", + "result": {"evaluation_request": {"subject": "agent_change", "feedback": "Updated parser"}}, + }, + ] + ) + + assert store_module.ConsoleStore._evaluation_request_for_iteration(run, 2, "2-1") == { + "subject": "agent_change", + "feedback": "Updated parser", + "changed_files": [], + "validation": "", + } + assert store_module.ConsoleStore._evaluation_request_for_iteration(run, 2, "2-2") is None + + def test_supervisor_result_normalizes_a_numeric_string_score(): result = store_module.ConsoleStore._validated_supervisor_result( '{"evaluation":{"score":"8","approval":"pending","summary":"ok"},"improvements":[],"reported_issues":[]}' @@ -1101,6 +1137,19 @@ def test_supervisor_result_normalizes_a_numeric_string_score(): assert result["evaluation"]["score"] == 8.0 +def test_supervisor_result_accepts_supervisor_observed_persona_journeys(): + result = store_module.ConsoleStore._validated_supervisor_result( + '{"persona_journeys":[{"persona_id":"jp_nisa_beginner","behavior_trace":' + '{"persona_goal":"Understand my NISA portfolio",' + '"current_action":"I checked the rendered holdings",' + '"decision":"I did not record a trade while the values disagree",' + '"next_action":"I will verify the displayed allocation",' + '"evidence":"The visible ACWI holding is zero"}}],' + '"improvements":[],"reported_issues":[]}' + ) + assert result["persona_journeys"][0]["persona_id"] == "jp_nisa_beginner" + + def test_supervisor_result_accepts_a_persona_journey_trace(): result = store_module.ConsoleStore._validated_supervisor_result( '{"evaluation":{"score":8,"approval":"pending","summary":"ok","behavior_trace":' @@ -1293,10 +1342,9 @@ def test_runner_templates_separate_direct_user_journeys_from_external_commands() compile(improvement, "native-improvement-cycle.py", "exec") compile(json_agent, "json-agent-cycle.py", "exec") compile(probe_gate, "evidence-gated-probe-cycle.py", "exec") - assert "playwright_journey" in user_journey + assert "@runner.phase" in user_journey + assert "orbit_runner_kit" not in user_journey assert "ORBIT_ADAPTER_COMMAND" not in user_journey - assert "previous_supervisor_feedback" in user_journey - assert "user-journey-state" in user_journey assert "ORBIT_ADAPTER_COMMAND" in adapter assert "playwright_journey" not in improvement assert "complete_model" in improvement @@ -1317,7 +1365,7 @@ def test_runner_templates_separate_direct_user_journeys_from_external_commands() assert "Jgent" not in probe_gate -def test_site_exploration_quick_start_uses_the_langgraph_runner(): +def test_site_exploration_quick_start_declares_its_lifecycle(): store = store_module.ConsoleStore() quick_start = next( item for item in store._built_in_quick_starts() if item["id"] == "openorbit.site-exploration-review" @@ -1325,7 +1373,8 @@ def test_site_exploration_quick_start_uses_the_langgraph_runner(): runner = quick_start["assets"]["runner"] assert "LangGraph" in quick_start["description"] assert runner["template_id"] == "site-exploration" - assert "StateGraph" in runner["source"] + assert "@runner.phase" in runner["source"] + assert "orbit_runner_kit" not in runner["source"] assert "logout|signout|delete" in runner["source"] @@ -1344,6 +1393,21 @@ def preview(source): assert "@runner.phase" in captured["source"] +def test_shipped_template_graphs_are_previewable_with_unique_nodes(): + """Every shipped template publishes a valid graph for node-based execution.""" + store = store_module.ConsoleStore() + sources = [item["source"] for item in store.runner_templates()] + sources.extend(item["assets"]["runner"]["source"] for item in store._built_in_quick_starts()) + + for source in sources: + definition = store.preview_runner_graph(source) + assert definition is not None + nodes = definition["nodes"] + assert nodes + assert len({node["id"] for node in nodes}) == len(nodes) + assert all(node["phase"] for node in nodes) + + def test_saved_runner_graph_preview_uses_the_selected_version(monkeypatch): store = store_module.ConsoleStore() expected = {"nodes": [{"id": "start"}], "edges": []} @@ -1378,15 +1442,25 @@ def preview(source): @pytest.mark.parametrize( ("quick_start_id", "phases"), [ - ("openorbit.user-journey-smoke-test", ["before_all", "execute", "verify", "after_all"]), + ( + "openorbit.user-journey-smoke-test", + ["before_all", "before_all", "execute", "verify", "after_all"], + ), ("openorbit.site-exploration-review", ["before_all", "execute", "verify", "after_all"]), ( "openorbit.agent-self-improvement", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + [ + "before_all", + "before_all", + "before_each", + "execute", + "after_each", + "after_each", + ], ), ( "openorbit.ai-slo-drift-monitor", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ], ) @@ -1408,20 +1482,48 @@ def test_quick_start_runner_graph_matches_its_execution_purpose(monkeypatch, qui @pytest.mark.parametrize( ("template_id", "phases"), [ - ("user-journey-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "user-journey-cycle", + [ + "before_all", + "before_all", + "before_each", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ], + ), ( "external-command-adapter", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ( "native-improvement-cycle", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + [ + "before_all", + "before_all", + "before_each", + "before_each", + "execute", + "verify", + "after_each", + "after_all", + ], ), ("site-exploration", ["before_all", "execute", "verify", "after_all"]), - ("json-agent-cycle", ["before_all", "before_each", "execute", "verify", "after_each", "after_all"]), + ( + "source-aware-browser-journey", + ["before_all", "execute", "verify", "after_each", "after_all"], + ), + ( + "json-agent-cycle", + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ), ( "evidence-gated-probe-cycle", - ["before_all", "before_each", "execute", "verify", "after_each", "after_all"], + ["before_all", "before_all", "before_each", "execute", "verify", "after_each", "after_all"], ), ], ) @@ -1456,6 +1558,22 @@ def test_ai_slo_drift_quick_start_uses_a_recurring_evidence_gate(): assert "baseline" in quick_start["assets"]["prompt_template"]["content"] +def test_agent_self_improvement_quick_start_embeds_agent_parameters_in_its_runner(): + store = store_module.ConsoleStore() + quick_start = next( + item for item in store._built_in_quick_starts() if item["id"] == "openorbit.agent-self-improvement" + ) + + resolved = store._substitute( + quick_start["assets"]["runner"]["source"], + {"agent_provider": "claude-code", "agent_options": "--model sonnet"}, + ) + + assert 'AGENT_PROVIDER = "claude-code"' in resolved + assert 'AGENT_OPTIONS = "--model sonnet"' in resolved + assert "agent_provider" not in quick_start["build"] + + def test_ai_slo_drift_quick_start_persists_its_evaluator_command(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "CONFIG", tmp_path) monkeypatch.setattr(store_module, "SETTINGS", tmp_path / "settings.json") @@ -1829,6 +1947,25 @@ def test_target_test_case_sets_are_managed_as_assets(tmp_path, monkeypatch): assert updated["name"] == "Updated target tests" +def test_personas_are_managed_as_reusable_assets(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "PERSONAS", tmp_path / "personas.yaml") + store = store_module.ConsoleStore() + values = { + "id": "careful-investor", + "name": "Careful investor", + "locale": "en-US", + "timezone": "America/New_York", + "activity_windows": [{"days": ["mon"], "start": "08:00", "end": "18:00"}], + "definition": "# Goals\n\n- Understand the portfolio safely.\n\n# Constraints\n\n- Never place a real order.", + "context": {"plan": "free"}, + } + created = store.create_persona(values) + assert created["context"] == {"plan": "free"} + assert created["definition"].startswith("# Goals") + updated = store.update_persona("careful-investor", {**values, "name": "Cautious investor"}) + assert updated["name"] == "Cautious investor" + + def test_proposal_history_is_derived_from_evaluation_run_results(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") store = store_module.ConsoleStore() @@ -1882,24 +2019,83 @@ def test_proposal_history_is_derived_from_evaluation_run_results(tmp_path, monke "relative_path": "evidence.json", } ] - assert store.improvement_iteration_data("build-1") == [ - { - "build_id": "build-1", - "build_name": "Build 1", - "run_id": "run-1", - "iteration": 2, - "recorded_at": timestamp.isoformat(), - "data_files": [ + + +def test_issue_management_excludes_agent_assessment_feedback_from_issue_rationales(tmp_path, monkeypatch): + monkeypatch.setattr(store_module, "RUNS", tmp_path / "runs") + monkeypatch.setattr(store_module, "ISSUE_MANAGEMENT", tmp_path / "issue-management.yaml") + store = store_module.ConsoleStore() + timestamp = store_module.now() + issue = { + "title": "Observed policy claim", + "evaluation": {"approval": "approved", "score": 6, "summary": "Observed in the response."}, + } + store._save( + Run( + id="run-agent-review", + workflow_id="workflow", + workflow_name="Workflow", + status="succeeded", + created_at=timestamp, + updated_at=timestamp, + step_results=[ { - "label": "Iteration evidence", - "filename": "evidence.json", - "path": "/tmp/orbit/evidence.json", - "relative_path": "evidence.json", - "content_type": "", + "loop_index": 1, + "result": { + "agent_run": { + "feedback": "Added a policy guardrail.", + "changed_files": ["prompt.md"], + "proposal": {"fingerprint": "a" * 64}, + }, + "agent_proposal": {"issue": issue}, + }, } ], - } - ] + supervisor_results=[ + { + "iteration": 1, + "stage": "issue_assessment", + "response": { + "improvements": [ + {"title": "Issue proposal", "status": "proposed", "rationale": "Issue reason."} + ], + "reported_issues": [issue], + }, + }, + { + "iteration": 1, + "stage": "agent_proposal_assessment", + "response": { + "evaluation": { + "approval": "rejected", + "score": 2, + "summary": "Agent review summary.", + }, + "improvements": [ + { + "title": "Do not expose this as an Issue", + "status": "proposed", + "rationale": "Agent feedback.", + } + ], + "reported_issues": [], + }, + }, + ], + ) + ) + + lifecycle = store.proposal_lifecycles() + + assert [item["title"] for item in lifecycle] == ["Issue proposal"] + assessed_issue = lifecycle[0] + assert assessed_issue["proposal"]["agent_change"]["feedback"] == "Added a policy guardrail." + assert assessed_issue["proposal"]["agent_change"]["review"]["summary"] == "Agent review summary." + assert lifecycle[0]["decision_rationale"] == "Issue reason." + assert all(item["category"] == "other" for item in lifecycle) + managed_issue = next(item for item in store.issue_management_items() if item["title"] == "Issue proposal") + assert managed_issue["comments"][0]["body"] == "Agent review summary." + assert managed_issue["comments"][0]["assigner"] == "AI supervisor" def test_hello_accepts_unsaved_profile_settings(): @@ -1945,13 +2141,18 @@ def test_application_manager_prompt_is_separate_from_model_profiles(tmp_path, mo "manager_prompt_template": expected_prompt, "manager_output_locale": "en", "chat_model_profile_name": "", + "coding_agent_provider": "none", "assistant_tools": { "workspace_root": str(store_module.ROOT), "file_read_enabled": True, "file_search_enabled": True, "run_process_enabled": True, + "coding_agent_enabled": True, + "ui_context_enabled": True, + "ui_interaction_enabled": True, "terminal_enabled": True, "terminal_visible": True, + "mcp_server_url": "http://127.0.0.1:3000/mcp/", }, } store.save_settings( @@ -1965,6 +2166,9 @@ def test_application_manager_prompt_is_separate_from_model_profiles(tmp_path, mo } ) assert store.application_settings()["manager_prompt_template"] == expected_prompt + assert ( + store.save_application_settings({"coding_agent_provider": "kiro"})["coding_agent_provider"] == "kiro" + ) assert ( store.save_application_settings( { @@ -1993,6 +2197,21 @@ def test_application_manager_prompt_has_a_safe_default(tmp_path, monkeypatch): ) +def test_assistant_mcp_configuration_uses_a_valid_default_and_persists_json(tmp_path, monkeypatch): + mcp_config = tmp_path / "config" / "assistant-mcp.json" + monkeypatch.setattr(store_module, "ASSISTANT_MCP_CONFIG", mcp_config) + store = store_module.ConsoleStore() + + assert store.assistant_mcp_config()["content"] == ( + '{\n "mcpServers": {\n "openorbit": {\n "url": "http://127.0.0.1:3000/mcp/"\n }\n }\n}\n' + ) + saved = store.save_assistant_mcp_config('{"mcpServers": {"orbit": {"command": "uv"}}}') + + assert saved["content"] == '{\n "mcpServers": {\n "orbit": {\n "command": "uv"\n }\n }\n}\n' + with pytest.raises(ValueError, match="valid JSON"): + store.save_assistant_mcp_config("not json") + + def test_exact_legacy_manager_prompt_is_migrated_but_custom_prompt_is_preserved(tmp_path, monkeypatch): monkeypatch.setattr(store_module, "SETTINGS", tmp_path / "settings.json") (tmp_path / "settings.json").write_text( diff --git a/backend/tests/test_assistant_graph.py b/backend/tests/test_assistant_graph.py new file mode 100644 index 0000000..4ad16c3 --- /dev/null +++ b/backend/tests/test_assistant_graph.py @@ -0,0 +1,63 @@ +from app.assistant_graph import DEFAULT_MCP_URL, OrbitAssistantGraph, build_assistant_prompt +from app.providers import ModelSettings + + +class FakeProvider: + def complete(self, settings, prompt): + return f"plain:{prompt}" + + def complete_with_tools(self, settings, prompt, tools, execute): + assert tools == [{"name": "get_status", "description": "status", "parameters": {}}] + return f"tools:{execute('get_status', {})}" + + +class FakeLocalTools: + def definitions(self): + return [] + + def execute(self, name, arguments): + raise AssertionError("local tools should not receive an MCP tool call") + + +class FakeMcpTools: + def definitions(self): + return [{"name": "get_status", "description": "status", "parameters": {}}] + + def execute(self, name, arguments): + assert (name, arguments) == ("get_status", {}) + return '{"health":"ok"}' + + +def test_assistant_graph_routes_mcp_tools_through_the_langgraph_model_node(): + activity = [] + graph = OrbitAssistantGraph( + FakeProvider(), + ModelSettings(), + FakeLocalTools(), + FakeMcpTools(), + on_activity=lambda phase, tool=None: activity.append((phase, tool)), + ) + + assert graph.invoke("status") == 'tools:{"health":"ok"}' + assert activity == [("thinking", None), ("working", "get_status"), ("thinking", None)] + + +def test_assistant_prompt_uses_local_mcp_as_the_control_room_source_of_truth(): + prompt = build_assistant_prompt("What changed?", [("assistant", "Earlier answer")], "ko-KR") + + assert DEFAULT_MCP_URL == "http://127.0.0.1:3000/mcp/" + assert "OpenOrbit MCP tools as the source of truth" in prompt + assert "Execution environment:" in prompt + assert "- OS: " in prompt + assert "- Default shell: " in prompt + assert "- Current working directory: " in prompt + assert "- Current local time: " in prompt + assert "Respond in BCP 47 locale 'ko-KR'." in prompt + assert "Assistant: Earlier answer" in prompt + + +def test_assistant_prompt_prioritizes_ui_tools_for_browser_ui_requests(): + prompt = build_assistant_prompt("What screen is open?", [], "en", ui_enabled=True) + + assert "first call ui_get_context" in prompt + assert "Do not use file tools to infer browser UI state" in prompt diff --git a/backend/tests/test_assistant_tools.py b/backend/tests/test_assistant_tools.py index 88c053b..bd6fb14 100644 --- a/backend/tests/test_assistant_tools.py +++ b/backend/tests/test_assistant_tools.py @@ -13,8 +13,40 @@ def tool_settings(workspace, **overrides): } -def test_disabled_tools_are_not_advertised(tmp_path): - assert AssistantToolExecutor(tool_settings(tmp_path)).definitions() == [] +def test_coding_agent_is_advertised_even_without_a_selected_provider(tmp_path): + tools = AssistantToolExecutor(tool_settings(tmp_path)) + + assert [definition["name"] for definition in tools.definitions()] == ["coding_agent"] + result = json.loads(tools.execute("coding_agent", {"task": "inspect this project"})) + assert result["error_code"] == "coding_agent_not_configured" + assert result["settings_page"] == "settings" + + +def test_coding_agent_can_be_disabled_in_assistant_tool_settings(tmp_path): + tools = AssistantToolExecutor(tool_settings(tmp_path, coding_agent_enabled=False)) + + assert tools.definitions() == [] + + +def test_coding_agent_passes_workspace_environment_to_selected_cli(tmp_path, monkeypatch): + captured = {} + + def fake_run(command, **kwargs): + captured["command"] = command + captured.update(kwargs) + return type("Result", (), {"stdout": "done", "stderr": "", "returncode": 0})() + + monkeypatch.setattr("app.assistant_tools.shutil.which", lambda _: "/usr/bin/kiro-cli") + monkeypatch.setattr("app.assistant_tools.subprocess.run", fake_run) + tools = AssistantToolExecutor(tool_settings(tmp_path), coding_agent_provider="kiro") + + result = json.loads(tools.execute("coding_agent", {"task": "Update the README"})) + + assert captured["command"][:2] == ["kiro-cli", "chat"] + assert f"- Current working directory: {tmp_path}" in captured["command"][-1] + assert "- OS: " in captured["command"][-1] + assert captured["cwd"] == tmp_path + assert result["provider"] == "kiro" def test_file_read_is_bounded_to_workspace_and_line_range(tmp_path): diff --git a/backend/tests/test_assistant_ui.py b/backend/tests/test_assistant_ui.py new file mode 100644 index 0000000..a727570 --- /dev/null +++ b/backend/tests/test_assistant_ui.py @@ -0,0 +1,60 @@ +import json +import threading + +from app.assistant_ui import AssistantUiBroker, AssistantUiToolExecutor + + +def test_ui_broker_round_trips_a_tool_command_to_the_browser_session(): + broker = AssistantUiBroker() + sent = {} + sent_event = threading.Event() + + def send(command_id, command): + sent.update(id=command_id, command=command) + sent_event.set() + + broker.open("browser-1", send) + result = {} + + def request(): + result.update(broker.request("browser-1", {"type": "get_context"}, timeout=1)) + + thread = threading.Thread(target=request) + thread.start() + assert sent_event.wait(1) + assert sent["command"] == {"type": "get_context"} + assert broker.resolve("browser-1", sent["id"], {"ok": True, "revision": 3}) + thread.join(1) + assert result == {"ok": True, "revision": 3} + + +def test_ui_tool_executor_returns_browser_context_as_a_tool_result(): + broker = AssistantUiBroker() + sent = {} + sent_event = threading.Event() + + def send(command_id, command): + sent.update(id=command_id, command=command) + sent_event.set() + + broker.open("browser-1", send) + tools = AssistantUiToolExecutor(broker, "browser-1", context_enabled=True, interaction_enabled=True) + result = {} + + def execute(): + result["value"] = json.loads(tools.execute("ui_get_context", {"component_id": "runs"})) + + thread = threading.Thread(target=execute) + thread.start() + assert sent_event.wait(1) + assert sent["command"] == {"type": "get_context", "component_id": "runs"} + broker.resolve("browser-1", sent["id"], {"ok": True, "components": []}) + thread.join(1) + assert result["value"] == {"ok": True, "components": []} + + +def test_ui_context_and_interaction_tools_can_be_enabled_separately(): + broker = AssistantUiBroker() + read_only = AssistantUiToolExecutor(broker, "browser-1", context_enabled=True, interaction_enabled=False) + + assert [tool["name"] for tool in read_only.definitions()] == ["ui_get_context"] diff --git a/backend/tests/test_orbit_sdk.py b/backend/tests/test_orbit_sdk.py index 3cae02e..a9e36bb 100644 --- a/backend/tests/test_orbit_sdk.py +++ b/backend/tests/test_orbit_sdk.py @@ -54,6 +54,16 @@ def collect() -> None: assert graph.definition()["nodes"][0]["phase"] == "before_each" +def test_graph_exports_after_supervision_only_when_enabled(): + graph = sdk.Graph() + + @graph.step("propose", phase="after_each", after_supervision=True) + def propose() -> None: + pass + + assert graph.definition()["nodes"][0]["after_supervision"] is True + + def test_function_trace_emits_successful_function_evidence(tmp_path, capsys): ctx = context(tmp_path, iteration=1) @@ -63,6 +73,77 @@ def test_function_trace_emits_successful_function_evidence(tmp_path, capsys): assert "collect-source-evidence" in capsys.readouterr().out +def test_register_evaluation_emits_an_agent_change_request(tmp_path, capsys): + result = context(tmp_path, iteration=2).register_evaluation( + "Updated the retry behavior after reproducing the timeout.", + changed_files=["src/retry.py", "src/retry.py", "tests/test_retry.py"], + validation="pytest tests/test_retry.py", + ) + + emitted = [ + json.loads(line.removeprefix("__ORBIT_RESULT__")) + for line in capsys.readouterr().out.splitlines() + if line.startswith("__ORBIT_RESULT__") + ] + assert result["subject"] == "agent_change" + assert result["changed_files"] == ["src/retry.py", "tests/test_retry.py"] + assert emitted[-1]["evaluation_request"] == result + + +def test_register_evaluation_requires_actionable_feedback(tmp_path): + with pytest.raises(ValueError, match="feedback"): + context(tmp_path, iteration=1).register_evaluation(" ") + + +def test_run_ai_agent_uses_the_selected_cli_and_only_marker_feedback(tmp_path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + subprocess.run(["git", "init"], cwd=project, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=project, check=True) + subprocess.run(["git", "config", "user.name", "Orbit Test"], cwd=project, check=True) + (project / "README.md").write_text("test\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run(["git", "commit", "-m", "initial"], cwd=project, check=True, capture_output=True) + monkeypatch.setattr(sdk, "ORBIT_APP_DATA", tmp_path / "orbit-data") + ctx = context(project, iteration=1) + commands: list[list[str]] = [] + + def fake_exec(command, **_kwargs): + commands.append(command) + if command[0] == "codex": + return "completed work\nORBIT_AGENT_FEEDBACK: Updated retry behavior\n" + if command[:3] == ["git", "diff", "--name-only"]: + return "src/retry.py\n" + if command[:3] == ["git", "diff", "--binary"]: + return "diff --git a/src/retry.py b/src/retry.py\n" + return "" + + monkeypatch.setattr(ctx, "exec", fake_exec) + + result = ctx.run_ai_agent("Fix the retry behavior", provider="codex", options="--model gpt-5") + + assert commands[0][:4] == ["git", "worktree", "add", "-b"] + assert commands[0][5].startswith(str(tmp_path / "orbit-data" / "agent-worktrees")) + assert commands[1:] == [ + [ + "codex", + "exec", + "--approve-for-me", + "--model", + "gpt-5", + "Fix the retry behavior", + ], + ["git", "add", "--intent-to-add", "--all"], + ["git", "diff", "--name-only", "--"], + ["git", "diff", "--binary", "--"], + ] + assert result["feedback"] == "Updated retry behavior" + assert result["changed_files"] == ["src/retry.py"] + assert result["proposal"]["diff"] == "diff --git a/src/retry.py b/src/retry.py\n" + assert result["proposal"]["base_revision"] + assert result["proposal"]["branch"].startswith("orbit/agent-proposal/run-123/") + + def test_graph_step_automatically_traces_its_execution(tmp_path, capsys): graph = sdk.Graph() diff --git a/docs/API.md b/docs/API.md index 0c50cb0..d5e9082 100644 --- a/docs/API.md +++ b/docs/API.md @@ -6,6 +6,18 @@ each execution is a **pipeline**. The interactive OpenAPI documentation is available from a running server at `/api/docs`; the raw contract is `/api/openapi.json`. +## MCP server + +The same local process exposes a Streamable HTTP [Model Context Protocol +(MCP)](https://modelcontextprotocol.io/) endpoint at `/mcp/`. Point an MCP +client at `http://localhost:3000/mcp/`. It provides project and pipeline +inspection, pipeline start and approval actions, improvement lifecycle lookup, +and an emergency-stop tool. The `openorbit://openapi` MCP resource exposes the +generated OpenAPI contract for the full v1 HTTP API. + +MCP has the same local-first security boundary as the HTTP API: it has no +built-in authentication and must not be exposed directly to untrusted networks. + ## Local-first security The API has no built-in authentication in this release. It is intended for the diff --git a/frontend/e2e/orbit-ui.spec.ts b/frontend/e2e/orbit-ui.spec.ts index 99c6d62..07fa0c4 100644 --- a/frontend/e2e/orbit-ui.spec.ts +++ b/frontend/e2e/orbit-ui.spec.ts @@ -37,6 +37,19 @@ test('collapses the sidebar and preserves the preference after reload', async ({ await page.getByRole('button', { name: 'Expand navigation' }).click() }) +test('uses the default local MCP connection through the Orbit assistant', async ({ page }) => { + await page.goto('/dashboard') + await page.getByRole('button', { name: 'Open Orbit assistant' }).click() + const assistant = page.getByLabel('Orbit assistant') + await assistant.getByRole('textbox', { name: 'Message' }).fill( + 'Use the OpenOrbit MCP get_status tool and briefly report the local service health.', + ) + await assistant.getByRole('button', { name: 'Send message' }).click() + const response = assistant.locator('.chat-message--assistant').last() + await expect(response).not.toContainText('Chat request failed.', { timeout: 30_000 }) + await expect(response).toContainText(/health|healthy|status|ok/i, { timeout: 30_000 }) +}) + test('opens the build and profile editors with editable lifecycle fields', async ({ page }) => { await page.goto('/builds') await expect(page.getByRole('heading', { name: 'Builds' })).toBeVisible() diff --git a/frontend/package.json b/frontend/package.json index 28331b9..ac61b86 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,6 +11,8 @@ }, "dependencies": { "@codemirror/lang-html": "^6.4.12", + "@codemirror/lang-json": "^6.0.2", + "@codemirror/lang-markdown": "^6.5.2", "@codemirror/lang-python": "^6.2.1", "@codemirror/lang-yaml": "^6.1.3", "@codemirror/theme-one-dark": "^6.1.3", @@ -20,11 +22,14 @@ "@xterm/addon-fit": "^0.10.0", "@xterm/xterm": "^5.5.0", "@xyflow/react": "^12.11.6", + "locale-codes": "^2.0.0", "lucide-react": "latest", "react": "latest", "react-dom": "latest", "react-icons": "^5.7.0", "react-markdown": "^10.1.0", + "react-select": "^5.10.2", + "react-timezone-select": "^3.3.3", "recharts": "^3.10.1", "remark-gfm": "^4.0.1", "vite": "latest" diff --git a/frontend/src/app/app-shell.tsx b/frontend/src/app/app-shell.tsx index 30ee7c0..87c8933 100644 --- a/frontend/src/app/app-shell.tsx +++ b/frontend/src/app/app-shell.tsx @@ -7,13 +7,13 @@ import { FileCode2, PanelLeftClose, Play, - CircleAlert, + TriangleAlert, Settings, Sparkles, } from "lucide-react"; import { SiGithub } from "react-icons/si"; import { useEffect, useState, type ReactNode } from "react"; -import type { Page } from "../domain/models"; +import type { Page, SystemReadiness } from "../domain/models"; import type { Locale } from "../locales"; import { localeMessages, locales } from "../locales"; import { ChatAssistant } from "../components/chat-assistant"; @@ -27,6 +27,7 @@ export function AppShell({ theme, headerAction, activeRunCount = 0, + readiness, children, }: { page: Page; @@ -35,6 +36,7 @@ export function AppShell({ theme: string; headerAction?: ReactNode; activeRunCount?: number; + readiness: SystemReadiness | null; children: ReactNode; }) { const t = locales[locale]; @@ -61,7 +63,6 @@ export function AppShell({ ["builds", , t.builds], ["runs", , t.runs], ["improvements", , t.improvements], - ["issues", , t.issues], ["settings", , t.settings], ]; const dashboardRepositoryLinks = page === "dashboard" && ( @@ -120,6 +121,13 @@ export function AppShell({ githubLabel: string; githubTitle: string; }>(locale, "navigationAccessibility"); + const readinessCopy = localeMessages<{ + title: string; + description: string; + openSettings: string; + checks: Record }>; + }>(locale, "systemReadiness"); + const blockedChecks = readiness?.checks.filter((check) => check.status === "blocked") ?? []; const sidebarLabel = collapsed ? navigationLabels.expandNavigation : navigationLabels.collapseNavigation; @@ -192,6 +200,26 @@ export function AppShell({ {headerAction} + {blockedChecks.length > 0 && ( +
+
+ )} {children}