diff --git a/.gitignore b/.gitignore index 087ba9b20..10187e440 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,18 @@ rust/debug .artifacts/ rbnx-deploy tmp* -.libero/ \ No newline at end of file +.libero/ + +# memory service +agent_milvus.db* +agent_memory/ + +# python venv +venv/ +.venv/ + +# rbnx build +rbnx-build/ + +# trae IDE +.trae/ \ No newline at end of file diff --git a/docs b/docs index d268247ee..dcd86db22 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d268247ee44bd91f02e55eab1657bbb63c210fdf +Subproject commit dcd86db2265a572261f3e797fa3c73978b42e7d8 diff --git a/rust/crates/robonix-agent/src/react.rs b/rust/crates/robonix-agent/src/react.rs index cee8341b2..df7f0ce6b 100644 --- a/rust/crates/robonix-agent/src/react.rs +++ b/rust/crates/robonix-agent/src/react.rs @@ -108,6 +108,16 @@ where let mut vlm = vlm.lock().await; let mut history = history.lock().await; + if user_input.trim() == "quit" { + // Trigger memory compaction at session end if available + let mcp_tools_for_quit = load_mcp_tools(&mut sdk).await.unwrap_or_default(); + if mcp_tools_for_quit.contains_key("compact_memory") { + log::info!("[mcp] Compacting memory session before exit..."); + let _ = execute_mcp_tool(&mut sdk, &mcp_tools_for_quit, "compact_memory", "{}").await; + } + return Ok(String::new()); + } + let mcp_tools = load_mcp_tools(&mut sdk).await?; let merged_skills = match skills::load_merged_skills(&mut sdk).await { Ok(s) => s, @@ -116,11 +126,28 @@ where Vec::new() } }; + + let mut memory_context = String::new(); + let is_casual = user_input.trim().to_lowercase(); + let skip_memory = is_casual == "hi" || is_casual == "hello" || is_casual.starts_with("who are you") || is_casual == "你是谁" || is_casual == "你好"; + + if !skip_memory && mcp_tools.contains_key("search_memory") { + let args = serde_json::json!({ + "query": user_input + }); + let args_str = args.to_string(); + if let Ok(result) = execute_mcp_tool(&mut sdk, &mcp_tools, "search_memory", &args_str).await { + if !result.contains("No relevant memories found") { + memory_context = result; + } + } + } + let mcp_tool_defs = build_mcp_tool_defs(&mcp_tools); let mut all_tools = tools::builtin_tool_defs(); all_tools.extend(mcp_tool_defs); let soul = skills::load_agent_soul(); - let system_prompt = build_system_prompt(soul.as_deref(), &mcp_tools, &merged_skills); + let system_prompt = build_system_prompt(soul.as_deref(), &mcp_tools, &merged_skills, &memory_context); log::info!( "tools for this turn: {} total ({} builtin + {} mcp), skills={}", all_tools.len(), @@ -324,7 +351,12 @@ fn truncate_log(s: &str, max: usize) -> String { if s.len() <= max { s.to_string() } else { - format!("{}...", &s[..max]) + // Find the nearest valid character boundary before `max` + let mut max_idx = max; + while max_idx > 0 && !s.is_char_boundary(max_idx) { + max_idx -= 1; + } + format!("{}...", &s[..max_idx]) } } @@ -370,6 +402,7 @@ fn build_system_prompt( soul: Option<&str>, mcp_tools: &HashMap, skills: &[AgentSkill], + memory_context: &str, ) -> String { let mut p = String::new(); if let Some(s) = soul { @@ -380,6 +413,13 @@ fn build_system_prompt( p.push_str("\n\n---\n\n"); } } + + if !memory_context.is_empty() { + p.push_str("## Relevant past memories (System Context)\n\n"); + p.push_str(memory_context); + p.push_str("\n\n---\n\n"); + } + p.push_str( "\ You are the Robonix system agent running on a robotic platform. @@ -393,6 +433,8 @@ Rules: - If a request requires multiple steps, execute them all before responding. - When the user references something from earlier in the conversation, use that context. - Keep responses concise and direct. +- If you are unsure about historical context, user preferences, or past decisions, use the `search_memory` tool. +- When you learn an important user preference or complete a major milestone, use `save_memory` to record it. - When a tool returns a JSON result with a clear outcome (success/failure, error, etc.), report that result to the user immediately. - Use **list_dir** / **read_file** for skill playbooks (see Skill workflow) or when the user diff --git a/rust/examples/packages/memsearch_service/memsearch_service/__init__.py b/rust/examples/packages/memsearch_service/memsearch_service/__init__.py new file mode 100644 index 000000000..afd1f0ede --- /dev/null +++ b/rust/examples/packages/memsearch_service/memsearch_service/__init__.py @@ -0,0 +1 @@ +"""Memsearch MCP Service package.""" diff --git a/rust/examples/packages/memsearch_service/memsearch_service/service.py b/rust/examples/packages/memsearch_service/memsearch_service/service.py new file mode 100644 index 000000000..f135adfe9 --- /dev/null +++ b/rust/examples/packages/memsearch_service/memsearch_service/service.py @@ -0,0 +1,153 @@ +import json +import os +import sys +import asyncio +import logging +from pathlib import Path + +# Suppress verbose gRPC / absl warnings from milvus-lite +os.environ["GRPC_VERBOSITY"] = "ERROR" +os.environ["GLOG_minloglevel"] = "2" +logging.getLogger("absl").setLevel(logging.ERROR) + +def _ensure_proto_gen() -> None: + d = Path(__file__).resolve().parent + while d.parent != d: + pg = d / "proto_gen" + if pg.is_dir() and (pg / "robonix_runtime_pb2.py").exists(): + sys.path.insert(0, str(pg)) + return + d = d.parent + +_ensure_proto_gen() +import grpc +import robonix_runtime_pb2 as pb +import robonix_runtime_pb2_grpc as pb_grpc + +from mcp.server.fastmcp import FastMCP +from memsearch import MemSearch + +mcp = FastMCP("memsearch_provider") + +# Initialize MemSearch +MEMORY_DIR = os.environ.get("AGENT_MEMORY_DIR", "./agent_memory") +MILVUS_URI = os.environ.get("AGENT_MILVUS_URI", "./agent_milvus.db") + +mem = MemSearch( + paths=[MEMORY_DIR], + embedding_provider="onnx", + milvus_uri=MILVUS_URI +) + +@mcp.tool() +async def search_memory(query: str) -> str: + """Search the agent's long-term memory for relevant past context, decisions, or user preferences.""" + results = await mem.search(query, top_k=2) + if not results: + return "No relevant memories found." + context = "\n\n".join(f"- {m['content']}" for m in results) + return f"Relevant memories:\n{context}" + +@mcp.tool() +async def compact_memory() -> str: + """Compact and summarize recent memories. Call this at the end of a session.""" + # Note: requires llm_provider setup for compacting, defaulting to openai if not set + try: + summary_path = await mem.compact(llm_provider="openai") + return f"Memory compacted successfully to {summary_path}." + except Exception as e: + return f"Failed to compact memory: {e}" + +@mcp.tool() +async def save_memory(content: str) -> str: + """Save an important fact, user preference, or decision to long-term memory.""" + from datetime import date + p = Path(MEMORY_DIR) / f"{date.today()}_notes.md" + p.parent.mkdir(parents=True, exist_ok=True) + with open(p, "a") as f: + f.write(f"\n{content}\n") + await mem.index() + return "Memory saved and indexed." + +def main(): + import threading + + # 1. Register with Robonix Control Plane + channel = grpc.insecure_channel(os.environ.get("ROBONIX_SERVER", "localhost:50051")) + stub = pb_grpc.RobonixRuntimeStub(channel) + + # Pick a port for FastMCP SSE + port = int(os.environ.get("MEMSEARCH_PORT", "50105")) + endpoint = f"http://127.0.0.1:{port}" + + metadata = { + "endpoint": endpoint, + "tools": [ + { + "name": "search_memory", + "description": "Search the agent's long-term memory for relevant past context, decisions, or user preferences.", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + }, + { + "name": "compact_memory", + "description": "Compact and summarize recent memories. Call this at the end of a session.", + "input_schema": {"type": "object", "properties": {}} + }, + { + "name": "save_memory", + "description": "Save an important fact, user preference, or decision to long-term memory.", + "input_schema": { + "type": "object", + "properties": { + "content": {"type": "string"} + }, + "required": ["content"] + } + } + ] + } + + try: + stub.RegisterNode( + pb.RegisterNodeRequest( + node_id="com.robonix.services.memsearch", + namespace="robonix/prm/memsearch", + kind="service", + skill_md="# Memsearch MCP\nProvides memory operations." + ) + ) + + stub.DeclareInterface( + pb.DeclareInterfaceRequest( + node_id="com.robonix.services.memsearch", + name="mcp_interface", + supported_transports=["mcp"], + metadata_json=json.dumps(metadata), + listen_port=port, + ) + ) + print(f"[memsearch-service] Registered node robonix/prm/memsearch with endpoint {endpoint}") + except Exception as e: + print(f"[memsearch-service] Warning: Failed to register with control plane: {e}") + + # Initialize memory index once + asyncio.run(mem.index()) + + # Start FastMCP server using streamable HTTP + print(f"[memsearch-service] Starting MCP Streamable HTTP server on port {port}...") + import uvicorn + import logging + # Suppress verbose gRPC / absl warnings from milvus-lite + logging.getLogger("absl").setLevel(logging.ERROR) + + app = mcp.streamable_http_app() + uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning") + +if __name__ == "__main__": + main() diff --git a/rust/examples/packages/memsearch_service/requirements.txt b/rust/examples/packages/memsearch_service/requirements.txt new file mode 100644 index 000000000..59d7fb3e6 --- /dev/null +++ b/rust/examples/packages/memsearch_service/requirements.txt @@ -0,0 +1,6 @@ +mcp +fastmcp +memsearch[onnx] +grpcio +protobuf +uvicorn diff --git a/rust/examples/packages/memsearch_service/robonix_manifest.yaml b/rust/examples/packages/memsearch_service/robonix_manifest.yaml new file mode 100644 index 000000000..758e9c14d --- /dev/null +++ b/rust/examples/packages/memsearch_service/robonix_manifest.yaml @@ -0,0 +1,20 @@ +manifestVersion: 1 + +build: + script: scripts/build.sh + +package: + id: com.robonix.example.memsearch_service + name: memsearch_service + version: 0.1.0 + vendor: robonix + description: Memsearch MCP Server for Agent memory + license: MulanPSL-2.0 + +nodes: + - id: com.robonix.services.memsearch + type: python + start: | + export PYTHONPATH="$(pwd):$(pwd)/../../proto_gen:${PYTHONPATH:-}" + # FastMCP internal logs can be verbose, redirect them to a file or suppress + exec python -m memsearch_service.service > memsearch.log 2>&1 diff --git a/rust/examples/packages/memsearch_service/scripts/build.sh b/rust/examples/packages/memsearch_service/scripts/build.sh new file mode 100644 index 000000000..ee5d9613a --- /dev/null +++ b/rust/examples/packages/memsearch_service/scripts/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MulanPSL-2.0 +set -euo pipefail +PKG="${RBNX_PACKAGE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +if [[ "${RBNX_BUILD_CLEAN:-}" == "1" ]]; then + rm -rf "$PKG/rbnx-build" +fi +EXAMPLES="$(cd "$PKG/../.." && pwd)" +PROTO_GEN="$EXAMPLES/proto_gen" +mkdir -p "$PKG/rbnx-build/ws/install" +cat >"$PKG/rbnx-build/ws/install/setup.bash" <=1.78; older grpcio raises RuntimeError at import. -grpcio>=1.78.0 -grpcio-tools>=1.78.0 +# proto_gen/*_grpc.py is generated with grpcio>=1.78; older grpcio raises RuntimeError at import.# Core +grpcio>=1.60.0 +grpcio-tools>=1.60.0 protobuf>=6.30,<7 openai>=1.0 -mcp>=1.0 +jinja2 +typeguard + +# Memory Service +fastmcp>=0.1.0 +mcp>=1.0.0 +memsearch[onnx] numpy Pillow uvicorn diff --git a/rust/examples/run.sh b/rust/examples/run.sh index 38b479dfd..70913eb9c 100755 --- a/rust/examples/run.sh +++ b/rust/examples/run.sh @@ -29,12 +29,13 @@ fi export ROBONIX_SERVER="${ROBONIX_SERVER:-127.0.0.1:50051}" export ROBONIX_META_GRPC_ENDPOINT="${ROBONIX_META_GRPC_ENDPOINT:-$ROBONIX_SERVER}" -export RUST_LOG="${RUST_LOG:-robonix_server=info,robonix_agent=info}" +export RUST_LOG="${RUST_LOG:-robonix_server=info,robonix_agent=warn}" export START_VLM_SERVICE="${START_VLM_SERVICE:-1}" export START_SIM_STACK="${START_SIM_STACK:-1}" export START_AGENT="${START_AGENT:-1}" +export START_MEMSEARCH="${START_MEMSEARCH:-1}" -export PYTHONPATH="${PACKAGES}/vlm_service${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH="${PACKAGES}/vlm_service:${PACKAGES}/memsearch_service${PYTHONPATH:+:$PYTHONPATH}" rbnx() { (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "$@") @@ -50,7 +51,7 @@ rbnx_validate_build() { check_python_dep() { local module="$1" - python3 - </dev/null 2>&1 + python - </dev/null 2>&1 import importlib.util import sys sys.exit(0 if importlib.util.find_spec("$module") else 1) @@ -59,17 +60,18 @@ PY echo "[e2e] checking required Python modules..." missing=() -for m in grpc openai mcp numpy PIL uvicorn; do +for m in grpc openai mcp memsearch numpy PIL uvicorn; do check_python_dep "$m" || missing+=("$m") done if [ "${#missing[@]}" -ne 0 ]; then echo "[e2e] missing modules: ${missing[*]}" echo "[e2e] install: pip install -r \"${EXAMPLES_ROOT}/requirements.txt\"" + echo "[e2e] or make sure your virtual environment is active." exit 1 fi -if ! python3 -c 'import grpc; p=[int(x) for x in grpc.__version__.split(".")[:3]]; raise SystemExit(0 if tuple(p + [0]*(3-len(p))) >= (1,78,0) else 1)' 2>/dev/null; then - echo "[e2e] grpcio>=1.78 required — pip install -r \"${EXAMPLES_ROOT}/requirements.txt\"" +if ! python -c 'import grpc; p=[int(x) for x in grpc.__version__.split(".")[:3]]; raise SystemExit(0 if tuple(p + [0]*(3-len(p))) >= (1,60,0) else 1)' 2>/dev/null; then + echo "[e2e] grpcio>=1.60 required — pip install -r \"${EXAMPLES_ROOT}/requirements.txt\"" exit 1 fi @@ -89,6 +91,7 @@ fi pkill -9 -f 'robonix-server' 2>/dev/null || true pkill -9 -f 'robonix-agent' 2>/dev/null || true pkill -9 -f 'vlm_service.service' 2>/dev/null || true +pkill -9 -f 'memsearch_service.service' 2>/dev/null || true pkill -9 -f 'tiago_bridge.node' 2>/dev/null || true sleep 0.3 @@ -135,6 +138,13 @@ if [ "$START_VLM_SERVICE" = "1" ]; then sleep 1 fi +if [ "$START_MEMSEARCH" = "1" ]; then + echo "[e2e] rbnx start memsearch_service (background)..." + rbnx_validate_build "$PACKAGES/memsearch_service" + (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "${RBNX_START_OPTS[@]}" -p "$PACKAGES/memsearch_service" -n com.robonix.services.memsearch) & + sleep 1 +fi + if [ "$START_SIM_STACK" = "1" ]; then echo "[e2e] rbnx start tiago_sim_stack (background, docker compose)..." (cd "$RUST_ROOT" && cargo run -p robonix-cli -- "${RBNX_START_OPTS[@]}" -p "$PACKAGES/tiago_sim_stack" -n com.robonix.prm.tiago) &