diff --git a/accelerators/coral/server.py b/accelerators/coral/server.py index c7bd93ead..29a56d1a7 100644 --- a/accelerators/coral/server.py +++ b/accelerators/coral/server.py @@ -291,10 +291,38 @@ def tool_object_detect(args: dict) -> dict: return {"objects": objects, "count": len(objects), "model": MODELS[key]["file"], "image_width": w, "image_height": h} +def _out_path(args: dict, image: str, suffix: str) -> str: + """Where a tool may write its PNG: the caller's out_path, or beside the image. + + Writing anywhere new is the caller's choice; replacing a file that is not already a PNG never is (a source + image, a config or a record clobbered with mask bytes is data loss). An earlier output is replaced. The + same rule the Axera sidecar and the fleet caller's landing apply. + """ + out = args.get("out_path") + if out is not None and (not isinstance(out, str) or not out): + raise ValueError({"error": "bad_request", "detail": "out_path must be a non-empty string"}) + if not out: + out = os.path.splitext(image)[0] + suffix + d = os.path.dirname(os.path.abspath(out)) + if not os.path.isdir(d) or not os.access(d, os.W_OK): + raise ValueError({"error": "out_path_unwritable", "out_path": out}) + if os.path.lexists(out): + try: + with open(out, "rb") as fh: + is_png = fh.read(8) == b"\x89PNG\r\n\x1a\n" + except OSError: + is_png = False + if os.path.islink(out) or not os.path.isfile(out) or not is_png: + raise ValueError({"error": "out_path_exists", "out_path": out, + "detail": "an existing file that is not a PNG is never overwritten; choose a new path"}) + return out + + def tool_semantic_segment(args: dict) -> dict: image = args.get("image_path") if not isinstance(image, str) or not image: raise ValueError({"error": "bad_request", "detail": "image_path required"}) + out_path = _out_path(args, image, ".segmask.png") # refused before the TPU runs, never after key = "deeplab-pascal" it = _interpreter(key) d = it.get_input_details()[0] @@ -308,8 +336,9 @@ def tool_semantic_segment(args: dict) -> dict: t = t[0] mask = t.argmax(-1).astype(np.uint8) if t.ndim == 3 else t.astype(np.uint8) labels = _load_labels(key) - out_path = args.get("out_path") or (os.path.splitext(image)[0] + ".segmask.png") - Image.fromarray(mask, mode="L").resize((w, h), Image.NEAREST).save(out_path) + tmp = out_path + ".part.png" + Image.fromarray(mask, mode="L").resize((w, h), Image.NEAREST).save(tmp, format="PNG") + os.replace(tmp, out_path) ids, counts = np.unique(mask, return_counts=True) classes = [{"class_id": int(i), "label": labels[int(i)] if int(i) < len(labels) else str(int(i)), "pixels": int(c)} for i, c in zip(ids, counts)] diff --git a/changes/unreleased/fix-accel-independent-and-fallback.md b/changes/unreleased/fix-accel-independent-and-fallback.md new file mode 100644 index 000000000..603adc4d0 --- /dev/null +++ b/changes/unreleased/fix-accel-independent-and-fallback.md @@ -0,0 +1,10 @@ +### Fixed - an accelerator answers while the node's GPUs are leased, and a shared accelerator tool falls back to the next device + +- **Why.** Two rules kept accelerators idle while callers waited, both seen 2026-11-11. A text lease on a node's GPUs made `/fleet/dispatch` refuse every forwarded accelerator call with `node leased` for the whole measurement window, although an `accel` job runs on an NPU or TPU and never on a leased card. And a shared tool (`offload_object_detect`, `offload_classify_image`, `offload_image_embed`, `offload_semantic_segment`) had one owner and no second choice (ADR 0068: "nothing falls back at call time"), so with the owner's node off or refusing the tool deferred while two other devices that serve the same name were reachable. +- **What it does: the lease.** A GPU lease never refuses an `accel` job (`fleetnode.textLeaseAgainst`). Every other task type keeps its reading, and the lease is published in health as before. +- **What it does: the fallback.** The owner of a shared tool is tried first. When it cannot take the call (a defer, or a sidecar refusal about the device such as a missing model or a disabled NPU), the call runs on the next device that serves the name, in the listed order, through that device's own tool row and with only the arguments its schema declares. The answer carries `fallback: {from, reason, served_by}`; when no device answers, the owner's defer carries `fallback_tried`. A refusal of the caller's input is the answer and is not retried elsewhere. Nothing is remembered between calls. Both surfaces (the MCP tools and the agent loop) use one function, `agent.AccelFallback`, and a node running a forwarded job still cannot forward again. +- **Embeddings.** `offload_image_embed` and `offload_text_embed` move to another device only when the caller passes `any_space: true`, because that vector is in another space; without it the defer carries `fallback_withheld`. The argument is added to the schema only on a box where a fallback device exists. +- **Safety.** One security pass ran on the change; both findings are fixed here. A fallback could hand `out_path` to a device whose sidecar checks nothing: the Coral's `semantic_segment` overwrote any path it was given, the caller's own image included. The fallback now applies the lane's rule before any device runs (`out_path` is never the input image, its directory must exist, an existing target is replaced only when it is a regular PNG file; otherwise the defer carries `fallback_withheld`), and the Coral sidecar applies the same rule itself and writes through a temporary file. A fallback device that answers JSON `null` is a defer, not a crash. +- **What a caller sees.** A tool with a fallback device carries a `FALLBACK:` sentence in its description (kept by the compact tools profile), and `offload_status.accelerators..fallback_for` lists the names a device answers for another owner. A box with one device, or with no shared name, advertises exactly what it did. +- **Check.** Two tests, one per incident: an `accel` dispatch under a text lease is admitted while an image job on the same node is still refused, and through the MCP surface a tool whose owner has no node to reach runs on the next device with the owner named, an embedding stays put without `any_space`, and `any_space` never reaches a sidecar. +- **Docs.** ADR 0102 (supersedes one consequence of ADR 0068), `docs/systems/accelerators.md`, `docs/systems/gpu-lease.md`, the operator guide's lease paragraph. diff --git a/docs/OPERATOR-GUIDE.md b/docs/OPERATOR-GUIDE.md index 57b51b397..022d7739a 100644 --- a/docs/OPERATOR-GUIDE.md +++ b/docs/OPERATOR-GUIDE.md @@ -2330,7 +2330,8 @@ HAND-EDITED, while UNSTAMPED (every config rendered before 0.123.0) prints as a re-rendering with `install render`, never by hand-editing the file. **Lease in health (0.113.16).** `/fleet/health` carries `lease` while the node's GPU lease is held; a text lease makes the -node ineligible for new delegated work (and its dispatch answers 503, re-placeable). `gpu reserve --drain --unload-seat` and +node ineligible for new delegated work (and its dispatch answers 503, re-placeable). An `accel` job is the exception: it runs +on an NPU or TPU, never on a leased card, so a lease never refuses it (ADR 0102). `gpu reserve --drain --unload-seat` and `gpu release --warm-seat` are the maintenance verbs — see docs/systems/gpu-lease.md. **A held card is a place in line (0.115.2).** `gpu reserve` QUEUES behind a current holder for `--wait` (default 8h; diff --git a/docs/architecture/decisions/0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md b/docs/architecture/decisions/0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md index 1f77763a1..cdd5f525e 100644 --- a/docs/architecture/decisions/0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md +++ b/docs/architecture/decisions/0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md @@ -57,7 +57,8 @@ without an entry is decided exactly as before. so moving it changes the space a caller receives, and the tool description says which device serves it. - Nothing is chosen per call and nothing falls back at call time. If the named device's node is down, the call defers (ADR 0038) instead of running on the other device, which matches every other accelerator - lane. + lane. **Superseded by [ADR 0102](0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md):** + the named device is tried first, and the next device that serves the name answers when it cannot. ## Related diff --git a/docs/architecture/decisions/0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md b/docs/architecture/decisions/0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md new file mode 100644 index 000000000..f98f801a5 --- /dev/null +++ b/docs/architecture/decisions/0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md @@ -0,0 +1,112 @@ +--- +status: Accepted +date: "2026-11-11" +--- + +# An accelerator answers through a GPU lease, and a shared tool falls back to the next device + +## Context + +Two rules kept accelerators idle while callers waited. Both were seen on 2026-11-11, the day the AX650-class +NPU card ([ADR 0101](0101-axera-ax650-npu-card-as-an-accelerator.md)) took four shared tools on the delegator. + +**A reservation of the GPUs refused the NPU.** A text lease reserves a node's cards for a measurement, and +`/fleet/dispatch` refused every task type but an agent contract while one was held, because "the cards it +will use are chosen later, and unknown is every card". An `accel` job uses none of them: it runs on a TPU or +an NPU through a loopback sidecar, and on the NPU card in its own memory. For the hours a measurement held +the node's GPUs, every forwarded accelerator call was refused with `node leased`, with the device idle. + +**A shared tool had one device and no second choice.** A shared tool name has one owner per box +([ADR 0037](0037-a-capability-name-has-one-owner-per-box.md)), and the operator may name it +([ADR 0068](0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md)). ADR 0068 also decided +that "nothing is chosen per call and nothing falls back at call time". With detection, classification, +embedding and semantic segmentation handed to the NPU card on one node, those four tools deferred whenever +that node was off or refusing, while a Coral and an RK3588 NPU that serve the same names were reachable and +idle. The operator's standing order is that the harness adapts to the work. + +## Decision + +**1. A GPU lease never refuses an `accel` job.** `fleetnode.textLeaseAgainst` answers "not held" for the +`accel` task type. Every other task type keeps its reading: an agent contract by the seats it could run on, +the rest by the whole node. The lease is still published in `/fleet/health` unchanged. + +**2. The owner of a shared tool is the first choice, not the only one.** When the owner cannot take a call, +the call runs on the next device that serves the same name, in the listed order (local devices, then +`fleet_accelerators`), and the result says so. This supersedes ADR 0068's "nothing falls back at call time"; +the rest of ADR 0068 and ADR 0037 stand: one registered tool per name, the owner's description, schema and +model, the same owner on both surfaces. + +- **What "cannot take the call" means** (`agent.AccelCannotServe`). A defer: the node is off, no node + advertises the device, the dispatch was refused, the sidecar did not start. Or a sidecar refusal that is + about the device: a model that is missing, fails its digest or does not load, a disabled or absent NPU, an + internal error. A refusal of what the caller sent (`bad_request`, `image_missing`, `image_unreadable`, + `out_path_exists`, `out_path_unwritable`, `forbidden`, `not_found`, `unknown_model`) is an answer: another + device would refuse the same input. +- **Each fallback device runs its own tool.** Its own table row gives the sidecar tool, the required + argument and the schema. The caller sent the owner's arguments; only the ones that device's schema + declares travel to it. +- **The result is attributable.** A fallback answer carries + `fallback: {from, reason, served_by}`. When no device serves the call, the owner's answer comes back with + `fallback_tried: [{device, reason}]`. +- **An embedding does not move silently.** A vector from another device is in another space, and nothing in + the numbers says so. `offload_image_embed` and `offload_text_embed` fall back only when the caller passes + `any_space: true`; without it the owner's defer comes back with `fallback_withheld`, naming the devices + that could have answered. The argument is added to the tool's schema only on a box where a fallback device + exists, and it never reaches a sidecar. +- **`out_path` is checked before it travels.** The owner applied its own rule to `out_path` (the NPU card's + sidecar, or the caller-side landing of a fleet device). A fallback device's sidecar may apply none, and an + agent seat can pass any path, so the fallback applies the lane's rule itself before any device runs: + `out_path` is never the input image, its directory must exist, and an existing target is replaced only + when it is a regular file that is already a PNG. Otherwise the owner's answer comes back with + `fallback_withheld`. The Coral sidecar's `semantic_segment` gains the same rule at the sink, where it had + none (found by the security pass on this change). +- **Nothing is remembered between calls.** There is no "owner is down" state and no health pre-check: every + call tries the owner first. A node that is off costs the roster probe (about 2 s) before the fallback runs. +- **Where it lives.** At tool registration on both surfaces, through one function + (`agent.AccelFallback`): `mcpserver.handleAccelChain` and `agent.accelLaneTools`. A node running a + forwarded job calls its local lane directly (`pipeline.runAccelTask`), so a call cannot forward twice. +- **Status.** A device's `offload_status.accelerators` row gains `fallback_for`, the names it answers when + their owner cannot. `serves` and `owns` are unchanged. A tool with a fallback says so in its description. + +## Consequences + +- A measurement lease on a node's GPUs and accelerator work on the same node run side by side. The lease + holder's GPU numbers are untouched; the sidecar's pre- and post-processing does use host CPU for a few + milliseconds per call. +- A fallback answer comes from another model: a different detector, classifier or segmenter, with that + device's defaults for any argument the caller did not send. `fallback.served_by` and the result's `model` + say which one ran, and a caller that needs the owner's model reads them. +- A box with one device, or with no shared name, advertises exactly the tools it did before: descriptions and + schemas are extended as text and only where a fallback device exists. +- A call whose input is refused is not retried elsewhere; a call the owner fails for a device reason costs + one more call per fallback device before it defers. + +## Alternatives considered + +- **Keep the refusal and let the operator release the lease.** That is the rule that failed: the lease was + doing its job for the GPUs and had no claim on the NPU. +- **Scope the exemption to devices with their own memory.** The RK3588 NPU shares host RAM, but a lease + reserves GPUs, not RAM, and its declared RAM figure is for the holder's own load. One rule for every + accelerator is the one a reader can predict. +- **Route by health before the call.** It needs a second source of truth about the owner and goes stale + between the probe and the call. Trying the owner first reads the same health through `pickNode` and fails + in about 2 s. +- **Let an embedding fall back like the rest.** A similarity index built from two spaces is wrong without + any error, and two of the three devices can share a dimension. +- **Let the caller name a device per call.** It would work, and it is the device-suffixed tool ADR 0037 + rejected: a caller would have to know the hardware to ask for a capability. + +## Related code + +- `internal/fleetnode/leases.go` (`textLeaseAgainst`), `internal/fleetnode/accel_task_test.go` +- `internal/agent/accelfallback.go`, `internal/agent/acceltools.go` (`accelLaneTools`) +- `internal/mcpserver/acceltools.go` (`accelFallbackPlan`, `handleAccelChain`, `accelStatus`), + `internal/mcpserver/accelfallback_test.go` + +## Related docs + +- [systems/accelerators.md](../../systems/accelerators.md) +- [ADR 0037](0037-a-capability-name-has-one-owner-per-box.md), + [ADR 0038](0038-accelerator-work-travels-to-the-box-that-has-the-device.md), + [ADR 0068](0068-the-operator-may-name-the-owner-of-a-shared-accelerator-tool.md), + [ADR 0101](0101-axera-ax650-npu-card-as-an-accelerator.md) diff --git a/docs/architecture/decisions/README.md b/docs/architecture/decisions/README.md index c39bdd1a1..3465397f8 100644 --- a/docs/architecture/decisions/README.md +++ b/docs/architecture/decisions/README.md @@ -109,6 +109,7 @@ code, treat everything else as background. | [0099](0099-frontier-updates-floors-never-pins.md) | Accepted | Frontier updates: floors, never pins. Versions are never pinned: the installers and `local-offload update` resolve the newest complete build from the publisher and verify its digest; the only version literals are floors in `setup/floors.json`; a node keeps its current and previous generation behind a live path and rolls back on a failed gate; a lint keeps every other file clean; supersedes the second-build doctrine of [0081](0081-the-embeddinggemma2-entry-may-run-from-a-second-llama-build.md) | | [0100](0100-the-memory-wall-mapped-onto-the-fleet.md) | Proposed | The memory wall mapped onto the fleet (NVIDIA's five: NVFP4, WideEP, Dynamo, Vera, CMX; tiers G1-G4 plus G3.5): the reuse instrument reads vLLM's `cached_tokens`; llama.cpp slot files become the G3 tier on every one-slot seat at build b11506 or later, keyed node-side over the byte-exact prefix, saved on a prefix's second sighting, restored and judged by the next completion's cache share, and saved across a tool wait of 240 s or more so a seat that unloads mid-run restores instead of re-prefilling; nvfp4 KV beside a cache server is refused; WideEP, disaggregation, KV-aware routing, CMX hardware and tool-wait GPU-to-host offload are ruled out or already covered, each with its reason | | [0101](0101-axera-ax650-npu-card-as-an-accelerator.md) | Proposed | An AX650-class NPU on an M.2 card is the fourth accelerator, `axera-ax650`: fleet-visible, detected from `/proc/bus/pci/devices` (`1f4b0650` + driver `ax_pcie_dev_host`), its own 4 GB (2944 MiB CMM) never host RAM or VRAM, ten licence-clean publisher-compiled tools (RF-DETR, SigLIP2, DeepLabV3+, MobileSAM, Depth Anything 3, RTMPose, PP-OCRv5) with `offload_ocr` `engine:"npu"` going Hailo, then Axera, then fleet; ownership handed to it per tool by `accelerator_tool_owners`; telemetry read through the runtime library in a child process (never the single-instance vendor CLI); CPU pinned to its top step at boot; face, person re-id, low-light, speech and an NPU LLM seat not owned (later slices) | +| [0102](0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md) | Accepted | An accelerator answers through a GPU lease, and a shared tool falls back: a text lease never refuses an `accel` job (the device is not a card a lease names); the owner of a shared tool is the first choice and the next device that serves the name answers when it cannot (a defer or a device-side refusal, never a refusal of the caller's input), each fallback through its own table row with only the arguments it takes, the result carrying `fallback{from,reason,served_by}`; an embedding moves only with `any_space:true`; nothing is remembered between calls; supersedes ADR 0068's "nothing falls back at call time" | ## Lifecycle diff --git a/docs/systems/accelerators.md b/docs/systems/accelerators.md index 4c15cacce..34fcd9142 100644 --- a/docs/systems/accelerators.md +++ b/docs/systems/accelerators.md @@ -369,6 +369,34 @@ the one decision the MCP registration and the status block read; the loop's lane claims (`AccelLane.Claims`). Each device's status entry lists `serves` (what it registered) beside `owns` (what it could). +**The owner is the first choice, not the only one ([ADR 0102](../architecture/decisions/0102-an-accelerator-answers-through-a-gpu-lease-and-a-shared-tool-falls-back.md)).** +When the owner cannot take a call, the call runs on the next device that serves the same name, in the +listed order (local devices, then `fleet_accelerators`), on both surfaces (`agent.AccelFallback`, called +from `mcpserver.handleAccelChain` and `agent.accelLaneTools`). + +- **Cannot take the call** means a defer (the node is off, no node advertises the device, the dispatch was + refused, the sidecar did not start) or a sidecar refusal about the device (`model_missing`, + `model_load_failed`, a digest mismatch, `npu_disabled`, `npu_unavailable`, `runtime_missing`, `internal`). + A refusal of the caller's input (`bad_request`, `image_missing`, `image_unreadable`, `out_path_exists`, + `out_path_unwritable`, `forbidden`, `not_found`, `unknown_model`) is the answer and is not retried. +- **The fallback device runs its own tool row**: its sidecar tool, its required argument, and only the + arguments its schema declares. The model is that device's, so `score_threshold` defaults and the label + space of `offload_classify_image` are that device's too. +- **The result says so**: `fallback: {from, reason, served_by}` on an answer from another device, + `fallback_tried: [{device, reason}]` on the owner's defer when none could answer. +- **An embedding moves only when asked.** `offload_image_embed` and `offload_text_embed` fall back only with + `any_space: true`, because another device's vector is in another space; without it the defer carries + `fallback_withheld`. The argument is in the schema only where a fallback device exists. +- **`out_path` is checked before it travels**: never the input image, its directory must exist, and an + existing target is replaced only when it is a regular PNG file; otherwise the defer carries + `fallback_withheld` and no other device runs. The Coral sidecar's `semantic_segment` applies the same rule + itself since this release (it used to overwrite any path it was given), writing through a temporary file. +- **Nothing is remembered**: every call tries the owner first, and a node that is off costs the roster + probe (about 2 s) before the next device runs. +- `offload_status.accelerators..fallback_for` lists the names a device answers when their owner cannot, + and each such tool's description carries a `FALLBACK:` sentence (it names no device, so the compact + tools profile keeps it by its exact text; a forwarded tool's `[FLEET: ...]` note stays last). + **Both surfaces, generalised (0.114.0):** the MCP server keeps one per-device table (`internal/mcpserver/acceltools.go`) and one on-demand sidecar per device; the agent loop receives every device as an `agent.AccelLane` (`pipeline.NewLoopAccel`, config order) and @@ -562,6 +590,8 @@ entry; a Hailo owns those five too and keeps them unless an entry moves them. **Moving `offload_image_embed` changes the embedding space** (768-d `siglip2-base-p16-224` instead of the Coral's 1280-d EfficientNet or the Hailo's 512-d TinyCLIP): re-embed every stored vector, because vectors are comparable only within one space. `offload_status`'s `serves` per device shows what registered where. +When the card's node is off or cannot take a call, detection, classification and semantic segmentation run on +the next device that serves them, and the embedding moves only with `any_space: true` (ADR 0102, above). **Deliberately not owned**, each a ruling of [ADR 0101](../architecture/decisions/0101-axera-ax650-npu-card-as-an-accelerator.md): `face_detect` and `face_embed` (the InsightFace weights are non-commercial), `person_embed` (no AX650-compiled @@ -644,7 +674,14 @@ Ownership across the fleet is ADR 0037 extended: `accelerators` is walked before `accelerator_tool_owners` entry names the fleet device for it (ADR 0068, below), and a box that lists a device in both registers the local lane only. A box that lists nothing in `fleet_accelerators` is byte-identical to 0.114.x (pinned by -`TestFleetAcceleratorRegistersForwardedToolsOnly`). +`TestFleetAcceleratorRegistersForwardedToolsOnly`). The owner is tried first and the other devices that +serve the name answer when it cannot (ADR 0102, "The owner is the first choice" above). + +**A GPU lease never refuses an `accel` job (ADR 0102).** A text lease reserves a node's GPUs for a +measurement and its `/fleet/dispatch` answers `503 node leased` to work that would land on them. An +accelerator is not one of those cards, so an `accel` job is admitted while the lease is held and the two +run side by side (`fleetnode.textLeaseAgainst`). Before this a forwarded call was refused for the whole +measurement window with the device idle. **The Hailo-8L never travels** (register E-08, [ADR 0038](../architecture/decisions/0038-accelerator-work-travels-to-the-box-that-has-the-device.md) amendment): the standalone box that carries it keeps it local. `/fleet/health` omits it from diff --git a/docs/systems/gpu-lease.md b/docs/systems/gpu-lease.md index 89712965e..baabef934 100644 --- a/docs/systems/gpu-lease.md +++ b/docs/systems/gpu-lease.md @@ -3357,7 +3357,8 @@ local-offload gpu release --warm-seat **Alias-bound seats (0.113.20).** llama-swap's `/running` lists CANONICAL ids, while `agent_model` is normally an alias (`agent-pool` → `qwen3.8-27b-vllm`). The drain's reader matched `/running` by the configured name, so on an alias-bound seat it read "not loaded" and returned at once — a silent no-op from 0.113.16 to 0.113.19 on the reference workstation (, whose seat is bound by its id, drained correctly, which is why the live proofs passed). The reader now lives in `internal/seatload` and resolves the name through the roster before consulting `/running`; an unreadable roster falls back to the bare name. -Taking a text lease already makes the node a non-target (health `lease`, dispatch 503, the delegator's gate), but work placed +Taking a text lease already makes the node a non-target (health `lease`, dispatch 503, the delegator's gate; an `accel` job +is never refused, since it runs on an NPU or TPU and not on a leased card, ADR 0102), but work placed before the lease can still be in flight. `--drain` waits, after the lease is taken, until the agent seat reports nothing running or waiting: llama-swap's `/running` is read first — an unloaded seat is idle by definition, and its `/upstream//…` path is NEVER probed on an unloaded seat because that path loads the model on demand — then the loaded seat's own gauges diff --git a/internal/agent/accelfallback.go b/internal/agent/accelfallback.go new file mode 100644 index 000000000..992c20aac --- /dev/null +++ b/internal/agent/accelfallback.go @@ -0,0 +1,205 @@ +package agent + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "slices" + "strings" +) + +// Call-time fallback for a shared accelerator tool (ADR 0102). +// +// A shared tool name has one owner per box (ADR 0037, ADR 0068): that device's description, +// schema and model are what the caller sees. It used to be the only device that could ever +// answer as well, so a tool whose owner's node was off deferred while two other devices that +// serve the same name sat idle. The owner is now the FIRST choice: when it cannot take +// the call, the call runs on the next device that serves the name, in the listed order, and +// the result says so. Both surfaces (the MCP tools and the agent loop) use this one function, +// at the place a tool is registered. A node running a forwarded job never comes through here +// (pipeline.runAccelTask calls its local lane directly), so a call cannot forward twice. + +// AccelStep is one more device that serves a shared tool name: how to run the tool there. +type AccelStep struct { + // Device is the accelerator id. + Device string + // Props are the argument names this device's tool takes. The caller sent the owner's + // arguments; only these travel to this device. + Props []string + // Run calls the tool on this device and returns its dict (a defer included). + Run func(ctx context.Context, args map[string]any) map[string]any +} + +// AnySpaceArg is the caller's opt-in for an embedding from another device (see AccelFallback). +const AnySpaceArg = "any_space" + +// inputRefusals are the sidecars' refusals of what the caller SENT. Another device would refuse +// the same input, so they are answers, not a reason to try one. +var inputRefusals = []string{"bad_request", "image_missing", "image_unreadable", "out_path_exists", + "out_path_unwritable", "forbidden", "not_found", "unknown_model"} + +// AccelCannotServe says whether out means the device could not take the call, and why: a defer +// (its node is off, its sidecar did not start, the dispatch was refused), or a sidecar refusal +// that is about the device (a model that is missing or failed to load, a disabled or absent +// NPU, an internal error) and not about the caller's input. +func AccelCannotServe(out map[string]any) (reason string, cannot bool) { + if d, _ := out["deferred"].(bool); d { + reason, _ = out["reason"].(string) + return reason, true + } + kind, _ := out["error"].(string) + if kind == "" || slices.Contains(inputRefusals, kind) { + return "", false + } + if detail, _ := out["detail"].(string); detail != "" { + return kind + ": " + detail, true + } + return kind, true +} + +// AccelSpaceBound says whether a tool's answer only means something beside other answers from +// the SAME device: an embedding. A vector from another device is in another space, and nothing +// in the number itself says so. +func AccelSpaceBound(tool string) bool { return strings.HasSuffix(tool, "_embed") } + +// AccelFallback is called with the owner's answer when AccelCannotServe says the owner could not +// take the call. It runs the tool on each later device in order and returns the first answer a +// device gives, with a `fallback` block naming the owner, why it could not serve, and the device +// that did. When no device serves it, the owner's answer comes back with `fallback_tried`. +// +// An embedding does not move silently: the caller must pass any_space:true, because a vector +// from another device cannot be compared with the owner's. Without it the owner's answer comes +// back with `fallback_withheld` naming the devices that could have answered. +func AccelFallback(ctx context.Context, tool, owner string, out, args map[string]any, steps []AccelStep) map[string]any { + reason, cannot := AccelCannotServe(out) + if !cannot || len(steps) == 0 { + return out + } + if any, _ := args[AnySpaceArg].(bool); AccelSpaceBound(tool) && !any { + names := make([]string, len(steps)) + for i, s := range steps { + names[i] = s.Device + } + out["fallback_withheld"] = "not run on " + strings.Join(names, " or ") + ": an embedding from another device is in a different space and cannot be compared with " + owner + "'s; pass " + AnySpaceArg + ":true to take one anyway (the result's space names it)" + return out + } + if why := outPathRefusal(args); why != "" { + out["fallback_withheld"] = why + return out + } + var tried []map[string]string + for _, s := range steps { + if ctx.Err() != nil { + break + } + sub := map[string]any{} + for k, v := range args { + if slices.Contains(s.Props, k) { + sub[k] = v + } + } + got := s.Run(ctx, sub) + if got == nil { + got = map[string]any{"deferred": true, "reason": s.Device + ": empty answer"} + } + if why, failed := AccelCannotServe(got); failed { + tried = append(tried, map[string]string{"device": s.Device, "reason": why}) + continue + } + got["fallback"] = map[string]string{"from": owner, "reason": reason, "served_by": s.Device} + return got + } + out["fallback_tried"] = tried + return out +} + +// outPathRefusal says why a call's out_path must not travel to a fallback device, or "" when it +// may. The owner applied its own rule to out_path before it answered (the Axera sidecar's, or +// accelremote's landing for a fleet device); a fallback device's sidecar may apply none, and a +// model seat can pass any path. So the rule every guarded writer of this lane uses is applied +// here, before any device runs: out_path is never the input image, its directory must exist, +// and an existing target is replaced only when it is a regular file that is already a PNG. +func outPathRefusal(args map[string]any) string { + out, _ := args["out_path"].(string) + if out = strings.TrimSpace(out); out == "" { + return "" + } + if img, _ := args["image_path"].(string); strings.TrimSpace(img) != "" { + same := filepath.Clean(out) == filepath.Clean(img) + if a, ea := os.Stat(out); !same && ea == nil { + if b, eb := os.Stat(img); eb == nil { + same = os.SameFile(a, b) + } + } + if same { + return "not run on another device: out_path " + out + " is the input image and is never replaced" + } + } + if fi, err := os.Stat(filepath.Dir(out)); err != nil || !fi.IsDir() { + return "not run on another device: the directory of out_path " + out + " does not exist" + } + fi, err := os.Lstat(out) + if err != nil { + return "" // a new file + } + if fi.Mode().IsRegular() { + if f, oerr := os.Open(out); oerr == nil { + head := make([]byte, 8) + n, _ := io.ReadFull(f, head) + f.Close() + if n == 8 && string(head) == "\x89PNG\r\n\x1a\n" { + return "" // an earlier output + } + } + } + return "not run on another device: out_path " + out + " exists and is not a PNG file; it is never replaced" +} + +// AccelSchemaProps returns the argument names a tool's JSON schema declares. +func AccelSchemaProps(schema []byte) []string { + var s struct { + Properties map[string]json.RawMessage `json:"properties"` + } + if json.Unmarshal(schema, &s) != nil { + return nil + } + props := make([]string, 0, len(s.Properties)) + for k := range s.Properties { + props = append(props, k) + } + slices.Sort(props) + return props +} + +// The sentences a tool's description gains when another device can answer in its owner's place. +// They name no device, so one sentence serves every box (the compact tools profile keeps it by +// exact text); offload_status.accelerators..fallback_for names the devices. +const ( + AccelFallbackNote = "FALLBACK: when this device cannot take the call, the next device that serves this tool answers with its own model (arguments it does not take are dropped), and the result's fallback{from,reason,served_by} names it." + AccelFallbackSpaceNote = "FALLBACK only with " + AnySpaceArg + ":true: when this device cannot take the call, another device that serves this tool answers in a DIFFERENT embedding space, and the result's space and fallback{from,reason,served_by} name it." + anySpaceProp = `,"` + AnySpaceArg + `":{"type":"boolean","description":"true = when this device cannot take the call, accept another device's vector, in ANOTHER embedding space that cannot be compared with this one's (default false: the call defers)"}` +) + +// AccelFallbackSurface returns the owner's description and schema as a caller should see them +// when another device can answer in the owner's place (fallback). Both are extended as text, +// never re-encoded, and a tool with no other device is returned unchanged, so a box with one +// device advertises exactly what it always did. The sentence goes before a trailing +// "[FLEET: ...]" forwarding note, which stays the description's last sentence. +func AccelFallbackSurface(tool, desc, schema string, fallback bool) (string, string) { + if !fallback { + return desc, schema + } + note := AccelFallbackNote + if AccelSpaceBound(tool) { + note = AccelFallbackSpaceNote + if i := strings.Index(schema, `},"required"`); i >= 0 { + schema = schema[:i] + anySpaceProp + schema[i:] + } + } + if i := strings.LastIndex(desc, " [FLEET:"); i >= 0 { + return desc[:i] + " " + note + desc[i:], schema + } + return desc + " " + note, schema +} diff --git a/internal/agent/acceltools.go b/internal/agent/acceltools.go index 4ab20d67a..199eba2f0 100644 --- a/internal/agent/acceltools.go +++ b/internal/agent/acceltools.go @@ -157,7 +157,9 @@ func laneToolsFor(id string, call NPUFunc) []Tool { // which is also the order hwdetect.DetectAllAccelerators emits and the order // the MCP surface walks — so the two surfaces resolve a shared name identically. // A lane's Claims (accelerator_tool_owners, ADR 0068) take their names first, -// the same override mcpserver.accelOwnerPlan applies. +// the same override mcpserver.accelOwnerPlan applies. The owner is the first +// choice, not the only one: every other lane that serves the name answers when +// the owner cannot (AccelFallback, ADR 0102). func accelLaneTools(lanes []AccelLane, have []Tool) []Tool { taken := map[string]bool{} for _, t := range have { @@ -179,6 +181,7 @@ func accelLaneTools(lanes []AccelLane, have []Tool) []Tool { } } var out []Tool + ownerOf := map[string]string{} // tool name -> the lane that registered it here for _, lane := range lanes { if lane.Call == nil { continue @@ -188,6 +191,7 @@ func accelLaneTools(lanes []AccelLane, have []Tool) []Tool { continue } taken[t.Name] = true + ownerOf[t.Name] = lane.ID if lane.Remote { // The seat must know the image is read on THIS box and shipped: a // path on the far node would fail here, the reverse of the local lane. @@ -196,5 +200,76 @@ func accelLaneTools(lanes []AccelLane, have []Tool) []Tool { out = append(out, t) } } + // Every other lane that serves a registered name answers when the owner cannot (ADR 0102), + // in lane order: the owner is the first choice, never the only one. + steps := map[string][]AccelStep{} + for _, lane := range lanes { + if lane.Call == nil { + continue + } + for _, t := range laneToolsFor(lane.ID, lane.Call) { + owner, registered := ownerOf[t.Name] + if !registered || owner == lane.ID || slices.ContainsFunc(steps[t.Name], func(s AccelStep) bool { return s.Device == lane.ID }) { + continue + } + exec := t.Exec + steps[t.Name] = append(steps[t.Name], AccelStep{Device: lane.ID, Props: AccelSchemaProps(t.Schema), + Run: func(ctx context.Context, args map[string]any) map[string]any { + b, _ := json.Marshal(args) + return accelResult(exec(ctx, string(b))) + }}) + } + } + for i := range out { + chain := steps[out[i].Name] + if len(chain) == 0 { + continue + } + desc, schema := AccelFallbackSurface(out[i].Name, out[i].Description, string(out[i].Schema), true) + out[i].Description, out[i].Schema = desc, json.RawMessage(schema) + name, owner, exec := out[i].Name, ownerOf[out[i].Name], out[i].Exec + out[i].Exec = func(ctx context.Context, args string) (string, error) { + var in map[string]any + _ = json.Unmarshal([]byte(args), &in) + mine := args + if _, said := in[AnySpaceArg]; said { + // The caller's word to this loop, never a sidecar argument. + sub := map[string]any{} + for k, v := range in { + if k != AnySpaceArg { + sub[k] = v + } + } + b, _ := json.Marshal(sub) + mine = string(b) + } + res, err := exec(ctx, mine) + if err != nil { + return res, err + } + got := accelResult(res, nil) + if _, cannot := AccelCannotServe(got); !cannot { + return res, nil // the owner answered: its bytes, untouched + } + b, merr := json.Marshal(AccelFallback(ctx, name, owner, got, in, chain)) + if merr != nil { + return res, nil + } + return string(b), nil + } + } return out } + +// accelResult reads one lane tool's answer as a dict. An error or an answer that is not a JSON +// object is a defer, which is how every lane already reports a call it could not make. +func accelResult(res string, err error) map[string]any { + if err != nil { + return map[string]any{"deferred": true, "reason": err.Error()} + } + var m map[string]any + if json.Unmarshal([]byte(res), &m) != nil || m == nil { + return map[string]any{"deferred": true, "reason": "the device returned a non-object result"} + } + return m +} diff --git a/internal/fleetnode/accel_task_test.go b/internal/fleetnode/accel_task_test.go index e8b1940d0..2422ef71c 100644 --- a/internal/fleetnode/accel_task_test.go +++ b/internal/fleetnode/accel_task_test.go @@ -3,6 +3,7 @@ package fleetnode import ( "encoding/base64" "encoding/json" + "net/http" "os" "path/filepath" "strings" @@ -133,6 +134,24 @@ func TestBuildAccelRefusals(t *testing.T) { } } +// A text reservation of the node's GPUs never refuses an accel job (ADR 0102): the accelerator is not a +// card a lease names. Seen for real on 2026-11-11, when a measurement lease on a node's GPUs refused +// every forwarded NPU call for hours. The image job on the same server is the control: the refusal +// still stands for work that lands on a card. +func TestATextLeaseOfTheGPUsNeverRefusesAnAccelJob(t *testing.T) { + cfg := accelCfg(t) + cfg.ImageGenScript = imageCfg().ImageGenScript + s, _ := newTestServer(t, cfg, &fakeRunner{}, leasedOpts(textLease())) + rec := do(t, s, http.MethodPost, "/fleet/dispatch", + `{"job_id":"j-accel-leased","task_type":"accel","payload":{"accelerator":"coral-edgetpu","tool":"classify","args":{"top_k":1}}}`, nil) + if rec.Code != http.StatusAccepted { + t.Fatalf("an accel job under a text lease of the GPUs = %d, want 202: %s", rec.Code, rec.Body.String()) + } + if rec := dispatchImage(t, s, "j-image-leased"); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("control: an image job under the same lease = %d, want 503: %s", rec.Code, rec.Body.String()) + } +} + // accel drives a loopback sidecar, never the text endpoint: it must not hold a // fleet execution slot behind a five-minute digest contract. func TestAccelIsNotConcurrencyCapped(t *testing.T) { diff --git a/internal/fleetnode/leases.go b/internal/fleetnode/leases.go index dc3ddb41b..121874594 100644 --- a/internal/fleetnode/leases.go +++ b/internal/fleetnode/leases.go @@ -28,6 +28,7 @@ package fleetnode // - saturation and dispatch follow the same cards: the node says it is closed only when the // leases that refuse new work cover every card it has, and a text reservation refuses a // contract only when it holds every seat that contract could run on (textLeaseAgainst). +// An accel job runs on no card a lease can name and is never refused by one (ADR 0102). import ( "encoding/json" @@ -213,8 +214,13 @@ func (r leaseReading) closesNode(devices []gpuprobe.Device) bool { // agent payload this node cannot read, keeps the whole-node reading: the cards it will use // are chosen later, and unknown is every card. Every live text lease counts, not only the // lowest epoch. +// +// An accel job is never refused (ADR 0102). It runs on an accelerator that is not one of the +// cards a lease can name (an NPU or TPU with its own runtime, and on the Axera card its own +// memory), so a reservation of the GPUs says nothing about it: refused, the device sat idle +// for the whole measurement window and every forwarded tool deferred. func (s *Server) textLeaseAgainst(taskType string, payload json.RawMessage) (gpulease.Info, bool) { - if s.opts.Lease == nil { + if s.opts.Lease == nil || taskType == string(core.TaskAccel) { return gpulease.Info{}, false } info := s.opts.Lease() diff --git a/internal/mcpserver/accelfallback_test.go b/internal/mcpserver/accelfallback_test.go new file mode 100644 index 000000000..245872aae --- /dev/null +++ b/internal/mcpserver/accelfallback_test.go @@ -0,0 +1,102 @@ +package mcpserver + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/dmmdea/offload-harness/internal/config" + "github.com/dmmdea/offload-harness/internal/pipeline" +) + +// A shared tool whose owner cannot take the call runs on the next device that serves it (ADR 0102). +// Seen for real on 2026-11-11: four tools were handed to the Axera card on another node, and for the +// hours that node refused them they deferred while the Coral and the RK3588 NPU sat idle. Here the +// owner is the fleet Axera with no node to reach, and the next device is a Coral on this box. +// +// An embedding is the exception that must hold: another device's vector is in another space, so it +// moves only when the caller says any_space, and the caller's word never reaches a sidecar. +func TestASharedToolRunsOnTheNextDeviceWhenItsOwnerCannot(t *testing.T) { + var embedCalls, segmentCalls int + var detectArgs, embedArgs map[string]any + mux := http.NewServeMux() + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"enabled":true,"models_missing":[]}`)) }) + mux.HandleFunc("/v1/object_detect", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&detectArgs) + w.Write([]byte(`{"objects":[{"label":"cat","class_id":15,"x":1,"y":2,"w":3,"h":4,"score":0.9}],"count":1}`)) + }) + mux.HandleFunc("/v1/semantic_segment", func(w http.ResponseWriter, r *http.Request) { + segmentCalls++ + w.Write([]byte(`{"mask_path":"/tmp/x.segmask.png","classes":[],"width":1,"height":1}`)) + }) + mux.HandleFunc("/v1/embed", func(w http.ResponseWriter, r *http.Request) { + embedCalls++ + _ = json.NewDecoder(r.Body).Decode(&embedArgs) + w.Write([]byte(`{"embedding":[0.1,0.2],"dim":2,"space":"efficientnet-edgetpu-s"}`)) + }) + fake := httptest.NewServer(mux) + defer fake.Close() + + cfg := config.Default() + cfg.Home = t.TempDir() + cfg.Accelerators = []string{"coral-edgetpu"} + cfg.CoralEndpoint = fake.URL + cfg.FleetAccelerators = []string{"axera-ax650"} + cfg.AcceleratorToolOwners = map[string]string{"offload_object_detect": "axera-ax650", "offload_image_embed": "axera-ax650", "offload_semantic_segment": "axera-ax650"} + cs := doorSession(t, New(pipeline.New(cfg, nil, nil, nil))) + + call := func(name, args string) map[string]any { + t.Helper() + var out map[string]any + if err := json.Unmarshal([]byte(callTool(t, cs, name, args).Content[0].(*mcp.TextContent).Text), &out); err != nil { + t.Fatalf("%s: not a JSON object: %v", name, err) + } + return out + } + + got := call("offload_object_detect", `{"image_path":"/tmp/cat.jpg","score_threshold":0.4,"top_k":3}`) + fb, _ := got["fallback"].(map[string]any) + if got["count"] != float64(1) || fb["from"] != "axera-ax650" || fb["served_by"] != "coral-edgetpu" || fb["reason"] == "" { + t.Fatalf("object_detect did not run on the Coral with the owner named: %v", got) + } + if detectArgs["score_threshold"] != 0.4 || detectArgs["image_path"] != "/tmp/cat.jpg" { + t.Fatalf("the Coral did not get the arguments it takes: %v", detectArgs) + } + if _, sent := detectArgs["top_k"]; sent { + t.Fatalf("an argument the Coral's detector does not take travelled to it: %v", detectArgs) + } + + got = call("offload_image_embed", `{"image_path":"/tmp/cat.jpg"}`) + if got["deferred"] != true || got["fallback_withheld"] == nil || embedCalls != 0 { + t.Fatalf("an embedding moved to another device's space without the caller asking (%d calls): %v", embedCalls, got) + } + got = call("offload_image_embed", `{"image_path":"/tmp/cat.jpg","any_space":true}`) + fb, _ = got["fallback"].(map[string]any) + if got["space"] != "efficientnet-edgetpu-s" || fb["served_by"] != "coral-edgetpu" { + t.Fatalf("any_space:true did not take the Coral's vector: %v", got) + } + if _, sent := embedArgs["any_space"]; sent { + t.Fatalf("any_space reached the sidecar: %v", embedArgs) + } + + // The owner guards out_path; the next device's sidecar may not, and a model seat can pass any path + // (the security pass on this change). An out_path that is the input image, or an existing file that is + // not a PNG, never travels to another device. + src := filepath.Join(t.TempDir(), "street.jpg") + if err := os.WriteFile(src, []byte("jpeg bytes"), 0o644); err != nil { + t.Fatal(err) + } + args, _ := json.Marshal(map[string]any{"image_path": src, "out_path": src}) + got = call("offload_semantic_segment", string(args)) + if got["fallback_withheld"] == nil || segmentCalls != 0 { + t.Fatalf("an out_path that is the input image travelled to the next device (%d calls): %v", segmentCalls, got) + } + if b, _ := os.ReadFile(src); string(b) != "jpeg bytes" { + t.Fatalf("the input image was replaced: %q", b) + } +} diff --git a/internal/mcpserver/acceltools.go b/internal/mcpserver/acceltools.go index f0032a66c..774d85cf7 100644 --- a/internal/mcpserver/acceltools.go +++ b/internal/mcpserver/acceltools.go @@ -13,6 +13,7 @@ import ( "github.com/dmmdea/offload-harness/internal/accelclient" "github.com/dmmdea/offload-harness/internal/accelremote" + "github.com/dmmdea/offload-harness/internal/agent" "github.com/dmmdea/offload-harness/internal/config" ) @@ -188,19 +189,25 @@ func (s *Server) accelSidecar(id string) *accelclient.Sidecar { // way); the sidecar's own structured refusals pass through untouched — they // are results. func (s *Server) accelCall(ctx context.Context, id, tool string, args map[string]any) (*mcp.CallToolResult, error) { + return jsonResult(s.accelCallMap(ctx, id, tool, args)) +} + +// accelCallMap is accelCall's answer as the dict, for a caller that reads it before returning it +// (the fallback chain, ADR 0102). +func (s *Server) accelCallMap(ctx context.Context, id, tool string, args map[string]any) map[string]any { start := time.Now() sc := s.accelSidecar(id) if err := sc.Ensure(ctx); err != nil { s.p.RecordAccel(tool, id, sinceMs(start), true, id+": "+err.Error()) - return jsonResult(map[string]any{"deferred": true, "reason": id + ": " + err.Error()}) + return map[string]any{"deferred": true, "reason": id + ": " + err.Error()} } out, err := sc.Client().Call(ctx, tool, args) if err != nil { s.p.RecordAccel(tool, id, sinceMs(start), true, id+": "+err.Error()) - return jsonResult(map[string]any{"deferred": true, "reason": id + ": " + err.Error()}) + return map[string]any{"deferred": true, "reason": id + ": " + err.Error()} } s.p.RecordAccel(tool, id, sinceMs(start), false, "") - return jsonResult(out) + return out } // sinceMs is the wall since start in whole milliseconds, for a ledger row. @@ -286,17 +293,8 @@ func (s *Server) handleAccelTool(id, tool, requiredArg string) mcp.ToolHandler { // is returned in ignored, and the walk decides its name as if it were absent. A // fleet owner carries FleetOwnerSuffix. func accelOwnerPlan(cfg config.Config) (owner map[string]string, ignored []string) { - type dev struct{ id, owner string } - var devs []dev - for _, id := range cfg.Accelerators { - devs = append(devs, dev{id, id}) - } - for _, id := range cfg.FleetAccelerators { - if slices.Contains(cfg.Accelerators, id) { - continue // the device is here; its local lane owns its names - } - devs = append(devs, dev{id, id + FleetOwnerSuffix}) - } + type dev = accelDev + devs := accelDevices(cfg) serves := func(id, name string) bool { return slices.ContainsFunc(accelMCPTools(id), func(t accelMCPTool) bool { return t.name == name }) } @@ -335,6 +333,43 @@ func accelOwnerPlan(cfg config.Config) (owner map[string]string, ignored []strin return owner, ignored } +// accelDev is one device this box can reach: its id, and its owner tag (the id for a device on +// this box, the id + FleetOwnerSuffix for one reached over the fleet). +type accelDev struct{ id, owner string } + +// accelDevices lists the devices in the order every walk uses: the local ones in +// config.Accelerators order, then the fleet ones in config.FleetAccelerators order. +func accelDevices(cfg config.Config) []accelDev { + var devs []accelDev + for _, id := range cfg.Accelerators { + devs = append(devs, accelDev{id, id}) + } + for _, id := range cfg.FleetAccelerators { + if slices.Contains(cfg.Accelerators, id) { + continue // the device is here; its local lane owns its names + } + devs = append(devs, accelDev{id, id + FleetOwnerSuffix}) + } + return devs +} + +// accelFallbackPlan says, for every tool name in the owner plan, which other devices serve it, in +// the listed order (ADR 0102). The owner is the first choice; these answer when it cannot. A +// name only one device serves has no entry, and a device listed twice is counted once. +func accelFallbackPlan(cfg config.Config, owner map[string]string) map[string][]accelDev { + plan := map[string][]accelDev{} + for _, d := range accelDevices(cfg) { + for _, t := range accelMCPTools(d.id) { + o, ok := owner[t.name] + if !ok || o == d.owner || slices.Contains(plan[t.name], d) { + continue + } + plan[t.name] = append(plan[t.name], d) + } + } + return plan +} + // registerAccelTools registers every device's table under accelOwnerPlan: a // device registers the names the plan gives it and skips the rest, each skip // logged once at startup. Local devices call their loopback sidecar; a device @@ -346,6 +381,7 @@ func (s *Server) registerAccelTools(srv *mcp.Server, cfg config.Config) map[stri for _, why := range ignored { log.Printf("accelerator_tool_owners: %s; ignored, the first-listed rule decides that name (ADR 0068)", why) } + fallbacks := accelFallbackPlan(cfg, owner) // registered guards a device id listed twice: its second walk must not add the // same tools again. registered := map[string]bool{} @@ -359,7 +395,9 @@ func (s *Server) registerAccelTools(srv *mcp.Server, cfg config.Config) map[stri continue } registered[t.name] = true - srv.AddTool(&mcp.Tool{Name: t.name, Description: t.desc, InputSchema: json.RawMessage(t.schema)}, s.handleAccelTool(id, t.sidecar, t.arg)) + desc, schema := agent.AccelFallbackSurface(t.name, t.desc, t.schema, len(fallbacks[t.name]) > 0) + srv.AddTool(&mcp.Tool{Name: t.name, Description: desc, InputSchema: json.RawMessage(schema)}, + s.handleAccelChain(accelDev{id, id}, t, fallbacks[t.name])) } } for _, id := range cfg.FleetAccelerators { @@ -374,14 +412,69 @@ func (s *Server) registerAccelTools(srv *mcp.Server, cfg config.Config) map[stri log.Printf("fleet accelerator %s: %s is served by %s on this box; skipped (shared-name rule)", id, t.name, owner[t.name]) continue } - desc := t.desc + fleetAccelNote(id, t.arg) registered[t.name] = true - srv.AddTool(&mcp.Tool{Name: t.name, Description: desc, InputSchema: json.RawMessage(t.schema)}, s.handleFleetAccelTool(id, t.sidecar, t.arg)) + desc, schema := agent.AccelFallbackSurface(t.name, t.desc+fleetAccelNote(id, t.arg), t.schema, len(fallbacks[t.name]) > 0) + srv.AddTool(&mcp.Tool{Name: t.name, Description: desc, InputSchema: json.RawMessage(schema)}, + s.handleAccelChain(accelDev{id, id + FleetOwnerSuffix}, t, fallbacks[t.name])) } } return owner } +// accelDevCall runs one sidecar tool on one device: its loopback sidecar here, or the fleet node +// that carries it. +func (s *Server) accelDevCall(ctx context.Context, d accelDev, tool string, args map[string]any) map[string]any { + if d.owner == d.id { + return s.accelCallMap(ctx, d.id, tool, args) + } + return s.fleetAccelCallMap(ctx, d.id, tool, args) +} + +// handleAccelChain adapts one registered accelerator tool: the owner's handler (handleAccelTool +// or handleFleetAccelTool), and when the owner cannot take the call, the same tool on each +// fallback device through that device's own table row (agent.AccelFallback, ADR 0102). With no +// fallback device it is the owner's handler and nothing else. +func (s *Server) handleAccelChain(owner accelDev, row accelMCPTool, fallbacks []accelDev) mcp.ToolHandler { + if len(fallbacks) == 0 { + if owner.owner == owner.id { + return s.handleAccelTool(owner.id, row.sidecar, row.arg) + } + return s.handleFleetAccelTool(owner.id, row.sidecar, row.arg) + } + var steps []agent.AccelStep + for _, d := range fallbacks { + for _, t := range accelMCPTools(d.id) { + if t.name != row.name { + continue + } + steps = append(steps, agent.AccelStep{Device: d.id, Props: agent.AccelSchemaProps([]byte(t.schema)), + Run: func(ctx context.Context, args map[string]any) map[string]any { + if args[t.arg] == nil || args[t.arg] == "" { + return map[string]any{"deferred": true, "reason": d.id + ": empty " + t.arg} + } + return s.accelDevCall(ctx, d, t.sidecar, args) + }}) + } + } + return func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + var in map[string]any + if bad := parseArgs(req.Params.Arguments, &in); bad != nil { + return bad, nil + } + if in[row.arg] == nil || in[row.arg] == "" { + return jsonResult(map[string]any{"deferred": true, "reason": "empty " + row.arg}) + } + mine := map[string]any{} + for k, v := range in { + if k != agent.AnySpaceArg { // the caller's word to this server, never a sidecar argument + mine[k] = v + } + } + out := s.accelDevCall(ctx, owner, row.sidecar, mine) + return jsonResult(agent.AccelFallback(ctx, row.name, owner.id, out, in, steps)) + } +} + // fleetAccelNote is the note appended to a forwarded accelerator tool's description: the call goes to another node, and the // tool's own required argument says what travels with it. An image tool reads image_path on THIS box and its bytes go with the job // (the cap is accelremote's); a text tool has no file, so its text goes. The argument named is the table row's, never a fixed one. @@ -417,18 +510,23 @@ func (s *Server) handleFleetAccelTool(id, tool, requiredArg string) mcp.ToolHand // forwarding failure is a device-prefixed defer, the node's own result passes through untouched. // offload_ocr engine:"npu" reaches a fleet node's Axera card through it too. func (s *Server) fleetAccelCall(ctx context.Context, id, tool string, in map[string]any) (*mcp.CallToolResult, error) { + return jsonResult(s.fleetAccelCallMap(ctx, id, tool, in)) +} + +// fleetAccelCallMap is fleetAccelCall's answer as the dict (see accelCallMap). +func (s *Server) fleetAccelCallMap(ctx context.Context, id, tool string, in map[string]any) map[string]any { start := time.Now() out, err := accelremote.Call(ctx, s.p.Cfg(), id, tool, in) if err != nil { s.p.RecordAccel(tool, id+FleetOwnerSuffix, sinceMs(start), true, id+" (fleet): "+err.Error()) - return jsonResult(map[string]any{"deferred": true, "reason": id + " (fleet): " + err.Error()}) + return map[string]any{"deferred": true, "reason": id + " (fleet): " + err.Error()} } tier := id + FleetOwnerSuffix if n := accelPlacementNode(out); n != "" { tier = n + ":" + id } s.p.RecordAccel(tool, tier, sinceMs(start), false, "") - return jsonResult(out) + return out } // accelStatus is the status block's `accelerators` entry: one row per listed @@ -452,16 +550,32 @@ func accelStatus(ctx context.Context, cfg config.Config) map[string]any { slices.Sort(names) return names } + // fallback_for is what the device answers when the tool's owner cannot (ADR 0102): the names + // another device registered that this one serves too. A row gets the key only when it has any. + fallbacks := accelFallbackPlan(cfg, plan) + withFallbacks := func(owner string, entry map[string]any) map[string]any { + names := []string{} + for name, devs := range fallbacks { + if slices.ContainsFunc(devs, func(d accelDev) bool { return d.owner == owner }) { + names = append(names, name) + } + } + if len(names) > 0 { + slices.Sort(names) + entry["fallback_for"] = names + } + return entry + } for _, id := range cfg.FleetAccelerators { if slices.Contains(cfg.Accelerators, id) { continue } - out[id] = map[string]any{"owns": accelOwns(id), "serves": serves(id + FleetOwnerSuffix), "fleet": true, - "note": "not on this box: its tools forward to the first delegate_remotes node whose /fleet/health lists it (accelremote, cap 8 MiB per image)"} + out[id] = withFallbacks(id+FleetOwnerSuffix, map[string]any{"owns": accelOwns(id), "serves": serves(id + FleetOwnerSuffix), "fleet": true, + "note": "not on this box: its tools forward to the first delegate_remotes node whose /fleet/health lists it (accelremote, cap 8 MiB per image)"}) } for _, id := range cfg.Accelerators { lc, ok := accelLaneConfigFor(cfg, id) - entry := map[string]any{"owns": accelOwns(id), "serves": serves(id)} + entry := withFallbacks(id, map[string]any{"owns": accelOwns(id), "serves": serves(id)}) if !ok { entry["note"] = "listed in accelerators but this build has no adapter for it — nothing is registered" out[id] = entry diff --git a/internal/mcpserver/toolsprofile_gate_test.go b/internal/mcpserver/toolsprofile_gate_test.go index 4f1ce67f0..322819e4d 100644 --- a/internal/mcpserver/toolsprofile_gate_test.go +++ b/internal/mcpserver/toolsprofile_gate_test.go @@ -91,12 +91,12 @@ func isClassText(reason string) bool { return false } -// fleetSurfaceCfg is the same optional surfaces as fullSurfaceCfg with the three accelerators listed as FLEET accelerators, whose +// fleetSurfaceCfg is the same optional surfaces as fullSurfaceCfg with the four accelerators listed as FLEET accelerators, whose // descriptions carry the "[FLEET: ...]" suffix. func fleetSurfaceCfg(profile string) config.Config { c := browseCfg() c.AgentDelegationEnabled = true - c.FleetAccelerators = []string{"hailo-8l", "coral-edgetpu", "rknpu"} + c.FleetAccelerators = []string{"hailo-8l", "coral-edgetpu", "rknpu", "axera-ax650"} c.ToolsSchemaProfile = profile return c } diff --git a/internal/mcpserver/toolsprofile_keep.go b/internal/mcpserver/toolsprofile_keep.go index f4a08f1b2..4dd1c3c78 100644 --- a/internal/mcpserver/toolsprofile_keep.go +++ b/internal/mcpserver/toolsprofile_keep.go @@ -1,5 +1,7 @@ package mcpserver +import "github.com/dmmdea/offload-harness/internal/agent" + // toolsprofile_keep.go: the reviewed keep-list of the compact tools/list profile. // // Compact keeps the opening sentence of every description and, beyond it, only the sentences named here, in the tools named @@ -738,6 +740,19 @@ var compactKeep = []compactKeepEntry{ Sentence: "[FLEET: this box has no axera-ax650 — the call is forwarded to the fleet node that does; text travels with the job; the result carries placement{node,wall_ms}]", Why: "the forwarding note of a fleet accelerator, for the one tool that takes text and no image: the call is forwarded to another node and the text travels with it, which the opening sentence (on-box, free) does not say", }, + // The fallback sentence of a shared accelerator tool (agent.AccelFallbackSurface, ADR 0102): on a box where another device serves + // the same name, the tool's answer can come from that device's model. One sentence for every box (it names no device), and one + // for the two embedding tools, whose answer moves only when the caller asks. + { + Tools: []string{"offload_classify_image", "offload_object_detect", "offload_semantic_segment", "offload_segment", "offload_zero_shot", "offload_depth", "offload_pose"}, + Sentence: agent.AccelFallbackNote, + Why: "a shared accelerator tool falls back: when its device cannot take the call the next device answers with its own model and fewer arguments, which the opening sentence (one named device and model) does not say", + }, + { + Tools: []string{"offload_image_embed", "offload_text_embed"}, + Sentence: agent.AccelFallbackSpaceNote, + Why: "an embedding tool falls back only with any_space:true and then answers in a different embedding space: the one rule that keeps a caller from comparing vectors of two devices", + }, // The Axera card's own descriptions (acceltools.go): what its depth and embedding tools write and return, and the pairing rule of its // embedding space with its own text tower. Kept verbatim: they are what the caller reads the result shape and the space from. { diff --git a/internal/mcpserver/toolsprofile_test.go b/internal/mcpserver/toolsprofile_test.go index 9a88b5459..1c037d1f2 100644 --- a/internal/mcpserver/toolsprofile_test.go +++ b/internal/mcpserver/toolsprofile_test.go @@ -19,7 +19,7 @@ import ( func fullSurfaceCfg(profile string) config.Config { c := browseCfg() c.AgentDelegationEnabled = true - c.Accelerators = []string{"hailo-8l", "coral-edgetpu", "rknpu"} + c.Accelerators = []string{"hailo-8l", "coral-edgetpu", "rknpu", "axera-ax650"} c.ToolsSchemaProfile = profile return c }