fix(config): resolve the Caldera REST connection from core config

.env.example

@@ -9,7 +9,9 @@ MCP_LLM_API_BASE=
 # every request supplies its own via the UI's Global Model Configuration.
 MCP_LLM_API_KEY=
 
-# Caldera REST API
-# Defaults work for local development. Override for non-local deployments.
-CALDERA_URL=http://localhost:8888
-CORE_CALDERA_API_KEY=ADMIN123
+# Caldera REST API. Both auto-resolve from caldera's main config; leave unset
+# unless MCP cannot reach the address caldera binds to (CALDERA_URL), or
+# caldera rejects its stock key (CORE_CALDERA_API_KEY: use the API_TOKEN
+# logged once when conf/local.yml was generated).
+# CALDERA_URL=
+# CORE_CALDERA_API_KEY=

PLUGIN_MCP.md

@@ -116,15 +116,21 @@ Forwarded by the parent (see
 
 | Env var | Meaning |
 |---|---|
-| `CALDERA_URL` | Base URL of the Caldera REST API (`http://localhost:8888/api/v2/` for local dev) |
-| `CORE_CALDERA_API_KEY` | Admin key for Caldera REST (`ADMIN123` default) |
+| `CALDERA_URL` | Base URL of the Caldera REST API, always suffixed `/api/v2/`. Resolved from the host and port Caldera binds to |
+| `CORE_CALDERA_API_KEY` | Key for Caldera REST, resolved to whichever value Caldera's own `api_key_red` / `api_key_blue` accepts |
 | `DSPY_MODEL`, `DSPY_API_KEY`, `DSPY_API_BASE`, `DSPY_TEMPERATURE`, `DSPY_MAX_TOKENS` | LLM credentials, only if your tools call back into DSPy (e.g. for command synthesis) |
 | `PYTHONPATH` | Includes the venv's `site-packages` |
 
 **Do not** call `dotenv.load_dotenv()` inside the subprocess. The parent
 loads `.env` once at plugin enable; the subprocess inherits the
 environment. Re-loading masks parent overrides.
 
+Both Caldera variables are resolved by the parent from Caldera's own
+config, so read them and do not substitute defaults of your own. In
+particular, never fall back to `ADMIN123`: Caldera hashes its API keys at
+startup, and any server whose `conf/local.yml` was generated has a key
+that literal will never match.
+
 If your server needs DSPy itself (rare — only when your tools generate
 content with the LLM), import the lazy bootstrap from the MCP plugin's
 namespace:
@@ -145,9 +151,12 @@ import requests
 
 class CalderaClient:
     def __init__(self):
-        self.url = os.environ.get("CALDERA_URL", "http://localhost:8888/api/v2/")
+        # Both are resolved by the parent from Caldera's own config. A
+        # hardcoded default here would silently 401 on any server whose
+        # conf/local.yml was generated, so fail loudly instead.
+        self.url = os.environ["CALDERA_URL"]
         self.headers = {
-            "KEY": os.environ.get("CORE_CALDERA_API_KEY", "ADMIN123"),
+            "KEY": os.environ["CORE_CALDERA_API_KEY"],
             "Content-Type": "application/json",
         }
     def get(self, endpoint):
@@ -533,9 +542,13 @@ SERVER  = Path(__file__).resolve().parents[1] / "mcp_server.py"
 
 @pytest.mark.asyncio
 async def test_my_server_spawns_and_lists_tools():
+    # Resolve the same way the parent does rather than hardcoding, so this
+    # test spawns against whatever Caldera the checkout is configured for.
+    from plugins.mcp.app.config import caldera_connection
+    caldera = caldera_connection()
     env = os.environ.copy()
-    env.setdefault("CALDERA_URL", "http://localhost:8888/api/v2/")
-    env.setdefault("CORE_CALDERA_API_KEY", "ADMIN123")
+    env.setdefault("CALDERA_URL", caldera["url"])
+    env.setdefault("CORE_CALDERA_API_KEY", caldera["api_key"])
     params = StdioServerParameters(command=VENV_PY, args=[str(SERVER)], env=env)
     async with stdio_client(params) as (r, w):
         async with ClientSession(r, w) as session:

README.md

@@ -123,6 +123,8 @@ cp plugins/mcp/.env.example plugins/mcp/.env
 
 Both may instead be supplied per-session through the UI's Global Model Configuration. Avoid committing secrets.
 
+Nothing needs setting for the Caldera connection. The plugin builds the REST URL from the host and port CALDERA binds to, and picks the API key CALDERA's own `api_key_red` or `api_key_blue` accepts. Set `CALDERA_URL` only when MCP runs somewhere that cannot reach the address CALDERA binds to, such as a separate container or host; running behind the `ssl` plugin is not such a case, since haproxy forwards to the same local address. Set `CORE_CALDERA_API_KEY` on any server whose `conf/local.yml` was generated. A rejected key is reported on the plugin's splash page and in the CALDERA log at startup.
+
 ## Quick Start
 
 1. Start CALDERA from the repository root:
@@ -309,6 +311,7 @@ Use `--build` after changing Vue, CSS, or other Magma-loaded UI assets.
 - **MCP page does not appear**: Confirm `mcp` is listed in `conf/local.yml` and restart CALDERA.
 - **UI changes are missing**: Restart with `--build` so Magma rebuilds plugin UI bundles.
 - **Model calls fail**: Check the endpoint profile, API base, API key, and model name. The API base often needs a `/v1` suffix for OpenAI-compatible servers.
+- **Caldera tools return 401 or the splash reports a rejected key**: CALDERA hashes its API keys at startup, so a generated `conf/local.yml` has a key the plugin cannot guess. Set `CORE_CALDERA_API_KEY` in `plugins/mcp/.env` to the `API_TOKEN` printed once in the CALDERA log when that file was created.
 - **CTI/RAG does not run**: Select at least one generated STIX bundle in Plan and Execute. CTI selection automatically enables RAG for that workflow.
 - **Deploy spec has review gaps**: The CTI did not provide enough grounded evidence. Add richer CTI, select more STIX bundles, or review the gaps before deployment.
 

app/config.py

@@ -29,6 +29,9 @@
      so a deployment can pin one on disk.
   3. UI overrides (non-empty fields only, and only fields not declared
      fields_locked: true in yaml)
+
+The Caldera REST connection auto-resolves from caldera's own main config, so
+a stock deployment configures nothing. See caldera_connection().
 """
 import os
 
@@ -159,46 +162,110 @@ def _read_caldera_main_config_from_disk() -> dict:
     return {}
 
 
-def _caldera_url_from_server_config() -> str:
+def _caldera_main_config() -> dict:
+    """Caldera's `main` config, from the running server or parsed from disk.
+
+    get_config() raises KeyError in a subprocess, which never ran server.py.
+    """
     try:
         main = BaseWorld.get_config() or {}
     except Exception:
         main = {}
+    return main or _read_caldera_main_config_from_disk()
+
+
+def _dialable_host(host) -> str:
+    """Turn caldera's bind address into one we can connect to.
+
+    A wildcard bind means "every interface" and is not routable.
+    """
+    host = str(host or '').strip()
+    if host in ('', '0.0.0.0', '*'):
+        return '127.0.0.1'
+    if host in ('::', '[::]'):
+        return '[::1]'
+    if ':' in host and not host.startswith('['):
+        return f'[{host}]'  # bare IPv6 literal; unbracketed it cannot carry a port
+    return host
+
+
+def _caldera_url_from_server_config() -> str:
+    """Build the REST base url from the host and port caldera binds to.
+
+    Not `app.contact.http`: that is the callback url for sandcat agents,
+    usually a bridge-side IP, and it is mutable at runtime. host and port are
+    what server.py hands to web.TCPSite, and caldera refuses to change either.
+    """
+    main = _caldera_main_config()
+    return _normalise_api_url(
+        f"http://{_dialable_host(main.get('host'))}:{main.get('port') or 8888}"
+    )
 
-    # If BaseWorld is empty (we're running as a subprocess outside the
-    # caldera server process), fall back to parsing the main config
-    # files from disk so the URL still picks up the operator's port
-    # override (e.g. -E local with port: 8788) instead of defaulting
-    # to 8888.
-    if not main:
-        main = _read_caldera_main_config_from_disk()
-
-    # For MCP's local REST calls, always target localhost on the port
-    # from caldera's main config. We deliberately DON'T use
-    # `app.contact.http` here even though it carries a fully-formed URL:
-    # that field is the URL the operator wants *sandcat agents* (running
-    # inside the bridge network) to call home on — typically the
-    # bridge-side IP (e.g. http://10.10.0.1:8788). MCP and caldera run on
-    # the same host, so localhost:<port> is always reachable, never
-    # blocked by an iptables forward rule, and removes one source of
-    # config drift between sandcat callbacks and MCP API calls.
-    port = main.get('port') or 8888
-    return _normalise_api_url(f'http://localhost:{port}')
+
+_STOCK_API_KEY = 'ADMIN123'
+
+# argon2 verify costs ~33ms and the config never changes mid-process.
+_key_verdicts: dict = {}
+
+
+def _verify_key_against_core(candidate: str):
+    """Would the running caldera accept `candidate`? None if we cannot tell.
+
+    Caldera stores api_key_red/blue argon2-hashed and verifies the submitted
+    plaintext against the hash, so the stored value is never itself a usable
+    key. Ask its own hasher, in-process. Caldera <= 5.3.0 stored plaintext.
+    """
+    if not candidate:
+        return None
+    if candidate in _key_verdicts:
+        return _key_verdicts[candidate]
+
+    main = _caldera_main_config()
+    stored = [s for s in (main.get('api_key_red'), main.get('api_key_blue'))
+              if isinstance(s, str) and s]
+    if not stored:
+        return None
+
+    hashed = [s for s in stored if s.startswith('$argon2id$')]
+    if hashed:
+        try:
+            from app.utility.config_util import verify_hash
+        except Exception:
+            return None
+        verdict = any(verify_hash(s, candidate) for s in hashed)
+    else:
+        verdict = candidate in stored
+
+    _key_verdicts[candidate] = verdict
+    return verdict
 
 
 def caldera_connection() -> dict:
-    """Resolve the Caldera REST connection: {url, api_key}.
+    """Resolve the Caldera REST connection from caldera's own config.
+
+    Returns {url, api_key, key_valid, url_env, api_key_env}.
 
-    Both come from env vars whose names are declared in yaml's caldera
-    block. Falls back to local-dev values when the env vars are unset
-    so a fresh checkout works without configuration.
+    key_valid is diagnostic only: True/False/None for unchecked. The key is
+    returned either way and hook.py reports a mismatch, rather than swapping
+    in one the operator never configured. Both env vars keep top precedence.
     """
     cfg = dict(_load_defaults().get('caldera') or {})
     url_var = cfg.get('url_env', 'CALDERA_URL')
     key_var = cfg.get('api_key_env', 'CORE_CALDERA_API_KEY')
+
+    # Normalise the override too: raw, a missing /api/v2/ suffix hits the
+    # login page and returns HTML with a 200.
+    url_override = os.environ.get(url_var)
+    url = _normalise_api_url(url_override) or _caldera_url_from_server_config()
+
+    api_key = os.environ.get(key_var) or _STOCK_API_KEY
+    # An override may point at another caldera, whose keys are not ours to check.
+    key_valid = None if url_override else _verify_key_against_core(api_key)
+
     return {
-        'url': os.environ.get(url_var) or _caldera_url_from_server_config(),
-        'api_key': os.environ.get(key_var, 'ADMIN123'),
+        'url': url,
+        'api_key': api_key,
+        'key_valid': key_valid,
         'url_env': url_var,
         'api_key_env': key_var,
     }

app/mcp_gui.py

@@ -74,6 +74,8 @@ def _bootstrap_context(self) -> dict:
             "llm_api_base": llm.get("api_base") or "",
             "caldera_url": caldera.get("url") or "",
             "caldera_api_key_env": caldera.get("api_key_env") or "CORE_CALDERA_API_KEY",
+            # Unknown (config unreadable) is not a failure worth flagging.
+            "caldera_key_rejected": caldera.get("key_valid") is False,
             "server_names": sorted(servers.keys()),
             "workflow_names": sorted(workflows.keys()),
             "capability_names": sorted(capabilities.keys()),

app/mcp_server.py

@@ -82,8 +82,12 @@ def health_check():
     """
     Returns the health of the Caldera API.
     """
-    if isinstance(caldera_request.make_get_request("health"), dict):
-        return "Caldera API is UP!"
+    # The error object make_get_request returns on a non-200 is itself a dict,
+    # so an isinstance check reported "UP" on a 401.
+    result = caldera_request.make_get_request("health")
+    if isinstance(result, dict) and "error" in result:
+        return f"Caldera API at {caldera_request.caldera_url} is not usable: {result['error']}"
+    return "Caldera API is UP!"
 
 
 def filter_abilities(req, tactic: str, atomic: bool):

app/workflows/author.py

@@ -45,9 +45,12 @@ def get_env(lm_settings=None):
         if lm_settings.get('ssl_verify') is not None:
             env[ENV_SSL_VERIFY] = str(lm_settings.get('ssl_verify')).lower()
 
+    # Push the resolved values, not the raw environment: caldera_connection()
+    # already honours the env vars, and re-reading them here would hand the
+    # child a url missing the /api/v2/ suffix.
     caldera = caldera_connection()
-    env['CALDERA_URL'] = os.environ.get('CALDERA_URL') or caldera['url']
-    env['CORE_CALDERA_API_KEY'] = os.environ.get('CORE_CALDERA_API_KEY') or caldera['api_key']
+    env['CALDERA_URL'] = caldera['url']
+    env['CORE_CALDERA_API_KEY'] = caldera['api_key']
 
     return env
 

app/workflows/plan_execute.py

@@ -81,9 +81,12 @@ def get_env(lm_settings=None):
         if lm_settings.get('ssl_verify') is not None:
             env[ENV_SSL_VERIFY] = str(lm_settings.get('ssl_verify')).lower()
 
+    # Push the resolved values, not the raw environment: caldera_connection()
+    # already honours the env vars, and re-reading them here would hand the
+    # child a url missing the /api/v2/ suffix.
     caldera = caldera_connection()
-    env['CALDERA_URL'] = os.environ.get('CALDERA_URL') or caldera['url']
-    env['CORE_CALDERA_API_KEY'] = os.environ.get('CORE_CALDERA_API_KEY') or caldera['api_key']
+    env['CALDERA_URL'] = caldera['url']
+    env['CORE_CALDERA_API_KEY'] = caldera['api_key']
 
     return env
 

conf/default.yml

@@ -44,9 +44,10 @@ cti:
   offline: true
   use_mock: false
 
-# Caldera REST API used by the caldera_core MCP subprocess to call /api/v2/*
-# on the running Caldera server. Credentials live in plugins/mcp/.env;
-# this block only names the env vars to consult.
+# Caldera REST API used by the caldera_core MCP subprocess. The url and key
+# auto-resolve from caldera's own main config, so a stock deployment needs no
+# setup. This block only names the env vars that override that; their values
+# live in plugins/mcp/.env.
 caldera:
   url_env: CALDERA_URL
   api_key_env: CORE_CALDERA_API_KEY

hook.py

@@ -110,15 +110,41 @@ def is_port_open(port, host='127.0.0.1'):
     from plugins.mcp.app.discovery.servers import discover_mcp_servers
     from plugins.mcp.app.discovery.workflows import discover_workflows
     from plugins.mcp.app.discovery.capabilities import discover_capabilities
+    from plugins.mcp.app.config import caldera_connection
     logging.getLogger("litellm_logging").setLevel(logging.ERROR)
 
 except ImportError as e:
     log.error(f"[MCP] Error importing MCP plugin modules: {e}")
     traceback.print_exc()
 
+
+def report_caldera_connection():
+    """Log the resolved Caldera REST connection, and say so when it won't work.
+
+    A rejected key is otherwise near-undiagnosable: caldera 401s, the tool
+    wraps that in a successful result, and the operator sees only a confused
+    agent trajectory.
+    """
+    caldera = caldera_connection()
+    if caldera['key_valid'] is False:
+        log.warning(
+            f"[MCP] Caldera at {caldera['url']} rejects the configured API key. "
+            f"Set {caldera['api_key_env']} in plugins/mcp/.env to this server's "
+            f"API_TOKEN, printed once in the caldera log when conf/local.yml "
+            f"was generated."
+        )
+    else:
+        log.info(f"[MCP] Caldera REST resolved to {caldera['url']}")
+
+
 async def enable(services):
     app = services.get('app_svc').application
 
+    try:
+        report_caldera_connection()
+    except Exception as e:
+        log.warning(f"[MCP] Could not resolve the Caldera REST connection: {e}")
+
     # Discover MCP servers, workflows, and capabilities at boot. Each registry is
     # built once and handed to MCPService; nothing rescans at request time.
     import pathlib

templates/mcp.html

@@ -26,8 +26,17 @@ <h3 class="title is-6 mt-4">Plugin readiness</h3>
                 </li>
                 <li class="mt-2">
                     <strong>Caldera REST:</strong>
-                    <code>{{ caldera_url }}</code>
-                    &middot; key from <code>{{ caldera_api_key_env }}</code>
+                    {% if caldera_key_rejected %}
+                        <span class="tag is-warning is-light">key rejected</span>
+                        &nbsp;<code>{{ caldera_url }}</code> does not accept the
+                        configured key. Set <code>{{ caldera_api_key_env }}</code>
+                        in <code>plugins/mcp/.env</code> to this server's API_TOKEN,
+                        printed once in the caldera log when
+                        <code>conf/local.yml</code> was generated.
+                    {% else %}
+                        <span class="tag is-success is-light">resolved</span>
+                        &nbsp;<code>{{ caldera_url }}</code>
+                    {% endif %}
                 </li>
                 <li class="mt-2">
                     <strong>MCP servers discovered:</strong> {{ server_count }}