diff --git a/.env.example b/.env.example index afd03d4..4295239 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,32 @@ FORGE_EGRESS_ALLOW_PRIVATE_HOSTS=[] # e.g. ["localhost","127.0.0.1"] (Dock # resolves to a different host per deploy (dev/qa/prod). A template referencing a key NOT in this # map fails the call loudly. Blank/unset = {}. FORGE_TOOL_VARS= # e.g. {"api_base":"https://api.example.com"} +# --- Connectors: the vendor OAuth apps this deployment signs people in with --- +# This is the ONLY place a bundled connector's credentials live. Register each vendor app once, +# put it here, restart - and from then on every user just clicks Connect, signs in with their own +# account on the vendor's page, and approves. Nobody types a client secret into Forge's UI, and +# a connector whose group is missing here says so instead of showing a form. +# +# Keyed by CREDENTIAL GROUP, which is the vendor, not the connector: one "google" entry covers +# Gmail, Calendar, Drive and Sheets; one "microsoft" entry covers Outlook. +# google Gmail · Calendar · Drive · Sheets microsoft Outlook +# github GitHub hubspot HubSpot +# airtable Airtable +# Slack, Notion, Linear and Atlassian need NO entry: they publish OAuth metadata and Forge +# registers a client with them automatically (RFC 7591) the first time someone connects. +# +# Redirect/callback URL to register with every vendor (must match exactly): +# /v1/oauth/callback dev: http://localhost:8000/v1/oauth/callback +# +# Google: create a "Web application" OAuth client and paste the callback above into its +# Authorized redirect URIs. It must be a Web application client, not a Desktop one: Desktop +# clients have no redirect-URI field at all and accept only loopback addresses, so Forge's +# callback can never be registered against one and every sign-in fails with +# redirect_uri_mismatch. Note the "Authorized JavaScript origins" field strips the path - the +# full URL belongs in "Authorized redirect URIs". Until the app passes Google verification it +# is capped at 100 users and shows an "unverified app" warning; Gmail read/modify are +# RESTRICTED scopes needing a CASA audit. +FORGE_CONNECTOR_OAUTH_APPS= # {"google":{"client_id":"…","client_secret":"…"},"github":{…}} # Deployment-wide fallback for a per-user auth provider's `token_ctx_key`: the run-context key an # integration forwards its per-user token under (via X-Forge-Context) when a provider doesn't set # its own. Empty = off. e.g. user_token diff --git a/apps/api/forge/auth_providers/oauth_flow.py b/apps/api/forge/auth_providers/oauth_flow.py new file mode 100644 index 0000000..68014cf --- /dev/null +++ b/apps/api/forge/auth_providers/oauth_flow.py @@ -0,0 +1,115 @@ +"""Shared authorization-code connect flow (PKCE + signed state). + +Both the Auth Providers screen and the Connectors screen start the same 3-legged OAuth dance +against the same `/v1/oauth/callback`. Keeping the URL construction here means the security +properties - PKCE S256, a signed short-lived state, and carrying only the provider's declared +per-user dims - are implemented once instead of drifting between two call sites. +""" + +from __future__ import annotations + +import base64 +import hashlib +import secrets as _secrets +from urllib.parse import urlencode + +from forge.config import settings +from forge.models import AuthProvider +from forge.secrets.store import SecretStore +from forge.security import create_state_token + + +class OAuthNotConfigured(ValueError): + """The provider is missing something the authorize step needs (client id, authorize URL).""" + + +def redirect_uri(cfg: dict) -> str: + return cfg.get("redirect_uri") or f"{settings.public_base_url.rstrip('/')}/v1/oauth/callback" + + +def token_request(cfg: dict, data: dict) -> tuple[dict, dict]: + """Form body + headers for a token-endpoint POST, given a body that already carries + `client_id`/`client_secret`. Returns `(data, headers)`. + + Two things a naive POST gets wrong against real servers: + + * `Accept: application/json`. GitHub's token endpoint answers `application/x-www-form- + urlencoded` without it, so a perfectly successful exchange then dies in `.json()`. + * client_secret_basic. RFC 6749 §2.3.1 makes HTTP Basic the method a server MUST support + and form-post the one it MAY; a few (Airtable) accept only Basic. `token_auth: "basic"` + on the provider config moves the secret into the Authorization header. `client_id` stays + in the body, which every server tolerates and some still require. + """ + headers = {"Accept": "application/json"} + if cfg.get("token_auth") != "basic": + return {k: v for k, v in data.items() if v is not None}, headers + client_id, client_secret = data.get("client_id") or "", data.get("client_secret") or "" + creds = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + headers["Authorization"] = "Basic " + creds + return {k: v for k, v in data.items() if v is not None and k != "client_secret"}, headers + + +async def build_authorize_url( + ap: AuthProvider, *, tenant_id: str, project_id: str, context: dict | None = None, + secrets: SecretStore | None = None, +) -> str: + """The provider's authorize URL, carrying a signed state that binds the callback back to + this tenant/project/provider (and, for a per-user provider, to the end user being connected). + + The PKCE verifier rides inside the SIGNED state. That is safe for a CONFIDENTIAL client - + which is what Forge always registers, since it holds the client secret server-side in its + own encrypted store - because the token exchange also requires that secret. A public client + (no secret) would need the verifier held server-side instead; see routers/oauth.py. + """ + cfg = ap.config or {} + store = secrets or SecretStore() + client_id = None + if cfg.get("client_id_ref"): + try: + client_id = await store.read_ref(tenant_id=tenant_id, project_id=project_id, ref=cfg["client_id_ref"]) + except Exception as e: # noqa: BLE001 - surface as a configuration problem, not a 500 + raise OAuthNotConfigured("client_id secret is not set") from e + if not client_id: + raise OAuthNotConfigured("client_id secret is not set") + if not cfg.get("authorize_url"): + raise OAuthNotConfigured("authorize_url is not configured") + + verifier = _secrets.token_urlsafe(64) + challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + claims = {"tid": tenant_id, "pid": project_id, "ap": ap.id, "cv": verifier} + per_user = cfg.get("per_user_context_keys") or [] + ctx = context or {} + user_ctx = {k: ctx[k] for k in per_user if k in ctx} + if user_ctx: + claims["ctx"] = user_ctx + + q = { + "response_type": "code", + "client_id": str(client_id), + "redirect_uri": redirect_uri(cfg), + "state": create_state_token(claims), + "code_challenge": challenge, + "code_challenge_method": "S256", + } + if cfg.get("scope"): + q["scope"] = cfg["scope"] + # RFC 8707: bind the issued token to the resource it is for, so a token minted for one MCP + # server can't be replayed against another. Only sent when the provider names a resource + # (connectors set it to the MCP server URL); omitted otherwise to avoid upsetting servers + # that reject unknown parameters. + if cfg.get("resource"): + q["resource"] = cfg["resource"] + # Vendor-specific authorize parameters - Google's `access_type`/`prompt` for a refresh token, + # Slack's `user_scope`, HubSpot's optional scopes. `authorize_params` is the ONE mechanism; + # there is deliberately no top-level special case for individual keys. There used to be one + # for `access_type`/`prompt`, and because nothing ever wrote them at the top level it was + # dead on the connector path - which is precisely how four Google manifests came to ship + # without asking for offline access while a block of code appeared to be handling it. + # + # Applied last but never over the protocol parameters above: a manifest must not be able to + # redirect the callback or weaken PKCE by declaring `redirect_uri` or `code_challenge_method` + # as an "extra". + for key, value in (cfg.get("authorize_params") or {}).items(): + if key not in q: + q[key] = str(value) + return f"{cfg['authorize_url']}?{urlencode(q)}" diff --git a/apps/api/forge/auth_providers/resolver.py b/apps/api/forge/auth_providers/resolver.py index 6f94e34..b8efd02 100644 --- a/apps/api/forge/auth_providers/resolver.py +++ b/apps/api/forge/auth_providers/resolver.py @@ -274,9 +274,14 @@ async def _refresh_oauth(self, provider, cfg: dict, read, bundle: dict, tenant_i "client_secret": await read(cfg.get("client_secret_ref")), } client = client or shared_async_client() + # Same client-auth method and Accept header the initial exchange used - a provider that + # only speaks client_secret_basic rejects the refresh just as readily as the exchange. + from forge.auth_providers.oauth_flow import token_request + + form, headers = token_request(cfg, data) r = await guarded_request( client, "POST", cfg["token_url"], - data={k: v for k, v in data.items() if v is not None}, timeout=30, follow_redirects=True, + data=form, headers=headers, timeout=30, follow_redirects=True, ) r.raise_for_status() body = r.json() diff --git a/apps/api/forge/auth_providers/templates.py b/apps/api/forge/auth_providers/templates.py index d468cf2..66d1bbb 100644 --- a/apps/api/forge/auth_providers/templates.py +++ b/apps/api/forge/auth_providers/templates.py @@ -11,6 +11,7 @@ from __future__ import annotations +import json import re from collections.abc import Collection from typing import Any @@ -42,55 +43,125 @@ def _lookup(path: str, vars: dict, strict_ns: Collection[str] = ()) -> Any: return cur -def _sub_one(mm: re.Match, vars: dict, strict_ns: Collection[str] = ()) -> str: +def _sub_one(mm: re.Match, vars: dict, strict_ns: Collection[str] = (), + escape_json: bool = False) -> str: # Embedded token (not a whole-string match): stringify the resolved value. Only a missing # value (None) becomes empty - a falsy-but-real value like 0 or False must render as "0"/ # "False", not "" (an `x or ""` here would silently drop legitimate zeros/booleans). v = _lookup(mm.group(1), vars, strict_ns) - return "" if v is None else str(v) - - -def render_template(s: str, vars: dict, *, strict_ns: Collection[str] = ()) -> Any: + if v is None: + return "" + # Inside a JSON body template the token sits between quotes, so the CONTENT has to be + # JSON-escaped: a newline, a double quote or a backslash in model-written text (a note, an + # email body, a search string) otherwise terminates the string early and the whole body stops + # being JSON. dumps()[1:-1] escapes the content without adding a second pair of quotes. + if escape_json and isinstance(v, str): + return json.dumps(v, ensure_ascii=False)[1:-1] + # A list/dict interpolated into a body template is being placed into JSON - `str()` there + # yields Python's repr (single quotes, True/None), which is not JSON, so the body fails to + # parse and gets sent as raw text. `[[1, 2]]` happens to be valid JSON and `[['a', 'b']]` is + # not, which is why this only ever showed up on string data. Scalars keep str(): a bare + # `{{input.flag}}` in a query string renders "False", which is what that context wants. + if isinstance(v, (list, dict)): + return json.dumps(v, ensure_ascii=False) + return str(v) + + +def render_template(s: str, vars: dict, *, strict_ns: Collection[str] = (), + escape_json: bool = False) -> Any: # `strict_ns` names namespaces whose missing keys raise MissingTemplateVar instead of # rendering empty (used for {{env.*}}). Defaults to lenient for all namespaces. - # Whole-string single token -> preserve native type (numbers, objects). + # `escape_json` JSON-escapes substituted strings, for a template that is a JSON document + # (the REST body path opts in). Whole-string single token -> preserve native type (numbers, + # objects), which needs no escaping because the value is never re-parsed from text. m = _TOKEN.fullmatch(s.strip()) if m: return _lookup(m.group(1), vars, strict_ns) - return _TOKEN.sub(lambda mm: _sub_one(mm, vars, strict_ns), s) + return _TOKEN.sub(lambda mm: _sub_one(mm, vars, strict_ns, escape_json), s) + +#: Directives that need STRUCTURAL rendering (parse the JSON, then walk it) rather than plain +#: string substitution, because they produce a value the surrounding text can't express. +DIRECTIVES = ("$each", "$mime") -def has_each_directive(obj: Any) -> bool: - """True if `obj` (a parsed JSON structure) contains a `$each` loop directive anywhere - i.e. - a dict that has "$each" as a KEY. Used to decide whether a body template needs structural + +def has_structural_directive(obj: Any) -> bool: + """True if `obj` (a parsed JSON structure) contains a `$each` or `$mime` directive anywhere - + i.e. a dict that has one as a KEY. Used to decide whether a body template needs structural rendering; a literal "$each" appearing inside a string value is NOT a directive and must not trigger it (that would silently change type coercion for unrelated templates).""" if isinstance(obj, dict): - if "$each" in obj: + if any(d in obj for d in DIRECTIVES): return True - return any(has_each_directive(v) for v in obj.values()) + return any(has_structural_directive(v) for v in obj.values()) if isinstance(obj, list): - return any(has_each_directive(v) for v in obj) + return any(has_structural_directive(v) for v in obj) return False def render_value(obj: Any, vars: dict, *, allow_each: bool = False, strict_ns: Collection[str] = ()) -> Any: - """Walk a parsed JSON structure, rendering `{{token}}` leaves. `$each` loop directives are - honored ONLY when `allow_each=True` (the REST body-template path opts in); every other caller - - auth token_fetch/extract rules, data-node payloads - passes the default False, so a literal - object key named "$each" stays an ordinary key instead of being reinterpreted as a loop. + """Walk a parsed JSON structure, rendering `{{token}}` leaves. Structural directives (`$each`, + `$mime`) are honored ONLY when `allow_each=True` (the REST body-template path opts in); every + other caller - auth token_fetch/extract rules, data-node payloads - passes the default False, + so a literal object key named "$each" stays an ordinary key instead of being reinterpreted. `strict_ns` propagates the fail-loud namespaces (e.g. env) to every leaf.""" if isinstance(obj, str): return render_template(obj, vars, strict_ns=strict_ns) if isinstance(obj, dict): if allow_each and "$each" in obj: return _render_each(obj, vars, strict_ns=strict_ns) + if allow_each and "$mime" in obj: + return _render_mime(obj["$mime"], vars, strict_ns=strict_ns) return {k: render_value(v, vars, allow_each=allow_each, strict_ns=strict_ns) for k, v in obj.items()} if isinstance(obj, list): return [render_value(v, vars, allow_each=allow_each, strict_ns=strict_ns) for v in obj] return obj +#: Header name -> the key a `$mime` spec uses for it. +_MIME_HEADERS = ( + ("To", "to"), ("Cc", "cc"), ("Bcc", "bcc"), ("From", "from"), ("Reply-To", "reply_to"), + ("Subject", "subject"), ("In-Reply-To", "in_reply_to"), ("References", "references"), +) + + +def _render_mime(spec: Any, vars: dict, *, strict_ns: Collection[str] = ()) -> str: + """Build an RFC 2822 message from `{to, subject, text, ...}` and return it base64url-encoded. + + This exists because Gmail's send endpoint accepts ONLY a base64url-encoded MIME message. + Declaring that as a tool argument means asking a language model to base64-encode by hand - + which it cannot do reliably - and the malformed result comes back as an opaque HTTP 400 with + nothing in it to debug. So the model supplies the fields a person would type and the encoding + happens here, where stdlib `email` gets header encoding, non-ASCII subjects, address lists + and CRLF line endings right. + + Address fields accept a string or a list. Supply `text` (and optionally `html` for a + multipart/alternative). Empty/absent headers are omitted rather than sent blank. + """ + import base64 + from email.message import EmailMessage + from email.policy import SMTP + + if not isinstance(spec, dict): + raise ValueError("$mime expects an object, e.g. {\"to\": \"…\", \"subject\": \"…\", \"text\": \"…\"}") + resolved = {k: render_value(v, vars, strict_ns=strict_ns) for k, v in spec.items()} + + msg = EmailMessage() + for header, key in _MIME_HEADERS: + val = resolved.get(key) + if isinstance(val, (list, tuple)): + val = ", ".join(str(v).strip() for v in val if str(v).strip()) + if val is None or str(val).strip() == "": + continue + msg[header] = str(val).strip() + msg.set_content(str(resolved.get("text") or resolved.get("body") or "")) + html = resolved.get("html") + if html: + msg.add_alternative(str(html), subtype="html") + # SMTP policy => CRLF line endings, as RFC 2822 requires. + return base64.urlsafe_b64encode(msg.as_bytes(policy=SMTP)).decode() + + def _render_each(directive: dict, vars: dict, *, strict_ns: Collection[str] = ()) -> list: """Expand a `{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` loop directive into a list: render `$do` once per item of the array `$each` resolves to, with the item bound under diff --git a/apps/api/forge/config.py b/apps/api/forge/config.py index d8517a5..92b49e8 100644 --- a/apps/api/forge/config.py +++ b/apps/api/forge/config.py @@ -100,6 +100,34 @@ def _parse_tool_vars(cls, v: object) -> dict[str, str]: raise ValueError('FORGE_TOOL_VARS must be a JSON object, e.g. {"api_base":"https://..."}') return {str(k): str(val) for k, val in parsed.items()} + @field_validator("connector_oauth_apps", mode="before") + @classmethod + def _parse_connector_oauth_apps(cls, v: object) -> dict[str, dict]: + """{"google": {"client_id": "...", "client_secret": "..."}} - one entry per credential + group. Fails loudly on a malformed value: a silently-empty map would send every user + back to pasting credentials with no indication why.""" + if v is None: + return {} + if isinstance(v, dict): + return {str(k): dict(val) for k, val in v.items() if isinstance(val, dict)} + s = str(v).strip() + if not s: + return {} + import json as _json + + parsed = _json.loads(s) + if not isinstance(parsed, dict): + raise ValueError( + 'FORGE_CONNECTOR_OAUTH_APPS must be a JSON object, e.g. ' + '{"google":{"client_id":"...","client_secret":"..."}}' + ) + out: dict[str, dict] = {} + for group, creds in parsed.items(): + if not isinstance(creds, dict): + raise ValueError(f"FORGE_CONNECTOR_OAUTH_APPS['{group}'] must be an object of credential values") + out[str(group)] = {str(k): str(val) for k, val in creds.items()} + return out + # --- App --- app_name: str = "Forge" environment: str = "development" @@ -208,6 +236,23 @@ def _parse_tool_vars(cls, v: object) -> dict[str, str]: # the call with a clear error (rather than silently sending a broken URL). # Env: FORGE_TOOL_VARS='{"api_base":"https://api.example.com"}'. Blank/unset = {}. tool_vars: Annotated[dict[str, str], NoDecode] = Field(default_factory=dict) + # --- Connector OAuth apps (deployment-wide) ------------------------------------------- + # Pre-registered vendor OAuth apps, keyed by a connector's `credential_group`. When a group + # is present here, installing any connector in that family asks the operator for NOTHING: + # the install seeds the project's credentials from this map and the user goes straight to + # "Connect account" -> vendor consent screen -> done. That is the one-click experience. + # + # Register ONE app per vendor for the whole deployment (or per fleet), with + # /v1/oauth/callback as its redirect URI. For Google that means a + # "Web application" OAuth client: only that client type has an Authorized redirect URIs + # field, and Forge's callback is a real https path on the API's own origin, not a loopback + # address. (A Desktop client cannot register it at all - the sign-in then fails with + # redirect_uri_mismatch.) Each connector's `setup_help` says the same thing; keep them + # in step. + # + # Env: FORGE_CONNECTOR_OAUTH_APPS='{"google":{"client_id":"...","client_secret":"..."}}' + # Unset = each project registers its own app and pastes the credentials (previous behavior). + connector_oauth_apps: Annotated[dict[str, dict], NoDecode] = Field(default_factory=dict) # Code tools run RestrictedPython (AST-sandboxed) but NOT OS-isolated: no CPU/memory # bound and a runaway thread can't be force-killed. RestrictedPython is a hardening # layer, not a sandbox, so it is OFF by default. Only enable it on a trusted, single- diff --git a/apps/api/forge/connectors/__init__.py b/apps/api/forge/connectors/__init__.py new file mode 100644 index 0000000..8ec62c3 --- /dev/null +++ b/apps/api/forge/connectors/__init__.py @@ -0,0 +1,31 @@ +"""Connectors - pre-built integrations (Slack, Gmail, Outlook, …) as installable manifests. + +A connector is a RECIPE, not a runtime. Installing one expands a manifest into the entities +Forge already executes: + + manifest -> AuthProvider (credentials + refresh) + -> ToolSet (the folder / MCP toolset / agent grant unit) + -> Tool * N (rest_api tools, or mcp tools behind an McpClient) + +Nothing downstream knows a connector exists: agents, the workflow `tool_call` node, the MCP +server, traces, and cost accounting all see ordinary tools. That is deliberate - it means the +connector layer can never become a second execution path that drifts from the first, and an +installed connector survives the connector subsystem being removed entirely. + +Independence: the catalog is a directory of JSON files shipped inside this package. Loading it +performs no network I/O and requires no third-party service, SDK, or API key. Optional +authoring-time sources (the public MCP registry) are strictly opt-in and never required to +install or run a connector. +""" + +from forge.connectors.catalog import CATALOG, get_manifest, list_manifests +from forge.connectors.manifest import ConnectorManifest, ManifestError, parse_manifest + +__all__ = [ + "CATALOG", + "ConnectorManifest", + "ManifestError", + "get_manifest", + "list_manifests", + "parse_manifest", +] diff --git a/apps/api/forge/connectors/catalog.py b/apps/api/forge/connectors/catalog.py new file mode 100644 index 0000000..4dcc87c --- /dev/null +++ b/apps/api/forge/connectors/catalog.py @@ -0,0 +1,124 @@ +"""The bundled connector catalog - JSON manifests shipped inside the package. + +Loading is lazy, cached, and performs NO network I/O: the catalog is a directory read. That is +the whole independence guarantee in one function - Forge boots, lists connectors, and installs +them on a machine with no outbound internet access at all (the connector's own API is the only +thing it ever talks to, and only when a tool actually runs). + +A malformed manifest is skipped with a logged warning rather than taking down the catalog: one +bad file in a 25-file directory must not blank the Connectors screen. +""" + +from __future__ import annotations + +import json +import logging +from functools import lru_cache +from pathlib import Path + +from forge.connectors.manifest import ConnectorManifest, ManifestError, parse_manifest + +log = logging.getLogger("forge.connectors") + +CATALOG_DIR = Path(__file__).parent / "catalog" +#: Manifests that are shipped but NOT in the gallery, because they can't honour its promise: +#: one click, no keys, your own account. An API key, a bot token or a per-tenant subdomain has +#: to be typed by somebody, so these are offered as starting points for the custom-connector +#: path instead of as a form bolted onto a screen that says there are no forms. +EXAMPLES_DIR = Path(__file__).parent / "examples" + + +def _read_dir(directory: Path) -> dict[str, tuple[ConnectorManifest, dict]]: + """Parsed manifests paired with the JSON exactly as authored. + + The raw document is kept because the custom-connector form offers examples as a STARTING + POINT to edit: round-tripping a parsed model there would either bury the author's file under + every defaulted field or, with defaults excluded, silently drop the backend discriminator and + produce a manifest that no longer validates. + """ + out: dict[str, tuple[ConnectorManifest, dict]] = {} + if not directory.is_dir(): + return out + for path in sorted(directory.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + manifest = parse_manifest(data) + except (OSError, json.JSONDecodeError, ManifestError) as e: + log.warning("skipping connector manifest %s: %s", path.name, e) + continue + if manifest.slug in out: + log.warning("duplicate connector slug %r in %s - keeping the first", manifest.slug, path.name) + continue + out[manifest.slug] = (manifest, data) + return out + + +@lru_cache(maxsize=1) +def _load() -> dict[str, ConnectorManifest]: + return {slug: m for slug, (m, _) in _read_dir(CATALOG_DIR).items()} + + +@lru_cache(maxsize=1) +def _load_examples() -> dict[str, tuple[ConnectorManifest, dict]]: + return _read_dir(EXAMPLES_DIR) + + +def list_examples() -> list[tuple[ConnectorManifest, dict]]: + """Each example as `(manifest, source_json)`, name-sorted.""" + return sorted(_load_examples().values(), key=lambda pair: pair[0].name.lower()) + + +class _Catalog: + """Dict-like view over the bundled manifests (so callers can do `CATALOG["slack"]`).""" + + def __getitem__(self, slug: str) -> ConnectorManifest: + return _load()[slug] + + def __contains__(self, slug: object) -> bool: + return slug in _load() + + def __iter__(self): + return iter(_load().values()) + + def __len__(self) -> int: + return len(_load()) + + def get(self, slug: str) -> ConnectorManifest | None: + return _load().get(slug) + + +CATALOG = _Catalog() + + +def list_manifests() -> list[ConnectorManifest]: + """Every bundled manifest, name-sorted (the order the gallery renders in).""" + return sorted(_load().values(), key=lambda m: m.name.lower()) + + +def get_manifest(slug: str) -> ConnectorManifest | None: + return _load().get(slug) + + +def categories() -> list[str]: + seen: list[str] = [] + for m in list_manifests(): + for c in m.categories: + if c not in seen: + seen.append(c) + return sorted(seen) + + +def roles() -> list[str]: + """Every role any manifest claims to be popular for - drives the "Popular for " + picker. Sorted, with the most-referenced roles first so the default lands on a useful one.""" + counts: dict[str, int] = {} + for m in list_manifests(): + for r in m.roles: + counts[r] = counts.get(r, 0) + 1 + return [r for r, _ in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0].lower()))] + + +def reload_catalog() -> None: + """Drop the cache (tests, and a hot-reload during catalog authoring).""" + _load.cache_clear() + _load_examples.cache_clear() diff --git a/apps/api/forge/connectors/catalog/airtable.json b/apps/api/forge/connectors/catalog/airtable.json new file mode 100644 index 0000000..ad83ce5 --- /dev/null +++ b/apps/api/forge/connectors/catalog/airtable.json @@ -0,0 +1,98 @@ +{ + "format": "forge.connector/1", + "slug": "airtable", + "name": "Airtable", + "version": "2.0.0", + "publisher": "forge", + "summary": "List, create, and update records in the Airtable bases you have access to.", + "categories": ["database", "productivity"], + "roles": ["Operations", "Marketing", "Product Manager"], + "icon": "grid", + "docs_url": "https://airtable.com/developers/web/api/introduction", + "setup_url": "https://airtable.com/create/oauth", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://airtable.com/oauth2/v1/authorize", + "token_url": "https://airtable.com/oauth2/v1/token", + "token_auth": "basic", + "scopes": ["data.records:read", "data.records:write", "schema.bases:read"], + "per_user": "required", + "credential_group": "airtable", + "setup_help": "Register one OAuth integration at airtable.com/create/oauth with the callback URL below, then put its Client ID and Client secret in FORGE_CONNECTOR_OAUTH_APPS under \"airtable\". Each person then grants access to their own bases.", + "setup": [ + { "key": "client_id", "label": "Client ID", "secret": true, "required": true }, + { "key": "client_secret", "label": "Client secret", "secret": true, "required": true } + ] + }, + "egress_hosts": ["api.airtable.com"], + "backend": { + "type": "rest", + "base_url": "https://api.airtable.com/v0", + "actions": [ + { + "name": "airtable_list_bases", + "display_name": "List bases", + "description": "List the Airtable bases this account can reach, with their ids. Call this first when you don't already know the base id.", + "request": { + "method": "GET", + "url_template": "/meta/bases", + "fields": [] + }, + "response": { "projection_jmespath": "bases[].{id: id, name: name}" } + }, + { + "name": "airtable_list_records", + "display_name": "List records", + "description": "List records from a table, optionally filtered with an Airtable formula, e.g. {Status}='Open'.", + "request": { + "method": "GET", + "url_template": "/{base_id}/{table}", + "fields": [ + { "path": "base_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Base id (app…), from list bases." }, + { "path": "table", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Table name or id." }, + { "path": "filterByFormula", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Airtable formula filter." }, + { "path": "maxRecords", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 20 }, + { "path": "view", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Restrict to a named view." } + ] + }, + "response": { "projection_jmespath": "records[].{id: id, fields: fields}" } + }, + { + "name": "airtable_create_record", + "display_name": "Create record", + "description": "Create one record. `fields` is an object of column name -> value.", + "request": { + "method": "POST", + "url_template": "/{base_id}/{table}", + "fields": [ + { "path": "base_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Base id (app…)." }, + { "path": "table", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Table name or id." }, + { "path": "fields", "in": "body", "type": "object", "required": true, "llm_visible": true, "description": "Column name -> value." } + ], + "body_template": "{\"fields\":{{input.fields}}}" + }, + "response": { "projection_jmespath": "{id: id, fields: fields}" } + }, + { + "name": "airtable_update_record", + "display_name": "Update record", + "description": "Update fields on an existing record. Only the fields supplied are changed.", + "request": { + "method": "PATCH", + "url_template": "/{base_id}/{table}/{record_id}", + "fields": [ + { "path": "base_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Base id (app…)." }, + { "path": "table", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "record_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Record id (rec…)." }, + { "path": "fields", "in": "body", "type": "object", "required": true, "llm_visible": true, "description": "Column name -> new value." } + ], + "body_template": "{\"fields\":{{input.fields}}}" + }, + "response": { "projection_jmespath": "{id: id, fields: fields}" } + } + ] + }, + "toolset": { + "description": "Airtable: find bases, then list, create, and update records." + } +} diff --git a/apps/api/forge/connectors/catalog/atlassian.json b/apps/api/forge/connectors/catalog/atlassian.json new file mode 100644 index 0000000..75ecfba --- /dev/null +++ b/apps/api/forge/connectors/catalog/atlassian.json @@ -0,0 +1,31 @@ +{ + "format": "forge.connector/1", + "slug": "atlassian", + "name": "Atlassian (Jira & Confluence)", + "version": "1.0.0", + "publisher": "forge", + "summary": "Search and update Jira issues and Confluence pages in Atlassian Cloud.", + "categories": ["project-management", "docs"], + "roles": ["Software Engineer", "Product Manager", "Engineering Manager", "Support"], + "icon": "layout-dashboard", + "docs_url": "https://developer.atlassian.com/cloud/jira/platform/rest/v3/", + "auth": { + "kind": "oauth2_authorization_code", + "discover": true, + "per_user": "required", + "setup_help": "Atlassian hosts an official remote MCP server for Jira and Confluence Cloud. Connect with your Atlassian account; site access follows your own permissions.", + "setup": [ + { "key": "client_id", "label": "OAuth client ID", "secret": true, "required": false, "help": "Leave blank to let Forge register automatically." }, + { "key": "client_secret", "label": "OAuth client secret", "secret": true, "required": false } + ] + }, + "egress_hosts": ["mcp.atlassian.com", "api.atlassian.com", "auth.atlassian.com"], + "backend": { + "type": "mcp", + "url": "https://mcp.atlassian.com/v1/sse", + "transport": "sse" + }, + "toolset": { + "description": "Atlassian Cloud: Jira issue search and updates, Confluence page search and creation." + } +} diff --git a/apps/api/forge/connectors/catalog/github.json b/apps/api/forge/connectors/catalog/github.json new file mode 100644 index 0000000..88383d5 --- /dev/null +++ b/apps/api/forge/connectors/catalog/github.json @@ -0,0 +1,120 @@ +{ + "format": "forge.connector/1", + "slug": "github", + "name": "GitHub", + "version": "1.0.0", + "publisher": "forge", + "summary": "Search code, manage issues and pull requests, and read repository activity.", + "categories": ["developer"], + "roles": ["Software Engineer", "Product Manager", "Engineering Manager"], + "icon": "git-branch", + "docs_url": "https://docs.github.com/rest", + "setup_url": "https://github.com/settings/developers", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://github.com/login/oauth/authorize", + "token_url": "https://github.com/login/oauth/access_token", + "scopes": ["repo", "read:org", "read:user"], + "per_user": "required", + "credential_group": "github", + "setup_help": "Register one OAuth App at github.com/settings/developers with the callback URL below, then put its Client ID and Client secret in FORGE_CONNECTOR_OAUTH_APPS under \"github\". Everyone signs in as themselves against that one app.", + "setup": [ + { "key": "client_id", "label": "Client ID", "secret": true, "required": true }, + { "key": "client_secret", "label": "Client secret", "secret": true, "required": true } + ] + }, + "egress_hosts": ["api.github.com"], + "backend": { + "type": "rest", + "base_url": "https://api.github.com", + "actions": [ + { + "name": "github_search_issues", + "display_name": "Search issues & PRs", + "description": "Search issues and pull requests with GitHub search syntax, e.g. 'repo:acme/api is:open label:bug'.", + "request": { + "method": "GET", + "url_template": "/search/issues", + "fields": [ + { "path": "q", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "GitHub issue search query." }, + { "path": "per_page", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.github+json" }] + }, + "response": { "projection_jmespath": "items[].{number: number, title: title, state: state, url: html_url, author: user.login, labels: labels[].name}" } + }, + { + "name": "github_get_issue", + "display_name": "Read issue", + "description": "Fetch one issue or pull request, including its body.", + "request": { + "method": "GET", + "url_template": "/repos/{owner}/{repo}/issues/{number}", + "fields": [ + { "path": "owner", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Repository owner or organisation." }, + { "path": "repo", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Repository name." }, + { "path": "number", "in": "path", "type": "integer", "required": true, "llm_visible": true, "description": "Issue or PR number." } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.github+json" }] + }, + "response": { "projection_jmespath": "{number: number, title: title, state: state, body: body, author: user.login, url: html_url, labels: labels[].name}" } + }, + { + "name": "github_create_issue", + "display_name": "Create issue", + "description": "Open a new issue in a repository.", + "request": { + "method": "POST", + "url_template": "/repos/{owner}/{repo}/issues", + "fields": [ + { "path": "owner", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "repo", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "title", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Issue title." }, + { "path": "body", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "Issue description (markdown)." }, + { "path": "labels", "in": "body", "type": "array", "required": false, "llm_visible": true, "description": "Label names to apply." } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.github+json" }] + }, + "response": { "projection_jmespath": "{number: number, url: html_url}" } + }, + { + "name": "github_comment_on_issue", + "display_name": "Comment on issue", + "description": "Add a comment to an issue or pull request.", + "request": { + "method": "POST", + "url_template": "/repos/{owner}/{repo}/issues/{number}/comments", + "fields": [ + { "path": "owner", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "repo", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "number", "in": "path", "type": "integer", "required": true, "llm_visible": true }, + { "path": "body", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Comment text (markdown)." } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.github+json" }] + }, + "response": { "projection_jmespath": "{id: id, url: html_url}" } + }, + { + "name": "github_list_pull_requests", + "display_name": "List pull requests", + "description": "List a repository's pull requests, most recently updated first.", + "request": { + "method": "GET", + "url_template": "/repos/{owner}/{repo}/pulls", + "fields": [ + { "path": "owner", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "repo", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "state", "in": "query", "type": "string", "required": false, "llm_visible": true, "default": "open", "description": "open | closed | all." }, + { "path": "sort", "in": "query", "type": "string", "required": false, "llm_visible": false, "default": "updated" }, + { "path": "per_page", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.github+json" }] + }, + "response": { "projection_jmespath": "[].{number: number, title: title, state: state, author: user.login, url: html_url, draft: draft}" } + } + ] + }, + "toolset": { + "description": "GitHub: search and read issues and pull requests, open issues, and comment." + } +} diff --git a/apps/api/forge/connectors/catalog/gmail.json b/apps/api/forge/connectors/catalog/gmail.json new file mode 100644 index 0000000..80d6ec4 --- /dev/null +++ b/apps/api/forge/connectors/catalog/gmail.json @@ -0,0 +1,233 @@ +{ + "format": "forge.connector/1", + "slug": "gmail", + "name": "Gmail", + "version": "1.1.0", + "publisher": "forge", + "summary": "Read, search, label, and send email from a Gmail mailbox.", + "categories": [ + "email" + ], + "roles": [ + "Support", + "Sales", + "Operations", + "Executive Assistant" + ], + "icon": "mail", + "docs_url": "https://developers.google.com/gmail/api/reference/rest", + "setup_url": "https://console.cloud.google.com/apis/credentials", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://accounts.google.com/o/oauth2/v2/auth", + "token_url": "https://oauth2.googleapis.com/token", + "scopes": [ + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.modify" + ], + "authorize_params": { + "access_type": "offline", + "prompt": "consent" + }, + "per_user": "required", + "setup_help": "Create an OAuth client (type: Web application) in Google Cloud Console, enable the Gmail API, and add the redirect URI shown above. Each user then connects their own mailbox - a shared Gmail credential would let every end user read one person's inbox.", + "setup": [ + { + "key": "client_id", + "label": "Client ID", + "secret": true, + "required": true, + "placeholder": "....apps.googleusercontent.com" + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ], + "credential_group": "google" + }, + "egress_hosts": [ + "gmail.googleapis.com", + "oauth2.googleapis.com", + "accounts.google.com" + ], + "backend": { + "type": "rest", + "base_url": "https://gmail.googleapis.com/gmail/v1/users/me", + "actions": [ + { + "name": "gmail_search_messages", + "display_name": "Search email", + "description": "Search the mailbox using Gmail query syntax (e.g. 'from:acme is:unread newer_than:7d'). Returns message ids to pass to gmail_get_message.", + "request": { + "method": "GET", + "url_template": "/messages", + "fields": [ + { + "path": "q", + "in": "query", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Gmail search query, e.g. 'is:unread from:billing@acme.com'." + }, + { + "path": "maxResults", + "in": "query", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 10, + "description": "How many messages to return (1-50)." + } + ] + }, + "response": { + "projection_jmespath": "messages[].{id: id, thread_id: threadId}" + } + }, + { + "name": "gmail_get_message", + "display_name": "Read email", + "description": "Fetch one message by id: sender, recipient, subject, date and a text snippet.", + "request": { + "method": "GET", + "url_template": "/messages/{message_id}", + "fields": [ + { + "path": "message_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message id returned by gmail_search_messages." + }, + { + "path": "format", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "metadata" + } + ] + }, + "response": { + "projection_jmespath": "{id: id, snippet: snippet, headers: payload.headers[?name=='From' || name=='To' || name=='Subject' || name=='Date'].{name: name, value: value}}" + } + }, + { + "name": "gmail_send_message", + "display_name": "Send email", + "description": "Send an email as the connected account. Supply the recipient, subject and message text the way you would type them - Forge builds and encodes the MIME message.", + "request": { + "method": "POST", + "url_template": "/messages/send", + "fields": [ + { + "path": "to", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Recipient address. Comma-separate several." + }, + { + "path": "subject", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Subject line." + }, + { + "path": "body", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message text (plain text; newlines are preserved)." + }, + { + "path": "cc", + "in": "body", + "type": "string", + "required": false, + "llm_visible": true, + "description": "Optional Cc address(es), comma-separated." + }, + { + "path": "bcc", + "in": "body", + "type": "string", + "required": false, + "llm_visible": true, + "description": "Optional Bcc address(es), comma-separated." + } + ], + "body_template": "{\"raw\": {\"$mime\": {\"to\": \"{{input.to}}\", \"cc\": \"{{input.cc}}\", \"bcc\": \"{{input.bcc}}\", \"subject\": \"{{input.subject}}\", \"text\": \"{{input.body}}\"}}}" + }, + "response": { + "projection_jmespath": "{id: id, thread_id: threadId}" + } + }, + { + "name": "gmail_list_labels", + "display_name": "List labels", + "description": "List the mailbox's labels (folders) with their ids, for use with gmail_modify_message.", + "request": { + "method": "GET", + "url_template": "/labels", + "fields": [] + }, + "response": { + "projection_jmespath": "labels[].{id: id, name: name, type: type}" + } + }, + { + "name": "gmail_modify_message", + "display_name": "Label or archive email", + "description": "Add or remove labels on a message. Removing 'INBOX' archives it; removing 'UNREAD' marks it read.", + "request": { + "method": "POST", + "url_template": "/messages/{message_id}/modify", + "fields": [ + { + "path": "message_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message id." + }, + { + "path": "addLabelIds", + "in": "body", + "type": "array", + "required": false, + "llm_visible": true, + "description": "Label ids to add." + }, + { + "path": "removeLabelIds", + "in": "body", + "type": "array", + "required": false, + "llm_visible": true, + "description": "Label ids to remove." + } + ] + }, + "response": { + "projection_jmespath": "{id: id, label_ids: labelIds}" + } + } + ] + }, + "toolset": { + "description": "Gmail mailbox actions: search, read, send, label and archive email." + } +} diff --git a/apps/api/forge/connectors/catalog/google-calendar.json b/apps/api/forge/connectors/catalog/google-calendar.json new file mode 100644 index 0000000..5725897 --- /dev/null +++ b/apps/api/forge/connectors/catalog/google-calendar.json @@ -0,0 +1,257 @@ +{ + "format": "forge.connector/1", + "slug": "google-calendar", + "name": "Google Calendar", + "version": "1.1.0", + "publisher": "forge", + "summary": "List, create, and update events on a Google Calendar.", + "categories": [ + "calendar", + "productivity" + ], + "roles": [ + "Executive Assistant", + "Sales", + "Operations", + "Support" + ], + "icon": "calendar", + "docs_url": "https://developers.google.com/calendar/api/v3/reference", + "setup_url": "https://console.cloud.google.com/apis/credentials", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://accounts.google.com/o/oauth2/v2/auth", + "token_url": "https://oauth2.googleapis.com/token", + "scopes": [ + "https://www.googleapis.com/auth/calendar" + ], + "authorize_params": { + "access_type": "offline", + "prompt": "consent" + }, + "per_user": "required", + "setup_help": "Create an OAuth client (Web application) in Google Cloud Console, enable the Google Calendar API, and add the redirect URI shown above.", + "setup": [ + { + "key": "client_id", + "label": "Client ID", + "secret": true, + "required": true, + "placeholder": "....apps.googleusercontent.com" + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ], + "credential_group": "google" + }, + "egress_hosts": [ + "www.googleapis.com", + "oauth2.googleapis.com", + "accounts.google.com" + ], + "backend": { + "type": "rest", + "base_url": "https://www.googleapis.com/calendar/v3", + "actions": [ + { + "name": "gcal_list_events", + "display_name": "List events", + "description": "List events on a calendar between two RFC 3339 timestamps, soonest first.", + "request": { + "method": "GET", + "url_template": "/calendars/{calendar_id}/events", + "fields": [ + { + "path": "calendar_id", + "in": "path", + "type": "string", + "required": false, + "llm_visible": true, + "default": "primary", + "description": "Calendar id, or 'primary' for the user's own calendar." + }, + { + "path": "timeMin", + "in": "query", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Window start, RFC 3339 (e.g. 2026-08-13T00:00:00Z)." + }, + { + "path": "timeMax", + "in": "query", + "type": "string", + "required": false, + "llm_visible": true, + "description": "Window end, RFC 3339." + }, + { + "path": "maxResults", + "in": "query", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 20 + }, + { + "path": "singleEvents", + "in": "query", + "type": "boolean", + "required": false, + "llm_visible": false, + "default": true + }, + { + "path": "orderBy", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "startTime" + } + ] + }, + "response": { + "projection_jmespath": "items[].{id: id, summary: summary, start: start.dateTime, end: end.dateTime, attendees: attendees[].email, link: htmlLink}" + } + }, + { + "name": "gcal_create_event", + "display_name": "Create event", + "description": "Create a calendar event. Times are RFC 3339 with an offset (e.g. 2026-08-14T15:00:00+01:00).", + "request": { + "method": "POST", + "url_template": "/calendars/{calendar_id}/events", + "fields": [ + { + "path": "calendar_id", + "in": "path", + "type": "string", + "required": false, + "llm_visible": true, + "default": "primary" + }, + { + "path": "summary", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Event title." + }, + { + "path": "description", + "in": "body", + "type": "string", + "required": false, + "llm_visible": true, + "description": "Event description." + }, + { + "path": "start", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Start time, RFC3339 WITH a UTC offset, e.g. 2026-08-20T14:00:00+05:30 or 2026-08-20T08:30:00Z. A naive local time without an offset is rejected." + }, + { + "path": "end", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "End time, RFC3339 WITH a UTC offset, e.g. 2026-08-20T14:00:00+05:30 or 2026-08-20T08:30:00Z. A naive local time without an offset is rejected." + }, + { + "path": "attendees", + "in": "body", + "type": "array", + "required": false, + "llm_visible": true, + "description": "Attendee email addresses." + } + ], + "body_template": "{\"summary\":\"{{input.summary}}\",\"description\":\"{{input.description}}\",\"start\":{\"dateTime\":\"{{input.start}}\"},\"end\":{\"dateTime\":\"{{input.end}}\"},\"attendees\":{\"$each\":\"{{input.attendees}}\",\"$as\":\"email\",\"$do\":{\"email\":\"{{email}}\"}}}" + }, + "response": { + "projection_jmespath": "{id: id, link: htmlLink, start: start.dateTime}" + } + }, + { + "name": "gcal_delete_event", + "display_name": "Delete event", + "description": "Delete an event from a calendar.", + "request": { + "method": "DELETE", + "url_template": "/calendars/{calendar_id}/events/{event_id}", + "fields": [ + { + "path": "calendar_id", + "in": "path", + "type": "string", + "required": false, + "llm_visible": true, + "default": "primary" + }, + { + "path": "event_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Event id from gcal_list_events." + } + ] + } + }, + { + "name": "gcal_freebusy", + "display_name": "Check availability", + "description": "Check when one or more calendars are busy in a time window - use this before proposing a meeting slot.", + "request": { + "method": "POST", + "url_template": "/freeBusy", + "fields": [ + { + "path": "timeMin", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Window start, RFC 3339." + }, + { + "path": "timeMax", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Window end, RFC 3339." + }, + { + "path": "calendars", + "in": "body", + "type": "array", + "required": true, + "llm_visible": true, + "description": "Calendar ids / email addresses to check." + } + ], + "body_template": "{\"timeMin\":\"{{input.timeMin}}\",\"timeMax\":\"{{input.timeMax}}\",\"items\":{\"$each\":\"{{input.calendars}}\",\"$as\":\"cal\",\"$do\":{\"id\":\"{{cal}}\"}}}" + }, + "response": { + "projection_jmespath": "calendars" + } + } + ] + }, + "toolset": { + "description": "Google Calendar: list and create events, delete events, and check free/busy availability." + } +} diff --git a/apps/api/forge/connectors/catalog/google-drive.json b/apps/api/forge/connectors/catalog/google-drive.json new file mode 100644 index 0000000..b7966d3 --- /dev/null +++ b/apps/api/forge/connectors/catalog/google-drive.json @@ -0,0 +1,162 @@ +{ + "format": "forge.connector/1", + "slug": "google-drive", + "name": "Google Drive", + "version": "1.1.0", + "publisher": "forge", + "summary": "Search Drive files and read document contents as text.", + "categories": [ + "docs", + "storage" + ], + "roles": [ + "Operations", + "Product Manager", + "Marketing", + "Executive Assistant" + ], + "icon": "book-open", + "docs_url": "https://developers.google.com/drive/api/reference/rest/v3", + "setup_url": "https://console.cloud.google.com/apis/credentials", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://accounts.google.com/o/oauth2/v2/auth", + "token_url": "https://oauth2.googleapis.com/token", + "scopes": [ + "https://www.googleapis.com/auth/drive.readonly" + ], + "authorize_params": { + "access_type": "offline", + "prompt": "consent" + }, + "per_user": "required", + "setup_help": "Create an OAuth client (Web application) in Google Cloud Console, enable the Google Drive API, and add the redirect URI shown above. Read-only by default - widen the scope only if the agent needs to write.", + "setup": [ + { + "key": "client_id", + "label": "Client ID", + "secret": true, + "required": true, + "placeholder": "....apps.googleusercontent.com" + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ], + "credential_group": "google" + }, + "egress_hosts": [ + "www.googleapis.com", + "oauth2.googleapis.com", + "accounts.google.com" + ], + "backend": { + "type": "rest", + "base_url": "https://www.googleapis.com/drive/v3", + "actions": [ + { + "name": "drive_search_files", + "display_name": "Search files", + "description": "Search Drive and return each match's ID. `q` uses Drive query syntax, e.g. \"name contains 'roadmap' and mimeType='application/vnd.google-apps.document'\". This is how you turn a document NAME into the id every other Google action needs - spreadsheets are mimeType='application/vnd.google-apps.spreadsheet'.", + "request": { + "method": "GET", + "url_template": "/files", + "fields": [ + { + "path": "q", + "in": "query", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Drive search query." + }, + { + "path": "pageSize", + "in": "query", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 15 + }, + { + "path": "fields", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "files(id,name,mimeType,modifiedTime,webViewLink,owners(emailAddress))" + } + ] + }, + "response": { + "projection_jmespath": "files[].{id: id, name: name, type: mimeType, modified: modifiedTime, link: webViewLink}" + } + }, + { + "name": "drive_export_doc", + "display_name": "Read document", + "description": "Export a Google Doc as plain text. Use drive_search_files first to get the file id. Only works for Google-native documents.", + "request": { + "method": "GET", + "url_template": "/files/{file_id}/export", + "fields": [ + { + "path": "file_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Drive file id, or paste the file's full URL and Forge will pull the id out. Use drive_search_files first if you only know the name.", + "extract": "/d/([a-zA-Z0-9_-]+)" + }, + { + "path": "mimeType", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "text/plain" + } + ] + } + }, + { + "name": "drive_get_metadata", + "display_name": "File details", + "description": "Fetch one file's metadata - name, type, owners, sharing link, last modified.", + "request": { + "method": "GET", + "url_template": "/files/{file_id}", + "fields": [ + { + "path": "file_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "extract": "/d/([a-zA-Z0-9_-]+)", + "description": "Drive file id, or paste the file's full URL and Forge will pull the id out. Use drive_search_files first if you only know the name." + }, + { + "path": "fields", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "id,name,mimeType,modifiedTime,size,webViewLink,owners(emailAddress)" + } + ] + }, + "response": { + "projection_jmespath": "{id: id, name: name, type: mimeType, modified: modifiedTime, link: webViewLink, owners: owners[].emailAddress}" + } + } + ] + }, + "toolset": { + "description": "Google Drive: search files, read Google Docs as text, and inspect file metadata." + } +} diff --git a/apps/api/forge/connectors/catalog/google-sheets.json b/apps/api/forge/connectors/catalog/google-sheets.json new file mode 100644 index 0000000..0a60c63 --- /dev/null +++ b/apps/api/forge/connectors/catalog/google-sheets.json @@ -0,0 +1,199 @@ +{ + "format": "forge.connector/1", + "slug": "google-sheets", + "name": "Google Sheets", + "version": "1.2.0", + "publisher": "forge", + "summary": "Read and append rows in a Google Sheets spreadsheet.", + "categories": [ + "database", + "productivity" + ], + "roles": [ + "Operations", + "Finance", + "Marketing", + "Product Manager" + ], + "icon": "grid", + "docs_url": "https://developers.google.com/sheets/api/reference/rest", + "setup_url": "https://console.cloud.google.com/apis/credentials", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://accounts.google.com/o/oauth2/v2/auth", + "token_url": "https://oauth2.googleapis.com/token", + "scopes": [ + "https://www.googleapis.com/auth/spreadsheets" + ], + "authorize_params": { + "access_type": "offline", + "prompt": "consent" + }, + "per_user": "required", + "setup_help": "Create an OAuth client (Web application) in Google Cloud Console, enable the Google Sheets API, and add the redirect URI shown above.", + "setup": [ + { + "key": "client_id", + "label": "Client ID", + "secret": true, + "required": true, + "placeholder": "....apps.googleusercontent.com" + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ], + "credential_group": "google" + }, + "egress_hosts": [ + "sheets.googleapis.com", + "oauth2.googleapis.com", + "accounts.google.com" + ], + "backend": { + "type": "rest", + "base_url": "https://sheets.googleapis.com/v4/spreadsheets", + "actions": [ + { + "name": "sheets_read_range", + "display_name": "Read rows", + "description": "Read a range in A1 notation, e.g. 'Sheet1!A1:D50'. Returns a list of rows.", + "request": { + "method": "GET", + "url_template": "/{spreadsheet_id}/values/{range}", + "fields": [ + { + "path": "spreadsheet_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "The spreadsheet's ID, or paste its full URL and Forge will pull the ID out. This is the long key in the sheet's link (docs.google.com/spreadsheets/d//edit) - NOT the document's name. If you only know the name, ask the user for the link: a title cannot be turned into an ID. If Google Drive is also connected, drive_search_files with mimeType='application/vnd.google-apps.spreadsheet' resolves a name to an ID.", + "extract": "/spreadsheets/d/([a-zA-Z0-9_-]+)" + }, + { + "path": "range", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "A1 range, e.g. Sheet1!A1:D50." + } + ] + }, + "response": { + "projection_jmespath": "{range: range, rows: values}" + } + }, + { + "name": "sheets_append_row", + "display_name": "Append rows", + "description": "Append one or more rows after the last row of data in a range. `rows` is ALWAYS a list of rows, even when adding a single row: [[\"a\",1]] adds one row, [[\"a\",1],[\"b\",2]] adds two. Send every row in ONE call rather than calling this repeatedly.", + "request": { + "method": "POST", + "url_template": "/{spreadsheet_id}/values/{range}:append", + "fields": [ + { + "path": "spreadsheet_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "The spreadsheet's ID, or paste its full URL and Forge will pull the ID out. This is the long key in the sheet's link (docs.google.com/spreadsheets/d//edit) - NOT the document's name. If you only know the name, ask the user for the link: a title cannot be turned into an ID. If Google Drive is also connected, drive_search_files with mimeType='application/vnd.google-apps.spreadsheet' resolves a name to an ID.", + "extract": "/spreadsheets/d/([a-zA-Z0-9_-]+)" + }, + { + "path": "range", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "A1 range to append into, e.g. Sheet1!A:D." + }, + { + "path": "rows", + "in": "body", + "type": "array", + "required": true, + "llm_visible": true, + "description": "Rows to append. Each row is a list of cell values, left to right." + }, + { + "path": "valueInputOption", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "USER_ENTERED" + }, + { + "path": "insertDataOption", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "INSERT_ROWS" + } + ], + "body_template": "{\"values\": {{input.rows}}}" + }, + "response": { + "projection_jmespath": "updates.{range: updatedRange, rows: updatedRows}" + } + }, + { + "name": "sheets_update_range", + "display_name": "Update cells", + "description": "Overwrite a range with new values. `rows` is a list of rows, each a list of cell values.", + "request": { + "method": "PUT", + "url_template": "/{spreadsheet_id}/values/{range}", + "fields": [ + { + "path": "spreadsheet_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "The spreadsheet's ID, or paste its full URL and Forge will pull the ID out. This is the long key in the sheet's link (docs.google.com/spreadsheets/d//edit) - NOT the document's name. If you only know the name, ask the user for the link: a title cannot be turned into an ID. If Google Drive is also connected, drive_search_files with mimeType='application/vnd.google-apps.spreadsheet' resolves a name to an ID.", + "extract": "/spreadsheets/d/([a-zA-Z0-9_-]+)" + }, + { + "path": "range", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true + }, + { + "path": "rows", + "in": "body", + "type": "array", + "required": true, + "llm_visible": true, + "description": "List of rows; each row is a list of cell values." + }, + { + "path": "valueInputOption", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "USER_ENTERED" + } + ], + "body_template": "{\"values\":{{input.rows}}}" + }, + "response": { + "projection_jmespath": "{range: updatedRange, cells: updatedCells}" + } + } + ] + }, + "toolset": { + "description": "Google Sheets: read ranges, append rows, and update cells." + } +} diff --git a/apps/api/forge/connectors/catalog/hubspot.json b/apps/api/forge/connectors/catalog/hubspot.json new file mode 100644 index 0000000..d9dbd77 --- /dev/null +++ b/apps/api/forge/connectors/catalog/hubspot.json @@ -0,0 +1,184 @@ +{ + "format": "forge.connector/1", + "slug": "hubspot", + "name": "HubSpot", + "version": "1.1.0", + "publisher": "forge", + "summary": "Search contacts and companies, read deals, and log CRM notes.", + "categories": [ + "crm" + ], + "roles": [ + "Sales", + "Marketing", + "Support", + "Operations" + ], + "icon": "users", + "docs_url": "https://developers.hubspot.com/docs/api/crm/contacts", + "setup_url": "https://developers.hubspot.com/docs/api/working-with-oauth", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://app.hubspot.com/oauth/authorize", + "token_url": "https://api.hubapi.com/oauth/v1/token", + "scopes": [ + "oauth", + "crm.objects.contacts.read", + "crm.objects.contacts.write", + "crm.objects.companies.read", + "crm.objects.deals.read" + ], + "per_user": "required", + "credential_group": "hubspot", + "setup_help": "Create a HubSpot public app, list the scopes below on it, add the callback URL, then put its Client ID and Client secret in FORGE_CONNECTOR_OAUTH_APPS under \"hubspot\". Each person picks their own HubSpot account when they connect.", + "setup": [ + { + "key": "client_id", + "label": "Client ID", + "secret": true, + "required": true + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ] + }, + "egress_hosts": [ + "api.hubapi.com" + ], + "backend": { + "type": "rest", + "base_url": "https://api.hubapi.com/crm/v3", + "actions": [ + { + "name": "hubspot_search_contacts", + "display_name": "Search contacts", + "description": "Search contacts by free text - an email address, name, or company.", + "request": { + "method": "POST", + "url_template": "/objects/contacts/search", + "fields": [ + { + "path": "query", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Search text, e.g. an email address." + }, + { + "path": "limit", + "in": "body", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 10 + } + ], + "body_template": "{\"query\":\"{{input.query}}\",\"limit\":{{input.limit}},\"properties\":[\"email\",\"firstname\",\"lastname\",\"company\",\"phone\",\"lifecyclestage\"]}" + }, + "response": { + "projection_jmespath": "results[].{id: id, email: properties.email, name: join(' ', [properties.firstname, properties.lastname]), company: properties.company, stage: properties.lifecyclestage}" + } + }, + { + "name": "hubspot_get_contact", + "display_name": "Read contact", + "description": "Fetch one contact by id, with its standard properties.", + "request": { + "method": "GET", + "url_template": "/objects/contacts/{contact_id}", + "fields": [ + { + "path": "contact_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "HubSpot contact id." + }, + { + "path": "properties", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "email,firstname,lastname,company,phone,lifecyclestage,hs_lead_status" + } + ] + }, + "response": { + "projection_jmespath": "{id: id, properties: properties}" + } + }, + { + "name": "hubspot_search_deals", + "display_name": "Search deals", + "description": "Search deals by name or associated company.", + "request": { + "method": "POST", + "url_template": "/objects/deals/search", + "fields": [ + { + "path": "query", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Search text." + }, + { + "path": "limit", + "in": "body", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 10 + } + ], + "body_template": "{\"query\":\"{{input.query}}\",\"limit\":{{input.limit}},\"properties\":[\"dealname\",\"amount\",\"dealstage\",\"closedate\",\"pipeline\"]}" + }, + "response": { + "projection_jmespath": "results[].{id: id, name: properties.dealname, amount: properties.amount, stage: properties.dealstage, close_date: properties.closedate}" + } + }, + { + "name": "hubspot_create_note", + "display_name": "Log a note", + "description": "Create a timeline note. `timestamp` is when the note happened, ISO 8601 - use the current time unless logging something historical.", + "request": { + "method": "POST", + "url_template": "/objects/notes", + "fields": [ + { + "path": "body", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Note text." + }, + { + "path": "timestamp", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "When the note happened, ISO 8601 (e.g. 2026-08-14T10:00:00Z). Epoch milliseconds also work." + } + ], + "body_template": "{\"properties\":{\"hs_note_body\":\"{{input.body}}\",\"hs_timestamp\":\"{{input.timestamp}}\"}}" + }, + "response": { + "projection_jmespath": "{id: id, created: createdAt}" + } + } + ] + }, + "toolset": { + "description": "HubSpot CRM: search contacts and deals, read a contact, and log notes." + } +} diff --git a/apps/api/forge/connectors/catalog/linear.json b/apps/api/forge/connectors/catalog/linear.json new file mode 100644 index 0000000..817ffde --- /dev/null +++ b/apps/api/forge/connectors/catalog/linear.json @@ -0,0 +1,32 @@ +{ + "format": "forge.connector/1", + "slug": "linear", + "name": "Linear", + "version": "1.0.0", + "publisher": "forge", + "summary": "Create, search, and update Linear issues, projects, and cycles.", + "categories": ["developer", "project-management"], + "roles": ["Software Engineer", "Product Manager", "Engineering Manager"], + "icon": "bolt", + "docs_url": "https://linear.app/developers", + "setup_url": "https://linear.app/settings/api", + "auth": { + "kind": "oauth2_authorization_code", + "discover": true, + "per_user": "required", + "setup_help": "Linear hosts an official remote MCP server that supports dynamic client registration - connecting usually needs no setup at all.", + "setup": [ + { "key": "client_id", "label": "OAuth client ID", "secret": true, "required": false, "help": "Leave blank to let Forge register automatically." }, + { "key": "client_secret", "label": "OAuth client secret", "secret": true, "required": false } + ] + }, + "egress_hosts": ["mcp.linear.app", "api.linear.app"], + "backend": { + "type": "mcp", + "url": "https://mcp.linear.app/mcp", + "transport": "streamable_http" + }, + "toolset": { + "description": "Linear: search, create, and update issues, projects, and cycles." + } +} diff --git a/apps/api/forge/connectors/catalog/notion.json b/apps/api/forge/connectors/catalog/notion.json new file mode 100644 index 0000000..3bf6e8f --- /dev/null +++ b/apps/api/forge/connectors/catalog/notion.json @@ -0,0 +1,32 @@ +{ + "format": "forge.connector/1", + "slug": "notion", + "name": "Notion", + "version": "1.0.0", + "publisher": "forge", + "summary": "Search the workspace, read pages, and create or update database entries.", + "categories": ["docs", "productivity"], + "roles": ["Product Manager", "Marketing", "Operations", "Software Engineer"], + "icon": "book-open", + "docs_url": "https://developers.notion.com/reference/intro", + "setup_url": "https://www.notion.so/my-integrations", + "auth": { + "kind": "oauth2_authorization_code", + "discover": true, + "per_user": "required", + "setup_help": "Notion hosts an official remote MCP server. Forge registers itself automatically where the server allows it; otherwise create an integration at notion.so/my-integrations and paste its credentials.", + "setup": [ + { "key": "client_id", "label": "OAuth client ID", "secret": true, "required": false, "help": "Leave blank to let Forge register automatically." }, + { "key": "client_secret", "label": "OAuth client secret", "secret": true, "required": false } + ] + }, + "egress_hosts": ["mcp.notion.com", "api.notion.com"], + "backend": { + "type": "mcp", + "url": "https://mcp.notion.com/mcp", + "transport": "streamable_http" + }, + "toolset": { + "description": "Notion workspace: search, read pages and databases, create and update entries." + } +} diff --git a/apps/api/forge/connectors/catalog/outlook.json b/apps/api/forge/connectors/catalog/outlook.json new file mode 100644 index 0000000..b58d2de --- /dev/null +++ b/apps/api/forge/connectors/catalog/outlook.json @@ -0,0 +1,269 @@ +{ + "format": "forge.connector/1", + "slug": "outlook", + "name": "Outlook", + "version": "1.1.0", + "publisher": "forge", + "summary": "Read, search, and send mail from an Outlook / Microsoft 365 mailbox.", + "categories": [ + "email" + ], + "roles": [ + "Support", + "Sales", + "Operations", + "Executive Assistant" + ], + "icon": "mail", + "docs_url": "https://learn.microsoft.com/graph/api/resources/mail-api-overview", + "setup_url": "https://entra.microsoft.com", + "auth": { + "kind": "oauth2_authorization_code", + "authorize_url": "https://login.microsoftonline.com/{setup.tenant}/oauth2/v2.0/authorize", + "token_url": "https://login.microsoftonline.com/{setup.tenant}/oauth2/v2.0/token", + "scopes": [ + "offline_access", + "Mail.ReadWrite", + "Mail.Send", + "User.Read" + ], + "per_user": "required", + "setup_help": "Register an application in Microsoft Entra ID, add the redirect URI shown above as a Web platform, and grant the delegated Mail.ReadWrite / Mail.Send permissions. Use tenant 'common' for multi-tenant, or your directory (tenant) ID to restrict it to your organisation.", + "setup": [ + { + "key": "tenant", + "label": "Directory (tenant) ID", + "secret": false, + "required": false, + "default": "common", + "help": "'common' works for most setups; use your tenant GUID to lock sign-in to your org." + }, + { + "key": "client_id", + "label": "Application (client) ID", + "secret": true, + "required": true + }, + { + "key": "client_secret", + "label": "Client secret", + "secret": true, + "required": true + } + ], + "credential_group": "microsoft" + }, + "egress_hosts": [ + "graph.microsoft.com", + "login.microsoftonline.com" + ], + "backend": { + "type": "rest", + "base_url": "https://graph.microsoft.com/v1.0/me", + "actions": [ + { + "name": "outlook_search_messages", + "display_name": "Search email", + "description": "Search the mailbox. Use `search` for free-text (e.g. 'invoice overdue') or `filter` for OData filters (e.g. \"isRead eq false\").", + "request": { + "method": "GET", + "url_template": "/messages", + "fields": [ + { + "path": "$search", + "in": "query", + "type": "string", + "required": false, + "llm_visible": true, + "description": "Free-text search across the mailbox. Quote the phrase, e.g. \"invoice\"." + }, + { + "path": "$filter", + "in": "query", + "type": "string", + "required": false, + "llm_visible": true, + "description": "OData filter, e.g. isRead eq false." + }, + { + "path": "$top", + "in": "query", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 10, + "description": "How many messages to return." + }, + { + "path": "$select", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "id,subject,from,receivedDateTime,isRead,bodyPreview" + } + ] + }, + "response": { + "projection_jmespath": "value[].{id: id, subject: subject, from: from.emailAddress.address, received: receivedDateTime, unread: isRead, preview: bodyPreview}" + } + }, + { + "name": "outlook_get_message", + "display_name": "Read email", + "description": "Fetch one message by id, including its plain-text body.", + "request": { + "method": "GET", + "url_template": "/messages/{message_id}", + "fields": [ + { + "path": "message_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message id from outlook_search_messages." + }, + { + "path": "$select", + "in": "query", + "type": "string", + "required": false, + "llm_visible": false, + "default": "id,subject,from,toRecipients,receivedDateTime,body" + } + ] + }, + "response": { + "projection_jmespath": "{id: id, subject: subject, from: from.emailAddress.address, to: toRecipients[].emailAddress.address, received: receivedDateTime, body: body.content}" + } + }, + { + "name": "outlook_send_message", + "display_name": "Send email", + "description": "Send an email as the connected account. `to`, `cc` and `bcc` are lists of addresses - pass every recipient in one call.", + "request": { + "method": "POST", + "url_template": "/sendMail", + "fields": [ + { + "path": "to", + "in": "body", + "type": "array", + "required": true, + "llm_visible": true, + "description": "Recipient addresses, e.g. [\"a@x.com\",\"b@x.com\"]." + }, + { + "path": "subject", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Subject line." + }, + { + "path": "body", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message text." + }, + { + "path": "cc", + "in": "body", + "type": "array", + "required": false, + "llm_visible": true, + "description": "Optional Cc addresses." + }, + { + "path": "bcc", + "in": "body", + "type": "array", + "required": false, + "llm_visible": true, + "description": "Optional Bcc addresses." + } + ], + "body_template": "{\"message\": {\"subject\": \"{{input.subject}}\", \"body\": {\"contentType\": \"Text\", \"content\": \"{{input.body}}\"}, \"toRecipients\": {\"$each\": \"{{input.to}}\", \"$as\": \"addr\", \"$do\": {\"emailAddress\": {\"address\": \"{{addr}}\"}}}, \"ccRecipients\": {\"$each\": \"{{input.cc}}\", \"$as\": \"addr\", \"$do\": {\"emailAddress\": {\"address\": \"{{addr}}\"}}}, \"bccRecipients\": {\"$each\": \"{{input.bcc}}\", \"$as\": \"addr\", \"$do\": {\"emailAddress\": {\"address\": \"{{addr}}\"}}}}, \"saveToSentItems\": true}" + }, + "response": { + "projection_jmespath": "{sent: `true`}" + } + }, + { + "name": "outlook_reply_to_message", + "display_name": "Reply to email", + "description": "Reply to an existing message, keeping it on the same thread.", + "request": { + "method": "POST", + "url_template": "/messages/{message_id}/reply", + "fields": [ + { + "path": "message_id", + "in": "path", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Message id to reply to." + }, + { + "path": "comment", + "in": "body", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Reply text." + } + ] + }, + "response": { + "projection_jmespath": "{sent: `true`}" + } + }, + { + "name": "outlook_list_events", + "display_name": "List calendar events", + "description": "List upcoming calendar events between two ISO-8601 timestamps.", + "request": { + "method": "GET", + "url_template": "/calendarView", + "fields": [ + { + "path": "startDateTime", + "in": "query", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Window start, ISO-8601 (e.g. 2026-08-13T00:00:00Z)." + }, + { + "path": "endDateTime", + "in": "query", + "type": "string", + "required": true, + "llm_visible": true, + "description": "Window end, ISO-8601." + }, + { + "path": "$top", + "in": "query", + "type": "integer", + "required": false, + "llm_visible": true, + "default": 20 + } + ] + }, + "response": { + "projection_jmespath": "value[].{id: id, subject: subject, start: start.dateTime, end: end.dateTime, organizer: organizer.emailAddress.address}" + } + } + ] + }, + "toolset": { + "description": "Outlook / Microsoft 365 mail and calendar: search, read, send, reply, and list events." + } +} diff --git a/apps/api/forge/connectors/catalog/slack.json b/apps/api/forge/connectors/catalog/slack.json new file mode 100644 index 0000000..eef11dc --- /dev/null +++ b/apps/api/forge/connectors/catalog/slack.json @@ -0,0 +1,36 @@ +{ + "format": "forge.connector/1", + "slug": "slack", + "name": "Slack", + "version": "1.0.0", + "publisher": "forge", + "summary": "Post messages, search history, and manage channels in a Slack workspace.", + "categories": ["messaging"], + "roles": ["Software Engineer", "Support", "Sales", "Marketing", "Product Manager"], + "icon": "message-square", + "docs_url": "https://api.slack.com/docs", + "setup_url": "https://api.slack.com/apps", + "auth": { + "kind": "oauth2_authorization_code", + "discover": true, + "authorize_url": "https://slack.com/oauth/v2/authorize", + "token_url": "https://slack.com/api/oauth.v2.access", + "scopes": ["chat:write", "channels:read", "channels:history", "search:read", "users:read"], + "per_user": "required", + "setup_help": "Slack hosts an official remote MCP server. Forge tries to register itself automatically; if your workspace requires a pre-created app, make one at api.slack.com/apps, add the redirect URI shown above, and paste its credentials here.", + "setup": [ + { "key": "client_id", "label": "Client ID", "secret": true, "required": false, "help": "Leave blank to let Forge register automatically." }, + { "key": "client_secret", "label": "Client secret", "secret": true, "required": false } + ] + }, + "egress_hosts": ["slack.com", "mcp.slack.com", "api.slack.com"], + "backend": { + "type": "mcp", + "url": "https://mcp.slack.com/mcp", + "transport": "streamable_http", + "deny": ["admin_*"] + }, + "toolset": { + "description": "Slack workspace actions: send and search messages, read channels and users." + } +} diff --git a/apps/api/forge/connectors/examples/custom-rest-api.json b/apps/api/forge/connectors/examples/custom-rest-api.json new file mode 100644 index 0000000..bad0f2b --- /dev/null +++ b/apps/api/forge/connectors/examples/custom-rest-api.json @@ -0,0 +1,44 @@ +{ + "format": "forge.connector/1", + "slug": "custom-rest-api", + "name": "Custom REST API", + "version": "1.0.0", + "publisher": "forge", + "summary": "A starter connector for your own internal API - install it, then edit the actions in the Tool Builder.", + "categories": ["custom"], + "roles": ["Software Engineer"], + "icon": "k_rest", + "docs_url": "https://github.com/", + "auth": { + "kind": "bearer", + "prefix": "Bearer ", + "per_user": "optional", + "setup_help": "Point this at your own service. It installs one GET action you can duplicate and edit in the Tool Builder - or write a full manifest and use Add → Custom connector instead.", + "setup": [ + { "key": "base_url", "label": "API base URL", "secret": false, "required": true, "placeholder": "https://api.internal.acme.com/v1" }, + { "key": "token", "label": "API token", "secret": true, "required": true } + ] + }, + "backend": { + "type": "rest", + "base_url": "{setup.base_url}", + "actions": [ + { + "name": "custom_api_get", + "display_name": "GET request", + "description": "Perform a GET against a path on the configured API. Edit or duplicate this action in the Tool Builder to model your real endpoints.", + "request": { + "method": "GET", + "url_template": "/{path}", + "fields": [ + { "path": "path", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Path segment to request, relative to the API base URL." } + ], + "headers": [{ "name": "Accept", "value": "application/json" }] + } + } + ] + }, + "toolset": { + "description": "Your own REST API - edit these actions in the Tool Builder." + } +} diff --git a/apps/api/forge/connectors/examples/discord.json b/apps/api/forge/connectors/examples/discord.json new file mode 100644 index 0000000..a02b5a5 --- /dev/null +++ b/apps/api/forge/connectors/examples/discord.json @@ -0,0 +1,62 @@ +{ + "format": "forge.connector/1", + "slug": "discord", + "name": "Discord", + "version": "1.0.0", + "publisher": "forge", + "summary": "Post messages and read channel history in a Discord server.", + "categories": ["messaging"], + "roles": ["Software Engineer", "Marketing", "Support"], + "icon": "message-square", + "docs_url": "https://discord.com/developers/docs/reference", + "setup_url": "https://discord.com/developers/applications", + "auth": { + "kind": "bearer", + "header_name": "Authorization", + "prefix": "Bot ", + "per_user": "never", + "setup_help": "Create an application at discord.com/developers/applications, add a Bot, copy its token, and invite the bot to your server with the Send Messages and Read Message History permissions.", + "setup": [ + { "key": "token", "label": "Bot token", "secret": true, "required": true } + ] + }, + "egress_hosts": ["discord.com"], + "backend": { + "type": "rest", + "base_url": "https://discord.com/api/v10", + "actions": [ + { + "name": "discord_send_message", + "display_name": "Post message", + "description": "Post a message to a Discord channel.", + "request": { + "method": "POST", + "url_template": "/channels/{channel_id}/messages", + "fields": [ + { "path": "channel_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Channel id (right-click a channel with developer mode on)." }, + { "path": "content", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Message text (max 2000 characters)." } + ], + "headers": [{ "name": "Content-Type", "value": "application/json" }] + }, + "response": { "projection_jmespath": "{id: id, channel_id: channel_id}" } + }, + { + "name": "discord_channel_messages", + "display_name": "Read channel history", + "description": "Read the most recent messages in a channel, newest first.", + "request": { + "method": "GET", + "url_template": "/channels/{channel_id}/messages", + "fields": [ + { "path": "channel_id", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 20 } + ] + }, + "response": { "projection_jmespath": "[].{id: id, author: author.username, content: content, timestamp: timestamp}" } + } + ] + }, + "toolset": { + "description": "Discord: post messages and read channel history." + } +} diff --git a/apps/api/forge/connectors/examples/jira.json b/apps/api/forge/connectors/examples/jira.json new file mode 100644 index 0000000..0fd7b6c --- /dev/null +++ b/apps/api/forge/connectors/examples/jira.json @@ -0,0 +1,94 @@ +{ + "format": "forge.connector/1", + "slug": "jira", + "name": "Jira (API token)", + "version": "1.0.0", + "publisher": "forge", + "summary": "Search, read, create, and comment on Jira issues with a site API token.", + "categories": ["project-management"], + "roles": ["Software Engineer", "Product Manager", "Engineering Manager", "Support"], + "icon": "validate", + "docs_url": "https://developer.atlassian.com/cloud/jira/platform/rest/v3/", + "setup_url": "https://id.atlassian.com/manage-profile/security/api-tokens", + "auth": { + "kind": "basic", + "per_user": "optional", + "setup_help": "Use this when you want a direct REST connection with an API token rather than the OAuth Atlassian connector - for a service account, or a Data Center site. Username is your Atlassian account email; password is an API token.", + "setup": [ + { "key": "site", "label": "Jira site host", "secret": false, "required": true, "placeholder": "acme.atlassian.net" }, + { "key": "username", "label": "Account email", "secret": true, "required": true }, + { "key": "password", "label": "API token", "secret": true, "required": true } + ] + }, + "egress_hosts": ["atlassian.net"], + "backend": { + "type": "rest", + "base_url": "https://{setup.site}/rest/api/3", + "actions": [ + { + "name": "jira_search_issues", + "display_name": "Search issues", + "description": "Search issues with JQL, e.g. 'project = ENG AND status = \"In Progress\" ORDER BY updated DESC'.", + "request": { + "method": "GET", + "url_template": "/search", + "fields": [ + { "path": "jql", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "JQL query." }, + { "path": "maxResults", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 15 }, + { "path": "fields", "in": "query", "type": "string", "required": false, "llm_visible": false, "default": "summary,status,assignee,priority,updated" } + ] + }, + "response": { "projection_jmespath": "issues[].{key: key, summary: fields.summary, status: fields.status.name, assignee: fields.assignee.displayName, priority: fields.priority.name, updated: fields.updated}" } + }, + { + "name": "jira_get_issue", + "display_name": "Read issue", + "description": "Fetch one issue by key (e.g. ENG-1234).", + "request": { + "method": "GET", + "url_template": "/issue/{issue_key}", + "fields": [ + { "path": "issue_key", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Issue key, e.g. ENG-1234." }, + { "path": "fields", "in": "query", "type": "string", "required": false, "llm_visible": false, "default": "summary,description,status,assignee,priority,labels" } + ] + }, + "response": { "projection_jmespath": "{key: key, summary: fields.summary, status: fields.status.name, assignee: fields.assignee.displayName, labels: fields.labels}" } + }, + { + "name": "jira_create_issue", + "display_name": "Create issue", + "description": "Create an issue in a project. `issue_type` is the name, e.g. 'Task' or 'Bug'.", + "request": { + "method": "POST", + "url_template": "/issue", + "fields": [ + { "path": "project_key", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Project key, e.g. ENG." }, + { "path": "summary", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Issue title." }, + { "path": "description", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "Issue description (plain text)." }, + { "path": "issue_type", "in": "body", "type": "string", "required": false, "llm_visible": true, "default": "Task", "description": "Issue type name." } + ], + "body_template": "{\"fields\":{\"project\":{\"key\":\"{{input.project_key}}\"},\"summary\":\"{{input.summary}}\",\"issuetype\":{\"name\":\"{{input.issue_type}}\"},\"description\":{\"type\":\"doc\",\"version\":1,\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"{{input.description}}\"}]}]}}}" + }, + "response": { "projection_jmespath": "{key: key, id: id}" } + }, + { + "name": "jira_add_comment", + "display_name": "Comment on issue", + "description": "Add a comment to an issue.", + "request": { + "method": "POST", + "url_template": "/issue/{issue_key}/comment", + "fields": [ + { "path": "issue_key", "in": "path", "type": "string", "required": true, "llm_visible": true }, + { "path": "body", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Comment text." } + ], + "body_template": "{\"body\":{\"type\":\"doc\",\"version\":1,\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"{{input.body}}\"}]}]}}" + }, + "response": { "projection_jmespath": "{id: id, created: created}" } + } + ] + }, + "toolset": { + "description": "Jira: search issues with JQL, read an issue, create issues, and comment." + } +} diff --git a/apps/api/forge/connectors/examples/pagerduty.json b/apps/api/forge/connectors/examples/pagerduty.json new file mode 100644 index 0000000..b1c85ac --- /dev/null +++ b/apps/api/forge/connectors/examples/pagerduty.json @@ -0,0 +1,76 @@ +{ + "format": "forge.connector/1", + "slug": "pagerduty", + "name": "PagerDuty", + "version": "1.0.0", + "publisher": "forge", + "summary": "List and acknowledge incidents, and see who is on call.", + "categories": ["operations", "developer"], + "roles": ["Software Engineer", "Engineering Manager", "Operations"], + "icon": "bolt", + "docs_url": "https://developer.pagerduty.com/api-reference/", + "setup_url": "https://support.pagerduty.com/docs/api-access-keys", + "auth": { + "kind": "api_key", + "location": "header", + "param_name": "Authorization", + "per_user": "never", + "setup_help": "Create a REST API key in PagerDuty. Paste it WITH the 'Token token=' prefix, e.g. 'Token token=u+abc123'.", + "setup": [ + { "key": "api_key", "label": "Authorization header value", "secret": true, "required": true, "placeholder": "Token token=…" } + ] + }, + "egress_hosts": ["api.pagerduty.com"], + "backend": { + "type": "rest", + "base_url": "https://api.pagerduty.com", + "actions": [ + { + "name": "pagerduty_list_incidents", + "display_name": "List incidents", + "description": "List incidents, most urgent first. Filter by status: triggered, acknowledged, or resolved.", + "request": { + "method": "GET", + "url_template": "/incidents", + "fields": [ + { "path": "statuses[]", "in": "query", "type": "string", "required": false, "llm_visible": true, "default": "triggered", "description": "triggered | acknowledged | resolved." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 15 } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.pagerduty+json;version=2" }] + }, + "response": { "projection_jmespath": "incidents[].{id: id, number: incident_number, title: title, status: status, urgency: urgency, service: service.summary, created: created_at, url: html_url}" } + }, + { + "name": "pagerduty_get_incident", + "display_name": "Read incident", + "description": "Fetch one incident by id, including its assignments.", + "request": { + "method": "GET", + "url_template": "/incidents/{incident_id}", + "fields": [ + { "path": "incident_id", "in": "path", "type": "string", "required": true, "llm_visible": true, "description": "Incident id." } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.pagerduty+json;version=2" }] + }, + "response": { "projection_jmespath": "incident.{id: id, title: title, status: status, urgency: urgency, description: description, assignees: assignments[].assignee.summary, url: html_url}" } + }, + { + "name": "pagerduty_list_oncalls", + "display_name": "Who is on call", + "description": "List the current on-call assignments across escalation policies.", + "request": { + "method": "GET", + "url_template": "/oncalls", + "fields": [ + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 20 } + ], + "headers": [{ "name": "Accept", "value": "application/vnd.pagerduty+json;version=2" }] + }, + "response": { "projection_jmespath": "oncalls[].{user: user.summary, policy: escalation_policy.summary, level: escalation_level, start: start, end: end}" } + } + ] + }, + "toolset": { + "description": "PagerDuty: list and read incidents, and check who is on call." + } +} diff --git a/apps/api/forge/connectors/examples/sendgrid.json b/apps/api/forge/connectors/examples/sendgrid.json new file mode 100644 index 0000000..62b7103 --- /dev/null +++ b/apps/api/forge/connectors/examples/sendgrid.json @@ -0,0 +1,51 @@ +{ + "format": "forge.connector/1", + "slug": "sendgrid", + "name": "SendGrid", + "version": "1.0.0", + "publisher": "forge", + "summary": "Send transactional email through SendGrid.", + "categories": ["email"], + "roles": ["Marketing", "Operations", "Software Engineer"], + "icon": "mail", + "docs_url": "https://www.twilio.com/docs/sendgrid/api-reference", + "setup_url": "https://app.sendgrid.com/settings/api_keys", + "auth": { + "kind": "bearer", + "prefix": "Bearer ", + "per_user": "never", + "setup_help": "Create a restricted API key with only Mail Send permission. The sender address must already be verified in SendGrid.", + "setup": [ + { "key": "token", "label": "API key", "secret": true, "required": true, "placeholder": "SG.…" }, + { "key": "from_email", "label": "Default sender address", "secret": false, "required": true, "placeholder": "noreply@acme.com", "help": "Must be a verified sender in SendGrid." } + ] + }, + "egress_hosts": ["api.sendgrid.com"], + "backend": { + "type": "rest", + "base_url": "https://api.sendgrid.com/v3", + "actions": [ + { + "name": "sendgrid_send_email", + "display_name": "Send email", + "description": "Send a plain-text email to one recipient from the configured sender address.", + "request": { + "method": "POST", + "url_template": "/mail/send", + "fields": [ + { "path": "to", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Recipient email address." }, + { "path": "subject", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Subject line." }, + { "path": "body", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Message body (plain text)." }, + { "path": "from_email", "in": "body", "type": "string", "required": false, "llm_visible": false, "default": "{setup.from_email}" } + ], + "body_template": "{\"personalizations\":[{\"to\":[{\"email\":\"{{input.to}}\"}]}],\"from\":{\"email\":\"{{input.from_email}}\"},\"subject\":\"{{input.subject}}\",\"content\":[{\"type\":\"text/plain\",\"value\":\"{{input.body}}\"}]}", + "headers": [{ "name": "Content-Type", "value": "application/json" }] + }, + "response": { "projection_jmespath": "{sent: `true`}" } + } + ] + }, + "toolset": { + "description": "SendGrid: send transactional email." + } +} diff --git a/apps/api/forge/connectors/examples/shopify.json b/apps/api/forge/connectors/examples/shopify.json new file mode 100644 index 0000000..b7393ea --- /dev/null +++ b/apps/api/forge/connectors/examples/shopify.json @@ -0,0 +1,89 @@ +{ + "format": "forge.connector/1", + "slug": "shopify", + "name": "Shopify", + "version": "1.0.0", + "publisher": "forge", + "summary": "Look up orders, customers, and products in a Shopify store.", + "categories": ["commerce"], + "roles": ["Support", "Operations", "Marketing"], + "icon": "credit-card", + "docs_url": "https://shopify.dev/docs/api/admin-rest", + "auth": { + "kind": "api_key", + "location": "header", + "param_name": "X-Shopify-Access-Token", + "per_user": "never", + "setup_help": "Create a custom app in your Shopify admin (Settings → Apps and sales channels → Develop apps), grant the read_orders / read_customers / read_products scopes, and install it to get an Admin API access token.", + "setup": [ + { "key": "store", "label": "Store domain", "secret": false, "required": true, "placeholder": "acme-store.myshopify.com" }, + { "key": "api_key", "label": "Admin API access token", "secret": true, "required": true, "placeholder": "shpat_…" } + ] + }, + "egress_hosts": ["myshopify.com"], + "backend": { + "type": "rest", + "base_url": "https://{setup.store}/admin/api/2026-01", + "actions": [ + { + "name": "shopify_list_orders", + "display_name": "List orders", + "description": "List recent orders, optionally filtered by status or customer email.", + "request": { + "method": "GET", + "url_template": "/orders.json", + "fields": [ + { "path": "status", "in": "query", "type": "string", "required": false, "llm_visible": true, "default": "any", "description": "open | closed | cancelled | any." }, + { "path": "email", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Filter to one customer's email." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ] + }, + "response": { "projection_jmespath": "orders[].{id: id, number: order_number, email: email, total: total_price, currency: currency, financial_status: financial_status, fulfillment_status: fulfillment_status, created: created_at}" } + }, + { + "name": "shopify_get_order", + "display_name": "Read order", + "description": "Fetch one order by id, including its line items and shipping address.", + "request": { + "method": "GET", + "url_template": "/orders/{order_id}.json", + "fields": [ + { "path": "order_id", "in": "path", "type": "integer", "required": true, "llm_visible": true, "description": "Shopify order id." } + ] + }, + "response": { "projection_jmespath": "order.{id: id, number: order_number, email: email, total: total_price, financial_status: financial_status, fulfillment_status: fulfillment_status, items: line_items[].{title: title, quantity: quantity, price: price}, ship_to: shipping_address.{city: city, country: country}}" } + }, + { + "name": "shopify_search_customers", + "display_name": "Find customer", + "description": "Search customers by email, name, or phone.", + "request": { + "method": "GET", + "url_template": "/customers/search.json", + "fields": [ + { "path": "query", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "Search text, e.g. an email address." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ] + }, + "response": { "projection_jmespath": "customers[].{id: id, email: email, name: join(' ', [first_name, last_name]), orders: orders_count, spent: total_spent}" } + }, + { + "name": "shopify_list_products", + "display_name": "List products", + "description": "List products in the store, with their variants and prices.", + "request": { + "method": "GET", + "url_template": "/products.json", + "fields": [ + { "path": "title", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Filter by exact product title." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ] + }, + "response": { "projection_jmespath": "products[].{id: id, title: title, status: status, variants: variants[].{sku: sku, price: price, inventory: inventory_quantity}}" } + } + ] + }, + "toolset": { + "description": "Shopify admin: look up orders, customers, and products." + } +} diff --git a/apps/api/forge/connectors/examples/slack-api.json b/apps/api/forge/connectors/examples/slack-api.json new file mode 100644 index 0000000..9b305cb --- /dev/null +++ b/apps/api/forge/connectors/examples/slack-api.json @@ -0,0 +1,77 @@ +{ + "format": "forge.connector/1", + "slug": "slack-api", + "name": "Slack (bot token)", + "version": "1.0.0", + "publisher": "forge", + "summary": "Post messages and read channel history with a Slack bot token - no MCP server needed.", + "categories": ["messaging"], + "roles": ["Software Engineer", "Support", "Operations"], + "icon": "message-square", + "docs_url": "https://api.slack.com/methods", + "setup_url": "https://api.slack.com/apps", + "auth": { + "kind": "bearer", + "prefix": "Bearer ", + "per_user": "never", + "setup_help": "Use this instead of the OAuth Slack connector when you want a plain bot token, or when the hosted MCP server is not reachable from your network. Create an app at api.slack.com/apps, add the chat:write and channels:history bot scopes, install it, and paste the Bot User OAuth Token.", + "setup": [ + { "key": "token", "label": "Bot user OAuth token", "secret": true, "required": true, "placeholder": "xoxb-…" } + ] + }, + "egress_hosts": ["slack.com"], + "backend": { + "type": "rest", + "base_url": "https://slack.com/api", + "actions": [ + { + "name": "slack_post_message", + "display_name": "Post message", + "description": "Post a message to a channel. Use a channel id (C…) or a #channel name the bot has joined.", + "request": { + "method": "POST", + "url_template": "/chat.postMessage", + "fields": [ + { "path": "channel", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Channel id or #name." }, + { "path": "text", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Message text (Slack mrkdwn)." }, + { "path": "thread_ts", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "Reply in this thread instead of the channel." } + ], + "headers": [{ "name": "Content-Type", "value": "application/json; charset=utf-8" }] + }, + "response": { "projection_jmespath": "{ok: ok, ts: ts, channel: channel, error: error}" } + }, + { + "name": "slack_list_channels", + "display_name": "List channels", + "description": "List public channels in the workspace with their ids.", + "request": { + "method": "GET", + "url_template": "/conversations.list", + "fields": [ + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 100 }, + { "path": "exclude_archived", "in": "query", "type": "boolean", "required": false, "llm_visible": false, "default": true }, + { "path": "types", "in": "query", "type": "string", "required": false, "llm_visible": false, "default": "public_channel" } + ] + }, + "response": { "projection_jmespath": "channels[].{id: id, name: name, members: num_members, topic: topic.value}" } + }, + { + "name": "slack_channel_history", + "display_name": "Read channel history", + "description": "Read recent messages in a channel, newest first.", + "request": { + "method": "GET", + "url_template": "/conversations.history", + "fields": [ + { "path": "channel", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "Channel id (C…)." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 20 } + ] + }, + "response": { "projection_jmespath": "messages[].{ts: ts, user: user, text: text, thread_ts: thread_ts}" } + } + ] + }, + "toolset": { + "description": "Slack via bot token: post messages, list channels, read history." + } +} diff --git a/apps/api/forge/connectors/examples/stripe.json b/apps/api/forge/connectors/examples/stripe.json new file mode 100644 index 0000000..ee412ed --- /dev/null +++ b/apps/api/forge/connectors/examples/stripe.json @@ -0,0 +1,89 @@ +{ + "format": "forge.connector/1", + "slug": "stripe", + "name": "Stripe", + "version": "1.0.0", + "publisher": "forge", + "summary": "Look up customers, payments, subscriptions, and issue refunds.", + "categories": ["payments"], + "roles": ["Support", "Finance", "Operations"], + "icon": "credit-card", + "docs_url": "https://docs.stripe.com/api", + "setup_url": "https://dashboard.stripe.com/apikeys", + "auth": { + "kind": "bearer", + "prefix": "Bearer ", + "per_user": "never", + "setup_help": "Use a restricted API key with only the read (and, if you want refunds, write) permissions the agent needs. A full secret key gives an agent your whole account.", + "setup": [ + { "key": "token", "label": "Secret / restricted API key", "secret": true, "required": true, "placeholder": "rk_live_… or sk_test_…" } + ] + }, + "egress_hosts": ["api.stripe.com"], + "backend": { + "type": "rest", + "base_url": "https://api.stripe.com/v1", + "actions": [ + { + "name": "stripe_search_customers", + "display_name": "Find customer", + "description": "Find a customer by email or name using Stripe search syntax, e.g. \"email:'jane@acme.com'\".", + "request": { + "method": "GET", + "url_template": "/customers/search", + "fields": [ + { "path": "query", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "Stripe search query, e.g. email:'jane@acme.com'." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 5 } + ] + }, + "response": { "projection_jmespath": "data[].{id: id, email: email, name: name, created: created, delinquent: delinquent}" } + }, + { + "name": "stripe_list_charges", + "display_name": "List payments", + "description": "List recent charges, optionally for one customer. Amounts are in the smallest currency unit (cents).", + "request": { + "method": "GET", + "url_template": "/charges", + "fields": [ + { "path": "customer", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Customer id (cus_…) to filter by." }, + { "path": "limit", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ] + }, + "response": { "projection_jmespath": "data[].{id: id, amount: amount, currency: currency, status: status, paid: paid, refunded: refunded, created: created, description: description}" } + }, + { + "name": "stripe_list_subscriptions", + "display_name": "List subscriptions", + "description": "List a customer's subscriptions and their current status.", + "request": { + "method": "GET", + "url_template": "/subscriptions", + "fields": [ + { "path": "customer", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "Customer id (cus_…)." }, + { "path": "status", "in": "query", "type": "string", "required": false, "llm_visible": true, "default": "all", "description": "active | past_due | canceled | all." } + ] + }, + "response": { "projection_jmespath": "data[].{id: id, status: status, current_period_end: current_period_end, cancel_at_period_end: cancel_at_period_end, plan: items.data[0].price.nickname}" } + }, + { + "name": "stripe_create_refund", + "display_name": "Refund a payment", + "description": "Refund a charge, fully or partially. Amount is in the smallest currency unit; omit it to refund in full.", + "request": { + "method": "POST", + "url_template": "/refunds", + "fields": [ + { "path": "charge", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Charge id (ch_…) to refund." }, + { "path": "amount", "in": "body", "type": "integer", "required": false, "llm_visible": true, "description": "Partial amount in cents; omit for a full refund." }, + { "path": "reason", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "duplicate | fraudulent | requested_by_customer." } + ] + }, + "response": { "projection_jmespath": "{id: id, status: status, amount: amount, currency: currency}" } + } + ] + }, + "toolset": { + "description": "Stripe: look up customers, payments and subscriptions, and issue refunds." + } +} diff --git a/apps/api/forge/connectors/examples/twilio.json b/apps/api/forge/connectors/examples/twilio.json new file mode 100644 index 0000000..d129fe2 --- /dev/null +++ b/apps/api/forge/connectors/examples/twilio.json @@ -0,0 +1,64 @@ +{ + "format": "forge.connector/1", + "slug": "twilio", + "name": "Twilio", + "version": "1.0.0", + "publisher": "forge", + "summary": "Send SMS messages and read message history.", + "categories": ["messaging"], + "roles": ["Support", "Operations", "Marketing"], + "icon": "message-square", + "docs_url": "https://www.twilio.com/docs/messaging/api", + "setup_url": "https://console.twilio.com", + "auth": { + "kind": "basic", + "per_user": "never", + "setup_help": "Username is your Account SID, password is the Auth Token (or an API Key SID/Secret pair). Both are on the Twilio console dashboard.", + "setup": [ + { "key": "account_sid", "label": "Account SID", "secret": false, "required": true, "placeholder": "AC…", "help": "Also used in the request URL." }, + { "key": "username", "label": "Account SID or API Key SID", "secret": true, "required": true }, + { "key": "password", "label": "Auth token or API key secret", "secret": true, "required": true }, + { "key": "from_number", "label": "Default sending number", "secret": false, "required": true, "placeholder": "+15551234567" } + ] + }, + "egress_hosts": ["api.twilio.com"], + "backend": { + "type": "rest", + "base_url": "https://api.twilio.com/2010-04-01/Accounts/{setup.account_sid}", + "actions": [ + { + "name": "twilio_send_sms", + "display_name": "Send SMS", + "description": "Send an SMS message. `to` must be in E.164 format, e.g. +15551234567.", + "request": { + "method": "POST", + "url_template": "/Messages.json", + "fields": [ + { "path": "To", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Recipient number in E.164 format." }, + { "path": "Body", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Message text." }, + { "path": "From", "in": "body", "type": "string", "required": false, "llm_visible": false, "default": "{setup.from_number}" } + ] + }, + "response": { "projection_jmespath": "{sid: sid, status: status, to: to}" } + }, + { + "name": "twilio_list_messages", + "display_name": "List messages", + "description": "List recent SMS messages, optionally filtered by the other party's number.", + "request": { + "method": "GET", + "url_template": "/Messages.json", + "fields": [ + { "path": "To", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Filter by recipient number." }, + { "path": "From", "in": "query", "type": "string", "required": false, "llm_visible": true, "description": "Filter by sender number." }, + { "path": "PageSize", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 20 } + ] + }, + "response": { "projection_jmespath": "messages[].{sid: sid, from: from, to: to, body: body, status: status, sent: date_sent}" } + } + ] + }, + "toolset": { + "description": "Twilio: send SMS and list message history." + } +} diff --git a/apps/api/forge/connectors/examples/zendesk.json b/apps/api/forge/connectors/examples/zendesk.json new file mode 100644 index 0000000..755e486 --- /dev/null +++ b/apps/api/forge/connectors/examples/zendesk.json @@ -0,0 +1,104 @@ +{ + "format": "forge.connector/1", + "slug": "zendesk", + "name": "Zendesk", + "version": "1.0.0", + "publisher": "forge", + "summary": "Search, read, comment on, and update Zendesk support tickets.", + "categories": ["support"], + "roles": ["Support", "Operations"], + "icon": "inbox", + "docs_url": "https://developer.zendesk.com/api-reference/ticketing/introduction/", + "auth": { + "kind": "basic", + "per_user": "never", + "setup_help": "Enable token access in Zendesk (Admin → Apps and integrations → APIs → Zendesk API). The username is your agent email with '/token' appended, e.g. agent@acme.com/token, and the password is the API token.", + "setup": [ + { "key": "subdomain", "label": "Zendesk subdomain", "secret": false, "required": true, "placeholder": "acme", "help": "The 'acme' in acme.zendesk.com." }, + { "key": "username", "label": "Agent email + /token", "secret": true, "required": true, "placeholder": "agent@acme.com/token" }, + { "key": "password", "label": "API token", "secret": true, "required": true } + ] + }, + "egress_hosts": ["zendesk.com"], + "backend": { + "type": "rest", + "base_url": "https://{setup.subdomain}.zendesk.com/api/v2", + "actions": [ + { + "name": "zendesk_search_tickets", + "display_name": "Search tickets", + "description": "Search tickets with Zendesk search syntax, e.g. 'type:ticket status:open requester:jane@acme.com'.", + "request": { + "method": "GET", + "url_template": "/search.json", + "fields": [ + { "path": "query", "in": "query", "type": "string", "required": true, "llm_visible": true, "description": "Zendesk search query." }, + { "path": "per_page", "in": "query", "type": "integer", "required": false, "llm_visible": true, "default": 10 } + ] + }, + "response": { "projection_jmespath": "results[].{id: id, subject: subject, status: status, priority: priority, requester_id: requester_id, updated: updated_at}" } + }, + { + "name": "zendesk_get_ticket", + "display_name": "Read ticket", + "description": "Fetch one ticket by id, including its description.", + "request": { + "method": "GET", + "url_template": "/tickets/{ticket_id}.json", + "fields": [ + { "path": "ticket_id", "in": "path", "type": "integer", "required": true, "llm_visible": true, "description": "Ticket id." } + ] + }, + "response": { "projection_jmespath": "ticket.{id: id, subject: subject, description: description, status: status, priority: priority, tags: tags}" } + }, + { + "name": "zendesk_list_comments", + "display_name": "Read ticket conversation", + "description": "List the comments on a ticket, oldest first - the full back-and-forth with the customer.", + "request": { + "method": "GET", + "url_template": "/tickets/{ticket_id}/comments.json", + "fields": [ + { "path": "ticket_id", "in": "path", "type": "integer", "required": true, "llm_visible": true } + ] + }, + "response": { "projection_jmespath": "comments[].{author_id: author_id, public: public, created: created_at, body: plain_body}" } + }, + { + "name": "zendesk_add_comment", + "display_name": "Reply to ticket", + "description": "Add a comment to a ticket. Set public=false for an internal note the customer will not see.", + "request": { + "method": "PUT", + "url_template": "/tickets/{ticket_id}.json", + "fields": [ + { "path": "ticket_id", "in": "path", "type": "integer", "required": true, "llm_visible": true }, + { "path": "body", "in": "body", "type": "string", "required": true, "llm_visible": true, "description": "Comment text." }, + { "path": "public", "in": "body", "type": "boolean", "required": false, "llm_visible": true, "default": true, "description": "false = internal note." } + ], + "body_template": "{\"ticket\":{\"comment\":{\"body\":\"{{input.body}}\",\"public\":{{input.public}}}}}" + }, + "response": { "projection_jmespath": "ticket.{id: id, status: status, updated: updated_at}" } + }, + { + "name": "zendesk_update_ticket", + "display_name": "Update ticket", + "description": "Change a ticket's status or priority.", + "request": { + "method": "PUT", + "url_template": "/tickets/{ticket_id}.json", + "fields": [ + { "path": "ticket_id", "in": "path", "type": "integer", "required": true, "llm_visible": true }, + { "path": "status", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "new | open | pending | hold | solved | closed." }, + { "path": "priority", "in": "body", "type": "string", "required": false, "llm_visible": true, "description": "low | normal | high | urgent." } + ], + "body_template": "{\"ticket\":{\"status\":\"{{input.status}}\",\"priority\":\"{{input.priority}}\"}}" + }, + "response": { "projection_jmespath": "ticket.{id: id, status: status, priority: priority}" } + } + ] + }, + "toolset": { + "description": "Zendesk support: search and read tickets, read conversations, reply, and update status." + } +} diff --git a/apps/api/forge/connectors/install.py b/apps/api/forge/connectors/install.py new file mode 100644 index 0000000..66b44e3 --- /dev/null +++ b/apps/api/forge/connectors/install.py @@ -0,0 +1,1022 @@ +"""Expanding a connector manifest into real Forge rows (and taking them back out again). + +The install is deliberately boring: it calls the SAME services the console calls when a person +builds these by hand (SecretStore, AuthProviderService, ToolSetService, ToolService), so an +installed connector is byte-for-byte the kind of row a user could have authored. There is no +"connector runtime" to diverge from the hand-built path. + +Everything created is recorded on the ConnectorInstall row, which is what makes `uninstall` +exact - it deletes the ids it created and nothing else, so a tool the user later moved into +another tool set, or an auth provider they repointed, is never collateral damage. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from forge.connectors.manifest import ConnectorManifest, McpBackend, RestBackend +from forge.models import AuthProvider, ConnectorInstall, McpClient, Project, Tool, ToolSet +from forge.secrets.store import SecretStore +from forge.services.auth_providers import AuthProviderService +from forge.services.tool_sets import ToolSetService +from forge.services.tools import ToolService + +log = logging.getLogger("forge.connectors") + + +class InstallError(RuntimeError): + """The install cannot proceed (missing credentials, name clash, unreachable server).""" + + +def secret_name(group: str, key: str) -> str: + """Stable, collision-proof name for a connector's stored credential. + + Namespaced by CREDENTIAL GROUP (which defaults to the connector's slug) so two unrelated + connectors can both have a `client_secret` without one overwriting the other's - while + connectors authorised by the same vendor app (all the Google ones) deliberately share. + """ + return f"connector_{group}_{key}" + + +#: The one place a catalog connector's vendor credentials may come from. Named in every error +#: message on this path, because "who types the client secret" is the single question an operator +#: has when a connector says it isn't available. +ENV_VAR = "FORGE_CONNECTOR_OAUTH_APPS" + + +def deployment_app(manifest: ConnectorManifest) -> dict[str, str]: + """Deployment-wide OAuth app credentials for this manifest's group, if the operator + configured one (FORGE_CONNECTOR_OAUTH_APPS). Empty dict = none, so each project registers + its own app.""" + from forge.config import settings + + return dict(settings.connector_oauth_apps.get(manifest.group) or {}) + + +def missing_app_keys(manifest: ConnectorManifest) -> list[str]: + """Credential keys a CATALOG install needs and this deployment hasn't configured. + + Empty for a connector that needs no vendor app at all: `auth.kind: none`, or an MCP server + that publishes OAuth metadata, where Forge registers a client on the fly (RFC 7591) the + first time someone connects. + """ + if manifest.auth.kind == "none" or manifest.auth.discover: + return [] + app = deployment_app(manifest) + return [f.key for f in manifest.auth.setup if f.secret and f.required and not app.get(f.key)] + + +def env_ready(manifest: ConnectorManifest) -> bool: + """True when a catalog connector can be connected with one click, right now. + + Catalog connectors never accept typed credentials. The vendor's OAuth app belongs to the + DEPLOYMENT - it is registered once by whoever runs Forge, in the environment, next to every + other deployment secret - and each person then signs in with their own account against it. + A connector whose app isn't configured is unavailable and says so, rather than falling back + to a form that asks an end user for a client secret they have no business holding. + """ + return not missing_app_keys(manifest) + + +def not_configured_message(manifest: ConnectorManifest) -> str: + """The message an operator needs, not the one the runtime happens to know: what to register, + which env key it goes under, and what the payoff is.""" + import json as _json + + example = _json.dumps({manifest.group: {k: "…" for k in missing_app_keys(manifest)}}) + return ( + f"{manifest.name} isn't set up on this Forge deployment. Whoever runs it registers one " + f"{manifest.group} OAuth app and adds it to {ENV_VAR} (e.g. {example}), then restarts the " + "API. After that everyone signs in with their own account - nothing to paste." + ) + + +async def group_has_credentials( + secrets: SecretStore, tenant_id: str, project_id: str, manifest: ConnectorManifest +) -> bool: + """True when this connector can be installed WITHOUT asking for credentials - either the + deployment ships a registered app for its group, or a sibling connector already set one up.""" + required = [f for f in manifest.auth.setup if f.secret and f.required] + if not required: + return False + app = deployment_app(manifest) + if app and all(app.get(f.key) for f in required): + return True + for field in required: + try: + value = await secrets.read_ref( + tenant_id=tenant_id, project_id=project_id, + ref=f"secret://proj/{secret_name(manifest.group, field.key)}", + ) + except Exception: # noqa: BLE001 - absent secret means the group isn't set up + return False + if not value: + return False + return True + + +def _mcp_tool_allowed(name: str, backend: McpBackend) -> bool: + """Apply the manifest's allow/deny filter to a discovered remote tool name. + + `allow` (when non-empty) is exact and wins outright; otherwise everything passes except + `deny` entries, which support a single trailing `*` so a manifest can drop a whole family + (e.g. "admin_*") without enumerating it.""" + if backend.allow: + return name in backend.allow + for pattern in backend.deny: + if pattern.endswith("*"): + if name.startswith(pattern[:-1]): + return False + elif name == pattern: + return False + return True + + +def _auth_config(manifest: ConnectorManifest, values: dict[str, str], *, per_user: bool) -> dict[str, Any]: + """Build the AuthProvider.config for this manifest + the installer's supplied values. + + Credential VALUES never land here - only `secret://proj/` refs to what was written to + the SecretStore, matching how a hand-built provider stores them.""" + auth = manifest.auth + ref = lambda key: f"secret://proj/{secret_name(manifest.group, key)}" # noqa: E731 - local alias + cfg: dict[str, Any] = {"connector_slug": manifest.slug} + if per_user: + # The dim every per-user surface already keys on (connections router, MCP PAT identity, + # widget context). Keeping this exact string is what makes the existing self-service + # /connections routes work for connectors with no extra wiring. + cfg["per_user_context_keys"] = ["end_user_id"] + + kind = auth.kind + if kind == "bearer": + cfg["token_ref"] = ref("token") + if auth.header_name: + cfg["header_name"] = auth.header_name + if auth.prefix is not None: + cfg["prefix"] = auth.prefix + elif kind == "api_key": + cfg["value_ref"] = ref("api_key") + cfg["in"] = auth.location + cfg["name"] = auth.param_name or auth.header_name or "Authorization" + elif kind == "basic": + cfg["username_ref"] = ref("username") + cfg["password_ref"] = ref("password") + elif kind in ("oauth2_authorization_code", "oauth2_client_credentials"): + cfg["client_id_ref"] = ref("client_id") + cfg["client_secret_ref"] = ref("client_secret") + if auth.token_url: + cfg["token_url"] = auth.token_url + if auth.authorize_url: + cfg["authorize_url"] = auth.authorize_url + if auth.scopes: + cfg["scope"] = " ".join(auth.scopes) + if auth.token_auth != "post": + cfg["token_auth"] = auth.token_auth + if auth.authorize_params: + cfg["authorize_params"] = dict(auth.authorize_params) + if auth.discover: + # Endpoints are filled in at connect time from the MCP server's published metadata. + cfg["oauth_discover"] = True + # Non-secret setup values (a tenant id, a region, an instance host) are exposed to request + # templates as {{env.*}}-style config on the provider AND substituted into URLs below. + extras = {f.key: (values.get(f.key) or f.default or "") for f in auth.setup if not f.secret} + if extras: + cfg["connector_vars"] = extras + # Bake the non-secret values into the OAuth endpoints too: a Microsoft-style + # authorize/token URL embeds the tenant, so leaving `{setup.tenant}` unsubstituted here + # would send the connect flow to a literal, non-existent URL. + for key in ("authorize_url", "token_url"): + if isinstance(cfg.get(key), str): + cfg[key] = _substitute(cfg[key], extras) + return cfg + + +def _substitute(value: Any, vars: dict[str, str]) -> Any: + """Replace `{setup.key}` placeholders in manifest URLs with the installer's non-secret values. + + Uses a distinct `{setup.x}` syntax rather than the runtime `{{...}}` templating on purpose: + these are resolved ONCE at install time and baked into the stored config, so they must not + look like the per-call templates the REST tool renders at run time.""" + if isinstance(value, str): + out = value + for k, v in vars.items(): + out = out.replace("{setup." + k + "}", v) + return out + if isinstance(value, dict): + return {k: _substitute(v, vars) for k, v in value.items()} + if isinstance(value, list): + return [_substitute(v, vars) for v in value] + return value + + +class ConnectorInstaller: + """Install / uninstall / upgrade one connector in one project.""" + + def __init__(self, secrets: SecretStore | None = None) -> None: + self.secrets = secrets or SecretStore() + + # --- queries --------------------------------------------------------------------------- + + @staticmethod + async def list_installs(session: AsyncSession, tenant_id: str, project_id: str) -> list[ConnectorInstall]: + rows = await session.execute( + select(ConnectorInstall) + .where(ConnectorInstall.tenant_id == tenant_id, ConnectorInstall.project_id == project_id) + .order_by(ConnectorInstall.name) + ) + return list(rows.scalars()) + + @staticmethod + async def get_install(session: AsyncSession, tenant_id: str, project_id: str, slug: str) -> ConnectorInstall | None: + row = await session.execute( + select(ConnectorInstall).where( + ConnectorInstall.tenant_id == tenant_id, + ConnectorInstall.project_id == project_id, + ConnectorInstall.slug == slug, + ) + ) + return row.scalar_one_or_none() + + # --- install --------------------------------------------------------------------------- + + async def install( + self, + session: AsyncSession, + tenant_id: str, + project_id: str, + manifest: ConnectorManifest, + *, + values: dict[str, str] | None = None, + auth_mode: str = "shared", + source: str = "catalog", + ) -> ConnectorInstall: + values = dict(values or {}) + existing = await self.get_install(session, tenant_id, project_id, manifest.slug) + if existing is not None: + raise InstallError(f"{manifest.name} is already installed in this project") + + # A CATALOG connector is deployment-managed: its vendor app comes from the environment + # and every account on it is personal. A CUSTOM one is the escape hatch - pasted + # credentials, and the installer's choice of shared vs per-user - because that is where + # a service account or an internal API legitimately belongs. + managed = source == "catalog" + per_user = self._resolve_per_user(manifest, auth_mode, managed=managed) + if managed: + if missing_app_keys(manifest): + raise InstallError(not_configured_message(manifest)) + # Ignore anything the caller sent. There is no supported way to type a credential + # into a catalog connector, so accepting one here would create a second, invisible + # source of truth that `env_ready` would then keep reporting as unconfigured. + values = {} + else: + # A sibling connector in the same credential group may already hold the vendor app's + # credentials (install Gmail, then Google Calendar reuses the same OAuth client), in + # which case this install asks for nothing. + reuse = await group_has_credentials(self.secrets, tenant_id, project_id, manifest) + if not reuse: + self._check_required(manifest, values) + + created_secrets: list[str] = [] + # Secrets this install actually BROUGHT INTO EXISTENCE, as opposed to re-wrote with the + # same deployment value. Only these may be cleared if the install fails: blanking a + # credential a sibling connector in the same group is already using (install Gmail, then + # a failed Calendar install) would break the sibling on the way out. + new_secrets: list[str] = [] + # Same idea for the egress allow-list: only entries this install ADDED may be taken back. + new_hosts: list[str] = [] + auth_provider_id: str | None = None + tool_set_id: str | None = None + mcp_client_id: str | None = None + tool_ids: list[str] = [] + + try: + # 1. Secrets. Skipped for per-user OAuth where the END USER supplies the token, but a + # per-user OAuth connector still needs the APP's client id/secret, so those are + # written either way when present. + # + # A deployment-registered app (FORGE_CONNECTOR_OAUTH_APPS) fills in anything the + # installer didn't type. Seeding the project's own store - rather than teaching + # the resolver a second credential source - means every downstream path (resolve, + # refresh, rotate, uninstall) stays identical to a hand-pasted credential. + app = deployment_app(manifest) + for field in manifest.auth.setup: + if not field.secret: + continue + val = values.get(field.key, "") or app.get(field.key, "") + if not val: + continue + name = secret_name(manifest.group, field.key) + if not await self._secret_exists(tenant_id, project_id, name): + new_secrets.append(name) + await self.secrets.write( + session, tenant_id=tenant_id, project_id=project_id, + name=name, value=val, kind="connector", + ) + created_secrets.append(name) + + # 2. Auth provider (unless the connector needs no credentials at all). + if manifest.auth.kind != "none": + cfg = _auth_config(manifest, values, per_user=per_user) + ap = await AuthProviderService.create( + session, tenant_id, project_id, + name=manifest.name, kind=manifest.auth.kind, config=cfg, + ) + auth_provider_id = ap.id + + # 3. Tool set - the folder the user sees, the unit an agent is granted, and the + # GitHub-style toolset published over MCP. One per connector. + ts = await ToolSetService.create( + session, tenant_id, project_id, + name=manifest.name, + description=manifest.toolset.description or manifest.summary, + icon=manifest.icon, + exposed=True, + ) + tool_set_id = ts.id + + # 4. Backend-specific tools. + # Non-secret values, with the manifest's default filling in for anything the + # installer left blank - a `{setup.tenant}` placeholder must never render empty and + # silently produce a malformed URL. + setup_vars = { + f.key: (values.get(f.key) or f.default or "") + for f in manifest.auth.setup if not f.secret + } + if isinstance(manifest.backend, RestBackend): + tool_ids = await self._create_rest_tools( + session, tenant_id, project_id, manifest, ts, auth_provider_id, setup_vars + ) + else: + mcp_client_id, tool_ids = await self._create_mcp_tools( + session, tenant_id, project_id, manifest, ts, auth_provider_id, setup_vars + ) + + # 5. Egress allow-list. A no-op unless the deployment runs a strict allow-list; it + # only ever ADDS the connector's own hosts and never relaxes block_private. The + # hosts it genuinely added are recorded so uninstall can take back exactly those. + new_hosts += await self._allow_egress(session, tenant_id, project_id, manifest.hosts()) + + row = ConnectorInstall( + tenant_id=tenant_id, project_id=project_id, slug=manifest.slug, name=manifest.name, + version=manifest.version, source=source, manifest=manifest.model_dump(mode="json"), + auth_provider_id=auth_provider_id, tool_set_id=tool_set_id, mcp_client_id=mcp_client_id, + created_tool_ids=tool_ids, created_secret_names=created_secrets, + created_egress_hosts=new_hosts, + auth_mode="per_user" if per_user else "shared", + status=self._initial_status(manifest, per_user=per_user), + ) + session.add(row) + await session.commit() + await session.refresh(row) + return row + except IntegrityError as e: + # The duplicate check above is a read, and `POST /{slug}/connect` installs on demand - + # so two people clicking Connect on the same connector in the same second both pass + # it and the loser hits `uq_connector_install_slug` here. That is the same condition + # the read detected, so report it the same way rather than letting a raw DB error + # surface as a 500 on the one-click path. + # + # NO secrets are cleared on this path, deliberately. Both racers found the group's + # credentials absent and both wrote them, so they are in this install's `new_secrets` + # - but the WINNER's install now owns them. Clearing them here would blank the + # credential the surviving install needs, turning a harmless race into "client_id + # secret is not set" for everybody. + # + # Allow-list entries ARE taken back, because `_revoke_egress` asks which hosts a + # surviving install still needs - and the winner's committed row answers for its own. + await self._rollback(session, tenant_id, project_id, tool_ids, tool_set_id, + auth_provider_id, mcp_client_id, [], new_hosts) + raise InstallError(f"{manifest.name} is already installed in this project") from e + except Exception: + # A half-installed connector is worse than none: it leaves orphan tools pointing at + # an auth provider the user can't see the origin of. Roll the created rows back. + await self._rollback(session, tenant_id, project_id, tool_ids, tool_set_id, + auth_provider_id, mcp_client_id, new_secrets, new_hosts) + raise + + async def _secret_exists(self, tenant_id: str, project_id: str, name: str) -> bool: + """Whether a credential is already in this project's store, so a failed install can tell + the secrets it created from the ones it merely re-wrote.""" + try: + return bool(await self.secrets.read_ref( + tenant_id=tenant_id, project_id=project_id, ref=f"secret://proj/{name}", + )) + except Exception: # noqa: BLE001 - absent / unreadable both mean "not already set up" + return False + + @staticmethod + def _resolve_per_user(manifest: ConnectorManifest, auth_mode: str, *, managed: bool = False) -> bool: + if manifest.auth.kind == "none": + return False + if managed: + # Every catalog connector is a personal connection, full stop. An account belongs to + # the person who signed in to it, and a project-wide token would mean one person's + # mailbox, drive or repo access silently becomes everyone's. Sharing one credential + # across a project is still possible - it is a deliberate act via a custom connector. + return True + policy = manifest.auth.per_user + if policy == "required": + return True + if policy == "never": + return False + return auth_mode == "per_user" + + @staticmethod + def _check_required(manifest: ConnectorManifest, values: dict[str, str]) -> None: + missing = [ + f.label for f in manifest.auth.setup + if f.required and not (values.get(f.key) or f.default) + ] + if missing: + raise InstallError("Missing required setup values: " + ", ".join(missing)) + + @staticmethod + def _initial_status(manifest: ConnectorManifest, *, per_user: bool) -> str: + if manifest.auth.kind == "none": + return "connected" + if manifest.auth.kind == "oauth2_authorization_code": + # Nobody has run the browser consent yet - true for a shared credential and for a + # per-user one alike. On a per-user connector this row-level status only ever means + # "at least one person has connected"; the status that matters to a given person is + # computed per caller (see routers/connectors.py), because their colleague having + # connected their own mailbox says nothing about theirs. + return "needs_auth" + return "connected" + + # --- backends -------------------------------------------------------------------------- + + async def _create_rest_tools( + self, session: AsyncSession, tenant_id: str, project_id: str, + manifest: ConnectorManifest, ts: ToolSet, auth_provider_id: str | None, setup_vars: dict[str, str], + ) -> list[str]: + backend = manifest.backend + assert isinstance(backend, RestBackend) + ids: list[str] = [] + for action in backend.actions: + cfg = self._rest_tool_config(manifest, backend, action, setup_vars) + tool = await ToolService.create( + session, tenant_id, project_id, + name=action.name, kind="rest_api", config=cfg, auth_provider_id=auth_provider_id, + ) + ids.append(tool.id) + await ToolSetService.add_member(session, ts, tool.id) + return ids + + async def _create_mcp_tools( + self, session: AsyncSession, tenant_id: str, project_id: str, + manifest: ConnectorManifest, ts: ToolSet, auth_provider_id: str | None, setup_vars: dict[str, str], + ) -> tuple[str, list[str]]: + backend = manifest.backend + assert isinstance(backend, McpBackend) + client = McpClient( + tenant_id=tenant_id, project_id=project_id, name=manifest.name, + transport=backend.transport, url=_substitute(backend.url, setup_vars), + args={}, enabled=True, auth_provider_id=auth_provider_id, + ) + session.add(client) + await session.flush() + + # Discovery needs live credentials. Before the OAuth consent has been run there are + # none, so an empty tool list here is EXPECTED, not an error - the connect flow calls + # sync_mcp_tools() afterwards to fill the set in. Failing the install because an + # unauthenticated probe returned nothing would make every OAuth connector uninstallable. + tool_ids = await self._sync_mcp_tools(session, tenant_id, project_id, manifest, client, ts, best_effort=True) + return client.id, tool_ids + + async def _sync_mcp_tools( + self, session: AsyncSession, tenant_id: str, project_id: str, manifest: ConnectorManifest, + client: McpClient, ts: ToolSet, *, best_effort: bool = False, context: dict | None = None, + ) -> list[str]: + """Discover the server's tools and materialize one Tool row per allowed remote tool. + + `context` carries the per-user dims (end_user_id) of whoever's credential should be used + to ASK. It is not optional in practice for a catalog connector: those are all per-user, + so without it the resolver looks up a shared token bundle that was never written, the + request goes out unauthenticated, and the server answers 401 - which is what "0 actions" + after a successful sign-in looks like. + + Idempotent: re-running after a connect (or after the vendor adds tools) creates only the + rows that don't exist yet, so tool ids - and therefore every workflow node and agent + grant that references them - stay stable across a re-sync.""" + from forge.tools.mcp import McpUnavailable, describe_mcp_error, discover_tools + + backend = manifest.backend + assert isinstance(backend, McpBackend) + try: + remote = await discover_tools(client, tenant_id, project_id, context) + except (McpUnavailable, Exception) as e: # noqa: BLE001 - unreachable/unauthed server + detail = describe_mcp_error(e) + if best_effort: + log.info("connector %s: tool discovery deferred (%s)", manifest.slug, detail) + return [] + raise InstallError(f"Could not list tools from {manifest.name}: {detail}") from e + + existing = await session.execute( + select(Tool).where(Tool.tenant_id == tenant_id, Tool.project_id == project_id, Tool.kind == "mcp") + ) + have = { + (t.config or {}).get("remote_tool_name"): t + for t in existing.scalars() + if (t.config or {}).get("mcp_client_id") == client.id + } + + ids: list[str] = [] + for entry in remote: + name = entry.get("name") or "" + if not name or not _mcp_tool_allowed(name, backend): + continue + found = have.get(name) + if found is not None: + ids.append(found.id) + continue + tool = await ToolService.create( + session, tenant_id, project_id, name=name, kind="mcp", + config={ + "name": name, + "kind": "mcp", + "description": entry.get("description", ""), + "mcp_client_id": client.id, + "remote_tool_name": name, + "connector_slug": manifest.slug, + }, + auth_provider_id=client.auth_provider_id, + ) + ids.append(tool.id) + await ToolSetService.add_member(session, ts, tool.id) + return ids + + async def sync_tools(self, session: AsyncSession, install: ConnectorInstall, + *, context: dict | None = None) -> int: + """Bring an installed connector's actions back in line with its manifest. + + For an MCP connector that means re-asking the server what it exposes (after a successful + connect, and from the UI as "Refresh actions"). For a REST connector it means re-applying + the CURRENT catalog manifest to the tool rows - which is how a fix to a bundled connector + actually reaches a project that already installed it, without uninstalling (and losing + everyone's sign-in) to pick it up. + + `context` is whose credential to ask with - `{"end_user_id": ...}` for the per-user + connectors the catalog ships. Callers that have an identity MUST pass it; see + `_sync_mcp_tools` for what happens when nobody does.""" + from forge.connectors.manifest import parse_manifest + + manifest = parse_manifest(install.manifest or {}) + if not isinstance(manifest.backend, McpBackend): + return await self._refresh_rest_tools(session, install, manifest) + if not install.mcp_client_id: + return len(install.created_tool_ids or []) + client = (await session.execute( + select(McpClient).where(McpClient.tenant_id == install.tenant_id, McpClient.id == install.mcp_client_id) + )).scalar_one_or_none() + ts = await ToolSetService.get(session, install.tenant_id, install.tool_set_id or "") + if client is None or ts is None: + return 0 + ids = await self._sync_mcp_tools(session, install.tenant_id, install.project_id, manifest, + client, ts, context=context) + merged = list(dict.fromkeys([*(install.created_tool_ids or []), *ids])) + install.created_tool_ids = merged + await session.commit() + return len(ids) + + async def _refresh_rest_tools(self, session: AsyncSession, install: ConnectorInstall, + frozen: ConnectorManifest) -> int: + """Re-apply a REST connector's manifest to the tool rows it created. + + A catalog connector installed last week holds a COPY of the manifest as it was then. When + a bundled manifest is corrected - as Gmail's send action was, once it turned out that + asking a model to base64-encode a MIME message produces an opaque 400 - the fix has to be + able to reach projects that already installed it. Uninstall/reinstall would work but + deletes the auth provider, so every person who signed in would have to do it again for a + change they had nothing to do with. + + Tool IDS are preserved: rows are matched by action name and updated in place, so every + workflow node and agent grant pointing at them keeps working. Actions the manifest no + longer declares are LEFT ALONE rather than deleted - a workflow may still reference one, + and breaking a live graph is a worse failure than carrying a stale tool. + """ + # The current catalog entry for a bundled connector; the frozen copy for a pasted one, + # which has no newer version to move to. + latest = frozen + if install.source == "catalog": + from forge.connectors import catalog as catalog_mod + + found = catalog_mod.get_manifest(install.slug) + if found is not None: + latest = found + backend = latest.backend + if not isinstance(backend, RestBackend): + return len(install.created_tool_ids or []) + + rows = list((await session.execute( + select(Tool).where( + Tool.tenant_id == install.tenant_id, Tool.project_id == install.project_id, + Tool.id.in_(install.created_tool_ids or ["__none__"]), + ) + )).scalars()) + by_name = {t.name: t for t in rows} + # An action this install ALREADY created that no longer has a row was deleted on purpose + # - a project that removed "send email" made a decision. Recreating it on refresh would + # quietly hand back a capability someone took away. The frozen manifest says what this + # install created, so a name in there with no live row is a deliberate deletion, while a + # name absent from it is genuinely new in the upgrade and worth adding. + previously_declared = { + a.name for a in (frozen.backend.actions if isinstance(frozen.backend, RestBackend) else []) + } + + ts = await ToolSetService.get(session, install.tenant_id, install.tool_set_id or "") + # The non-secret values THIS install was created with (a Jira site, a custom connector's + # base_url), kept on the auth provider as `connector_vars`. Rebuilding URLs from the + # manifest defaults instead would silently repoint every tool at the wrong host - the + # defaults are what the installer overrode. + setup_vars = {f.key: (f.default or "") for f in latest.auth.setup if not f.secret} + if install.auth_provider_id: + ap = (await session.execute( + select(AuthProvider).where( + AuthProvider.tenant_id == install.tenant_id, + AuthProvider.id == install.auth_provider_id, + ) + )).scalar_one_or_none() + for key, val in ((ap.config or {}).get("connector_vars") or {}).items() if ap else (): + if val: + setup_vars[key] = val + # Start from the tools that still EXIST. Ids of rows deleted from the Tools screen would + # otherwise sit in the receipt forever, growing the IN clause on every refresh and + # sending uninstall looking for rows that are already gone. + ids = [t.id for t in rows] + changed = 0 + for action in backend.actions: + cfg = self._rest_tool_config(latest, backend, action, setup_vars) + tool = by_name.get(action.name) + if tool is not None: + if (tool.config or {}) != cfg: + tool.config = cfg + changed += 1 + continue + if action.name in previously_declared: + log.info("connector %s: leaving %s deleted (removed from this project on purpose)", + install.slug, action.name) + continue + created = await ToolService.create( + session, install.tenant_id, install.project_id, + name=action.name, kind="rest_api", config=cfg, + auth_provider_id=install.auth_provider_id, + ) + ids.append(created.id) + changed += 1 + if ts is not None: + await ToolSetService.add_member(session, ts, created.id) + + install.created_tool_ids = ids + install.manifest = self._stored_manifest(latest, frozen, slug=install.slug) + install.version = latest.version + await session.commit() + log.info("connector %s: refreshed %d action(s) to v%s", install.slug, changed, latest.version) + return len(ids) + + @staticmethod + def _stored_manifest(latest: ConnectorManifest, frozen: ConnectorManifest, *, slug: str) -> dict: + """The manifest to store after an upgrade: the new one, with this install's ORIGINAL + credential group pinned. + + Everything else about a manifest may legitimately move on - new actions, corrected + request templates, a new summary. The credential GROUP may not, because it is not a + description, it is a pointer: `secret_name(group, ...)` names the secrets this install + actually wrote, and `_group_still_in_use` compares groups to decide whether uninstalling + one connector may clear the vendor app the rest of its family shares. Letting a catalog + edit repoint that would make an install's own credentials unreachable, and would let + uninstalling Gmail blank the Google client secret while Calendar, Drive and Sheets are + still using it. + """ + stored = latest.model_dump(mode="json") + if latest.group != frozen.group: + log.warning( + "connector %s: catalog moved it from credential group %r to %r; keeping %r, " + "which is where this install's credentials actually live", + slug, frozen.group, latest.group, frozen.group, + ) + auth = dict(stored.get("auth") or {}) + auth["credential_group"] = frozen.group + stored["auth"] = auth + return stored + + @staticmethod + def _rest_tool_config(manifest: ConnectorManifest, backend: RestBackend, action, + setup_vars: dict[str, str]) -> dict[str, Any]: + """The `Tool.config` one REST action expands to. Shared by install and refresh so the two + paths cannot drift into producing different tools from the same manifest.""" + base = _substitute(backend.base_url, setup_vars) + request = _substitute(dict(action.request), setup_vars) + url = str(request.get("url_template", "")) + # A relative action URL composes with the manifest's base_url; an absolute one wins, so a + # single connector can reach a second host (e.g. an upload endpoint) when needed. + if base and not url.startswith(("http://", "https://")): + request["url_template"] = base.rstrip("/") + "/" + url.lstrip("/") + cfg: dict[str, Any] = { + "name": action.name, + "description": action.description, + "kind": "rest_api", + "request": request, + "connector_slug": manifest.slug, + } + if action.display_name: + cfg["display_name"] = action.display_name + if action.response: + cfg["response"] = action.response + for key in ("timeout_seconds", "retry", "rate_limit", "cache"): + val = getattr(action, key) + if val is not None: + cfg[key] = val + return cfg + + # --- egress ---------------------------------------------------------------------------- + + @staticmethod + async def _allow_egress(session: AsyncSession, tenant_id: str, project_id: str, + hosts: list[str]) -> list[str]: + """Add this connector's hosts to the project's allow-list, and report which ones were + genuinely NEW. + + The return value is the receipt uninstall needs. A host already on the list was put there + by a person or by a sibling connector, so it is not this install's to remove later. + """ + if not hosts: + return [] + project = (await session.execute( + select(Project).where(Project.tenant_id == tenant_id, Project.id == project_id) + )).scalar_one_or_none() + if project is None: + return [] + cfg = dict(project.config or {}) + egress = dict(cfg.get("egress") or {}) + allow = list(egress.get("allow_hosts") or []) + added = [h for h in hosts if h not in allow] + if not added: + return [] + egress["allow_hosts"] = [*allow, *added] + cfg["egress"] = egress + project.config = cfg + await session.commit() + return added + + @staticmethod + async def _revoke_egress(session: AsyncSession, tenant_id: str, project_id: str, + hosts: list[str], *, exclude_install_id: str | None = None) -> None: + """Take back allow-list entries a connector added, minus anything a SURVIVING install + still needs. + + Two things this must not do. It must not remove a host that was on the list before any + connector existed - hence `hosts` is the install's recorded receipt, never + `manifest.hosts()`. And it must not strand a sibling: Gmail and Sheets both reach + `oauth2.googleapis.com`, so uninstalling one while the other remains has to leave it. + + Survivors are compared on their MANIFEST hosts rather than their own receipts, which + deliberately over-approximates: the first Google connector installed recorded + `oauth2.googleapis.com` and the second recorded nothing (it was already allowed), so + reading receipts would let uninstalling the first one revoke a host the second is using. + + A host that IS still needed has its receipt handed to the survivor that needs it. Without + that hand-over the record dies with the first install - uninstall Gmail (correctly keeping + `oauth2.googleapis.com` for Calendar) and then Calendar, and the host is orphaned on the + allow-list forever, because the only install that ever claimed to have added it is gone. + """ + if not hosts: + return + project = (await session.execute( + select(Project).where(Project.tenant_id == tenant_id, Project.id == project_id) + )).scalar_one_or_none() + if project is None: + return + cfg = dict(project.config or {}) + egress = dict(cfg.get("egress") or {}) + allow = list(egress.get("allow_hosts") or []) + if not allow: + return + + from forge.connectors.manifest import parse_manifest + + stmt = select(ConnectorInstall).where( + ConnectorInstall.tenant_id == tenant_id, ConnectorInstall.project_id == project_id, + ) + if exclude_install_id: + stmt = stmt.where(ConnectorInstall.id != exclude_install_id) + # host -> the surviving install that will inherit responsibility for it. + keeper_of: dict[str, ConnectorInstall] = {} + for other in (await session.execute(stmt)).scalars(): + try: + needed = parse_manifest(other.manifest or {}).hosts() + except Exception: # noqa: BLE001 - an unparseable sibling keeps everything it might need + needed = list(other.created_egress_hosts or []) + for host in needed: + keeper_of.setdefault(host, other) + + drop = [h for h in hosts if h not in keeper_of] + changed = False + for host in hosts: + keeper = keeper_of.get(host) + if keeper is None: + continue + receipt = list(keeper.created_egress_hosts or []) + if host not in receipt: + keeper.created_egress_hosts = [*receipt, host] + changed = True + + remaining = [h for h in allow if h not in set(drop)] + if remaining != allow: + egress["allow_hosts"] = remaining + cfg["egress"] = egress + project.config = cfg + changed = True + if changed: + await session.commit() + + # --- uninstall ------------------------------------------------------------------------- + + async def uninstall(self, session: AsyncSession, install: ConnectorInstall) -> None: + """Remove exactly what this install created. + + Tools are deleted through ToolService so their tool-set memberships go with them. + Agents/workflows that referenced a deleted tool id keep working - the compiler already + skips ids it can't resolve - so uninstalling breaks a graph's capability, never its run. + """ + tenant_id = install.tenant_id + for tool_id in install.created_tool_ids or []: + tool = await ToolService.get(session, tenant_id, tool_id) + if tool is not None: + await ToolService.delete(session, tool) + + if install.mcp_client_id: + client = (await session.execute( + select(McpClient).where(McpClient.tenant_id == tenant_id, McpClient.id == install.mcp_client_id) + )).scalar_one_or_none() + if client is not None: + await session.delete(client) + await session.commit() + from forge.tools.mcp import invalidate_client + await invalidate_client(install.mcp_client_id) + + if install.tool_set_id: + ts = await ToolSetService.get(session, tenant_id, install.tool_set_id) + if ts is not None: + await ToolSetService.delete(session, ts) + + if install.auth_provider_id: + ap = (await session.execute( + select(AuthProvider).where(AuthProvider.tenant_id == tenant_id, AuthProvider.id == install.auth_provider_id) + )).scalar_one_or_none() + if ap is not None: + await AuthProviderService.delete(session, ap) + + # Stored credentials are overwritten with an empty value rather than row-deleted, which + # keeps the secret's audit history intact (the store audits reads at one choke point) + # while making the credential unusable immediately. + # + # EXCEPT when a sibling connector still shares this credential group: Gmail, Calendar, + # Drive and Sheets are one Google OAuth app, so blanking the client secret on the way + # out would silently break every other Google connector in the project. + shared = await self._group_still_in_use(session, install) + if shared: + log.info("connector uninstall: keeping %s's shared credentials (still used by %s)", + install.slug, shared) + else: + # Clear the whole GROUP's credentials, not just the names this install happens to + # have recorded. When two connectors share a vendor app only the FIRST one records + # the secret names, so uninstalling that one first (kept, correctly, because a + # sibling remained) and then the sibling would otherwise leave the credential + # orphaned in the store forever with no install left to point at it. + for name in self._group_secret_names(install): + try: + await self.secrets.write( + session, tenant_id=tenant_id, project_id=install.project_id, + name=name, value="", kind="connector", + ) + except Exception as e: # noqa: BLE001 - a missing secret is already the goal state + log.debug("connector uninstall: could not clear secret %s (%s)", name, e) + + # Give back the allow-list entries this install added. Without this, a deployment running + # a strict allow-list only ever grows one: install Gmail, Sheets, HubSpot and Stripe to + # evaluate them, uninstall all four, and their API hosts stay permanently reachable with + # nothing referencing them - which is the thing default-deny exists to prevent. + # + # An install from before `created_egress_hosts` existed has an empty receipt, and that + # means "unknown", not "added nothing". Leaving its hosts alone is the safe reading: the + # alternative is guessing from the manifest and revoking a host a person allow-listed. + await self._revoke_egress( + session, tenant_id, install.project_id, + list(install.created_egress_hosts or []), exclude_install_id=install.id, + ) + + await session.delete(install) + await session.commit() + + @staticmethod + def _group_secret_names(install: ConnectorInstall) -> list[str]: + """Every secret name belonging to this install's credential group: the ones it recorded + at install time, plus the ones its manifest declares (which a SIBLING may have written).""" + from forge.connectors.manifest import parse_manifest + + names = list(install.created_secret_names or []) + try: + manifest = parse_manifest(install.manifest or {}) + except Exception: # noqa: BLE001 - fall back to just the recorded names + return names + for field in manifest.auth.setup: + if not field.secret: + continue + name = secret_name(manifest.group, field.key) + if name not in names: + names.append(name) + return names + + @staticmethod + async def _group_still_in_use(session: AsyncSession, install: ConnectorInstall) -> str | None: + """The name of another installed connector sharing this one's credential group, if any. + + Read from each install's STORED manifest rather than the live catalog, so the answer + reflects what is actually deployed even if a catalog update later regrouped things. An + upgrade rewrites that manifest but pins the credential group (see `_stored_manifest`), + which is what keeps this comparison meaningful across refreshes.""" + from forge.connectors.manifest import parse_manifest + + try: + mine = parse_manifest(install.manifest or {}) + except Exception: # noqa: BLE001 - an unparseable manifest can't claim a shared group + return None + rows = await session.execute( + select(ConnectorInstall).where( + ConnectorInstall.tenant_id == install.tenant_id, + ConnectorInstall.project_id == install.project_id, + ConnectorInstall.id != install.id, + ) + ) + for other in rows.scalars(): + try: + if parse_manifest(other.manifest or {}).group == mine.group: + return other.name + except Exception: # noqa: BLE001 - skip a sibling we can't parse + continue + return None + + async def _rollback( + self, session: AsyncSession, tenant_id: str, project_id: str, tool_ids: list[str], + tool_set_id: str | None, auth_provider_id: str | None, mcp_client_id: str | None, + secret_names: list[str], egress_hosts: list[str] | None = None, + ) -> None: + """Best-effort teardown of a failed partial install. Each step is independently guarded: + the reason we're here is that something already went wrong, so a second failure must not + mask the original exception being re-raised by the caller. + + `secret_names` must be only the credentials this install BROUGHT INTO EXISTENCE (see + `new_secrets` in `install`). Every service on this path commits as it goes, so there is + no transaction to unwind - the rows and the secrets both have to be removed explicitly, + and leaving the secrets behind would strand a vendor client id/secret in the store with + no install pointing at it, which `group_has_credentials` then reads as "this group is + already set up". + """ + try: + await session.rollback() + except Exception: # noqa: BLE001 + pass + try: + # Reuse uninstall's teardown, minus the final row delete (nothing was ever added). + for tool_id in tool_ids or []: + tool = await ToolService.get(session, tenant_id, tool_id) + if tool is not None: + await ToolService.delete(session, tool) + if mcp_client_id: + client = (await session.execute( + select(McpClient).where(McpClient.tenant_id == tenant_id, McpClient.id == mcp_client_id) + )).scalar_one_or_none() + if client is not None: + await session.delete(client) + await session.commit() + if tool_set_id: + ts = await ToolSetService.get(session, tenant_id, tool_set_id) + if ts is not None: + await ToolSetService.delete(session, ts) + if auth_provider_id: + ap = (await session.execute( + select(AuthProvider).where(AuthProvider.tenant_id == tenant_id, AuthProvider.id == auth_provider_id) + )).scalar_one_or_none() + if ap is not None: + await AuthProviderService.delete(session, ap) + except Exception as e: # noqa: BLE001 + log.warning("connector install rollback was incomplete: %s", e) + for name in secret_names or []: + try: + await self.secrets.write( + session, tenant_id=tenant_id, project_id=project_id, + name=name, value="", kind="connector", + ) + except Exception as e: # noqa: BLE001 - an unwritable secret is not worth masking + log.warning("connector install rollback could not clear secret %s: %s", name, e) + try: + # The install may have widened the egress allow-list before it failed. There is no + # install row to point at those hosts now, so leaving them behind is the same + # permanent widening uninstall exists to prevent. + await self._revoke_egress(session, tenant_id, project_id, list(egress_hosts or [])) + except Exception as e: # noqa: BLE001 + log.warning("connector install rollback could not revoke egress hosts: %s", e) diff --git a/apps/api/forge/connectors/manifest.py b/apps/api/forge/connectors/manifest.py new file mode 100644 index 0000000..0f0d41b --- /dev/null +++ b/apps/api/forge/connectors/manifest.py @@ -0,0 +1,309 @@ +"""The `forge.connector/1` manifest - the one format a connector is authored in. + +A manifest declares WHAT to create, never HOW to execute. Two backends cover the field: + +* `mcp` - the vendor runs an MCP server (Slack, Google Workspace, Microsoft 365, Notion, + Linear, GitHub, Atlassian). Forge registers it, discovers its tools, and lets the + vendor own their own API churn. This is the right default in 2026 for any vendor + that ships one. +* `rest` - Forge calls the API directly. Each action is a VERBATIM `Tool.config` payload for + the existing rest_api tool, so there is no second tool format to keep in sync, and + an action can use every feature the Tool Builder already has (templating, response + projection, retry, rate limits, caching). + +Secrets never appear in a manifest. `auth.setup[]` declares which credentials to ASK the +installer for; the values go straight to the SecretStore and the created AuthProvider refers to +them as `secret://` refs - the same rule services/portability.py enforces for bundles. +""" + +from __future__ import annotations + +import re +from typing import Any, Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +FORMAT = "forge.connector/1" + +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") +# A tool name is used verbatim as the LLM tool name, so hold it to the provider-safe charset +# every model API accepts (mirrors the component-name rule in services/portability.py). +_TOOL_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") + +# Auth kinds a manifest may declare. Deliberately a SUBSET of AuthResolver's kinds: +# `custom_script` is an audited advanced path and must never be reachable by installing a +# pasted manifest, and `csrf_session` is bespoke enough that it belongs in the Auth Providers +# screen rather than in a shareable catalog entry. +AuthKind = Literal["none", "bearer", "api_key", "basic", "oauth2_authorization_code", "oauth2_client_credentials"] + + +class ManifestError(ValueError): + """A manifest is malformed or declares something an install refuses to create.""" + + +class SetupField(BaseModel): + """One credential the installer must supply (rendered as a form field). + + `secret=True` values are written to the SecretStore and referenced as `secret://proj/`; + non-secret values are substituted into the manifest as plain config (e.g. a tenant id or a + region that forms part of a URL). + """ + + key: str + label: str + help: str | None = None + secret: bool = True + required: bool = True + placeholder: str | None = None + default: str | None = None + + @field_validator("key") + @classmethod + def _key_ok(cls, v: str) -> str: + if not re.fullmatch(r"[a-z0-9_]{1,40}", v or ""): + raise ValueError("setup key must be lowercase alphanumeric/underscore") + return v + + +class AuthSpec(BaseModel): + kind: AuthKind = "none" + # oauth2_* + authorize_url: str | None = None + token_url: str | None = None + scopes: list[str] = Field(default_factory=list) + # How the client authenticates at the token endpoint. "post" (client id/secret in the form + # body) is what most vendors accept; "basic" is the HTTP Basic method RFC 6749 makes + # mandatory for servers and that a few - Airtable among them - accept exclusively. + token_auth: Literal["post", "basic"] = "post" + # Extra authorize-URL parameters a vendor needs and the generic flow has no opinion about + # (Google's access_type/prompt for a refresh token, Slack's user_scope, HubSpot's optional + # scopes). Copied verbatim onto the query string. + authorize_params: dict[str, str] = Field(default_factory=dict) + # api_key / bearer + header_name: str | None = None + prefix: str | None = None + location: Literal["header", "query"] = "header" + param_name: str | None = None + # Whether each end user connects their own account, or the project shares one. + # never - shared only (a service/bot credential) + # optional - installer chooses (default) + # required - per-user only (a personal mailbox: a shared token would be wrong) + per_user: Literal["never", "optional", "required"] = "optional" + setup: list[SetupField] = Field(default_factory=list) + setup_help: str | None = None + # Connectors that are authorised by the SAME vendor OAuth app share a credential group, so + # the app is registered once and every connector in the family reuses it. Gmail, Calendar, + # Drive and Sheets are one Google Cloud OAuth client; Outlook and Teams are one Entra app. + # Without this, a user pastes identical credentials four times and then has four copies to + # rotate. Defaults to the connector's own slug (a family of one). + credential_group: str | None = None + # For MCP-backed connectors whose server advertises OAuth metadata (RFC 9728/8414): let + # Forge discover the endpoints and register a client dynamically (RFC 7591) instead of + # requiring the installer to create an app by hand. Falls back to the static urls above. + discover: bool = False + + @model_validator(mode="after") + def _coherent(self) -> AuthSpec: + if self.kind in ("oauth2_authorization_code", "oauth2_client_credentials"): + # `discover` supplies these at connect time for MCP servers that advertise metadata. + if not self.discover and not self.token_url: + raise ValueError(f"{self.kind} requires token_url (or discover: true)") + if self.kind == "oauth2_authorization_code" and not self.discover and not self.authorize_url: + raise ValueError("oauth2_authorization_code requires authorize_url (or discover: true)") + if self.kind == "api_key" and self.location == "query" and not self.param_name: + raise ValueError("api_key in query requires param_name") + return self + + +class McpBackend(BaseModel): + type: Literal["mcp"] = "mcp" + url: str + transport: Literal["streamable_http", "sse"] = "streamable_http" + # Which of the server's tools to expose. `allow` (if non-empty) is an exact allow-list; + # otherwise everything is exposed except `deny` (supports a trailing `*` wildcard). + allow: list[str] = Field(default_factory=list) + deny: list[str] = Field(default_factory=list) + + @field_validator("url") + @classmethod + def _https(cls, v: str) -> str: + if not v.startswith(("http://", "https://")): + raise ValueError("mcp backend url must be http(s)") + return v + + +class RestAction(BaseModel): + """One REST action == one `rest_api` Tool. + + `request` and `response` are the EXACT structures `tools/rest.py` already consumes and the + Tool Builder already edits - `{method, url_template, fields[], headers[], body_template}` + and `{projection_jmespath}` / `{fields[]}`. They are copied into `Tool.config` untouched. + + Passing them through verbatim rather than inventing a friendlier connector-specific schema + is the point: it means an action gets every capability the Tool Builder has (per-field + `in: query|header|body|cookie`, `llm_visible: false` for server-injected values, `{{ctx.*}}` + templating, `$each` loops, response projection) for free, an installed action can be opened + and edited in the Tool Builder like any hand-built tool, and there is no second request + format to keep in sync with the executor. + """ + + name: str + description: str = "" + display_name: str | None = None + request: dict[str, Any] + response: dict[str, Any] | None = None + # Optional passthroughs to the rest tool's reliability knobs. + timeout_seconds: int | None = None + retry: dict[str, Any] | None = None + rate_limit: dict[str, Any] | None = None + cache: dict[str, Any] | None = None + + @field_validator("name") + @classmethod + def _name_ok(cls, v: str) -> str: + if not _TOOL_NAME_RE.fullmatch(v or ""): + raise ValueError("action name must be 1-64 chars of [A-Za-z0-9_-]") + return v + + @model_validator(mode="after") + def _request_ok(self) -> RestAction: + if not self.request.get("url_template"): + raise ValueError(f"action {self.name!r}: request.url_template is required") + if not self.request.get("method"): + raise ValueError(f"action {self.name!r}: request.method is required") + for field in self.request.get("fields") or []: + if not isinstance(field, dict) or not field.get("path"): + raise ValueError(f"action {self.name!r}: every request field needs a `path`") + where = field.get("in", "query") + if where not in ("query", "header", "body", "cookie", "path"): + raise ValueError(f"action {self.name!r}: field {field['path']!r} has invalid `in`: {where!r}") + # `extract` pulls an id out of a pasted URL at call time. Compile it HERE so a typo in + # a manifest fails at install, not on someone's first tool call. + pattern = field.get("extract") + if pattern is not None: + try: + re.compile(pattern) + except re.error as e: + raise ValueError( + f"action {self.name!r}: field {field['path']!r} has an invalid `extract` regex: {e}" + ) from e + return self + + +class RestBackend(BaseModel): + type: Literal["rest"] = "rest" + base_url: str = "" + actions: list[RestAction] = Field(default_factory=list) + + @model_validator(mode="after") + def _has_actions(self) -> RestBackend: + if not self.actions: + raise ValueError("rest backend needs at least one action") + names = [a.name for a in self.actions] + dupes = {n for n in names if names.count(n) > 1} + if dupes: + raise ValueError(f"duplicate action names: {sorted(dupes)}") + return self + + +class ToolSetSpec(BaseModel): + slug: str | None = None + description: str = "" + + +class ConnectorManifest(BaseModel): + format: str = FORMAT + slug: str + name: str + version: str = "1.0.0" + publisher: Literal["forge", "community", "custom"] = "custom" + summary: str = "" + categories: list[str] = Field(default_factory=list) + icon: str | None = None + docs_url: str | None = None + setup_url: str | None = None + # Roles this connector is "popular for" - drives the suggestion row at the top of the + # Connectors screen. Free-form strings matched case-insensitively against the viewer's + # selected role. + roles: list[str] = Field(default_factory=list) + auth: AuthSpec = Field(default_factory=AuthSpec) + # Hosts this connector talks to. Appended to the project's egress allow-list on install so + # a deployment running a strict allow-list doesn't have to be edited by hand. Never used to + # widen anything else - the SSRF guard's default-deny stays in force. + egress_hosts: list[str] = Field(default_factory=list) + backend: McpBackend | RestBackend = Field(discriminator="type") + toolset: ToolSetSpec = Field(default_factory=ToolSetSpec) + + @field_validator("format") + @classmethod + def _format_ok(cls, v: str) -> str: + if v != FORMAT: + raise ValueError(f"unsupported manifest format {v!r} (expected {FORMAT!r})") + return v + + @field_validator("slug") + @classmethod + def _slug_ok(cls, v: str) -> str: + if not _SLUG_RE.fullmatch(v or ""): + raise ValueError("slug must be lowercase [a-z0-9_-], max 64 chars") + return v + + @model_validator(mode="after") + def _auth_backend_coherent(self) -> ConnectorManifest: + # A per-user credential is stored per end user and injected per call. The REST path + # supports that for every auth kind; the MCP path resolves the provider per (user, + # server) connection. Both are fine - what is NOT fine is declaring per_user on a + # connector with no auth at all, which would silently do nothing. + if self.auth.kind == "none" and self.auth.per_user == "required": + raise ValueError("per_user: required is meaningless with auth.kind: none") + return self + + # --- derived helpers ------------------------------------------------------------------ + + @property + def kind_label(self) -> str: + return "MCP" if self.backend.type == "mcp" else "REST" + + @property + def group(self) -> str: + """The credential namespace this connector's secrets live under.""" + return self.auth.credential_group or self.slug + + def hosts(self) -> list[str]: + """Every host this connector needs egress to: the declared list plus the hosts implied + by its own backend/token URLs, so a manifest can't forget to declare its own endpoint.""" + from urllib.parse import urlparse + + out = list(self.egress_hosts) + candidates = [self.auth.token_url, self.auth.authorize_url] + if isinstance(self.backend, McpBackend): + candidates.append(self.backend.url) + else: + candidates.append(self.backend.base_url or None) + for url in candidates: + if not url: + continue + host = urlparse(url).hostname + if host and host not in out: + out.append(host) + return out + + +def parse_manifest(data: dict[str, Any]) -> ConnectorManifest: + """Validate a manifest dict, raising ManifestError with a readable message. + + Pydantic's own error text is verbose and nested; a pasted-manifest form needs one clear + line per problem, so flatten it here rather than at every call site.""" + if not isinstance(data, dict): + raise ManifestError("manifest must be a JSON object") + try: + return ConnectorManifest.model_validate(data) + except Exception as e: # noqa: BLE001 - normalize pydantic/ValueError into one type + errors = getattr(e, "errors", None) + if callable(errors): + lines = [] + for err in e.errors(): # type: ignore[attr-defined] + loc = ".".join(str(p) for p in err.get("loc", ()) if p != "__root__") + lines.append(f"{loc or 'manifest'}: {err.get('msg', 'invalid')}") + raise ManifestError("; ".join(lines[:8])) from e + raise ManifestError(str(e)) from e diff --git a/apps/api/forge/connectors/mcp_auth.py b/apps/api/forge/connectors/mcp_auth.py new file mode 100644 index 0000000..ba72711 --- /dev/null +++ b/apps/api/forge/connectors/mcp_auth.py @@ -0,0 +1,168 @@ +"""Forge as an OAuth *client* of a remote MCP server (the MCP authorization spec, client half). + +`routers/mcp_oauth.py` already implements the SERVER half - Forge issuing tokens so Claude +Desktop / Cursor can call Forge's MCP endpoint. This module is its mirror image: Forge +discovering, registering with, and holding tokens for SOMEBODY ELSE'S MCP server, which is what +`mcp.slack.com`, the Google Workspace and Microsoft servers, Notion, Linear and Atlassian all +require. + +Flow, all of it SSRF-guarded: + + 1. GET /.well-known/oauth-protected-resource (RFC 9728) -> authorization_servers[] + 2. GET /.well-known/oauth-authorization-server (RFC 8414) -> authorize/token/register + 3. POST (RFC 7591) -> client_id [+ secret] + 4. authorization-code + PKCE via the existing /v1/oauth/callback + +Steps 1-3 are best-effort: a manifest that declares static `authorize_url`/`token_url` skips +discovery entirely, so a server with no metadata (or a locked-down network) still connects. That +fallback is why `auth.discover` is opt-in per manifest rather than always-on. +""" + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urljoin, urlparse + +from forge.config import settings +from forge.util.http import shared_async_client +from forge.util.ssrf import guarded_request + +log = logging.getLogger("forge.connectors.mcp_auth") + +# Discovery must not stall a connect click. These endpoints are small JSON documents on the +# vendor's own edge; a server that can't answer in this window is one we fall back for. +_DISCOVERY_TIMEOUT = 10.0 + + +class DiscoveryError(RuntimeError): + pass + + +async def _get_json(url: str, *, client=None) -> dict[str, Any] | None: + try: + r = await guarded_request( + client or shared_async_client(), "GET", url, + timeout=_DISCOVERY_TIMEOUT, follow_redirects=True, + headers={"Accept": "application/json", "MCP-Protocol-Version": "2026-07-28"}, + ) + except Exception as e: # noqa: BLE001 - unreachable/blocked/timeout are all "no metadata" + log.debug("oauth discovery: %s failed (%s)", url, e) + return None + if r.status_code >= 400: + return None + try: + body = r.json() + except Exception: # noqa: BLE001 - a non-JSON 200 is not metadata + return None + return body if isinstance(body, dict) else None + + +def _origin(url: str) -> str: + parts = urlparse(url) + return f"{parts.scheme}://{parts.netloc}" + + +async def discover(mcp_url: str, *, client=None) -> dict[str, Any]: + """Resolve a remote MCP server URL to its OAuth endpoints. + + Returns {authorize_url, token_url, registration_url, scopes_supported, issuer} with any + key absent when the server didn't advertise it. Never raises for a server that simply has + no metadata - the caller falls back to the manifest's static endpoints. + """ + out: dict[str, Any] = {} + origin = _origin(mcp_url) + path = urlparse(mcp_url).path.rstrip("/") + + # RFC 9728 permits both a path-scoped and a root-scoped resource document. Try the + # path-scoped one first: a host serving several MCP resources distinguishes them that way. + prm = None + for candidate in ( + f"{origin}/.well-known/oauth-protected-resource{path}" if path else None, + f"{origin}/.well-known/oauth-protected-resource", + ): + if not candidate: + continue + prm = await _get_json(candidate, client=client) + if prm: + break + + as_urls: list[str] = [] + if prm: + servers = prm.get("authorization_servers") or [] + as_urls = [s for s in servers if isinstance(s, str)] + if prm.get("scopes_supported"): + out["scopes_supported"] = prm["scopes_supported"] + if not as_urls: + # No protected-resource document: the MCP server's own origin is the conventional + # fallback issuer, which is what most single-tenant servers implement. + as_urls = [origin] + + # Endpoints are accumulated PER SERVER and only merged in once one server has yielded a + # complete pair. Accumulating straight into `out` would let an authorization server that + # advertises only an authorize endpoint be paired with a token endpoint from the NEXT + # server in the list - a config that looks valid and then fails the exchange, because the + # consent and the token call went to different issuers. + for as_url in as_urls: + as_origin = _origin(as_url) + as_path = urlparse(as_url).path.rstrip("/") + for candidate in ( + f"{as_origin}/.well-known/oauth-authorization-server{as_path}" if as_path else None, + f"{as_origin}/.well-known/oauth-authorization-server", + urljoin(as_origin + "/", ".well-known/openid-configuration"), + ): + if not candidate: + continue + meta = await _get_json(candidate, client=client) + if not meta: + continue + if not (meta.get("authorization_endpoint") and meta.get("token_endpoint")): + continue + out["authorize_url"] = meta["authorization_endpoint"] + out["token_url"] = meta["token_endpoint"] + if meta.get("registration_endpoint"): + out["registration_url"] = meta["registration_endpoint"] + if meta.get("issuer"): + out["issuer"] = meta["issuer"] + if meta.get("scopes_supported") and "scopes_supported" not in out: + out["scopes_supported"] = meta["scopes_supported"] + return out + return out + + +async def register_client( + registration_url: str, *, client_name: str, redirect_uri: str, scopes: list[str] | None = None, client=None +) -> dict[str, Any]: + """Dynamic Client Registration (RFC 7591) against the remote authorization server. + + Requests a CONFIDENTIAL client (`client_secret_post`) because Forge holds the secret + server-side in its own encrypted store - a public client would be the wrong choice here and + would force PKCE to carry the whole burden. Servers that only issue public clients simply + return no `client_secret`, which the token exchange handles. + """ + body = { + "client_name": client_name, + "redirect_uris": [redirect_uri], + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "token_endpoint_auth_method": "client_secret_post", + } + if scopes: + body["scope"] = " ".join(scopes) + r = await guarded_request( + client or shared_async_client(), "POST", registration_url, + json=body, timeout=_DISCOVERY_TIMEOUT, follow_redirects=True, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + if r.status_code >= 400: + raise DiscoveryError(f"client registration failed ({r.status_code}): {r.text[:300]}") + data = r.json() + if not data.get("client_id"): + raise DiscoveryError("client registration returned no client_id") + return data + + +def callback_url() -> str: + """The single redirect URI Forge registers everywhere - the same endpoint the auth-provider + OAuth connect flow already uses, so an operator only ever whitelists one URL per install.""" + return f"{settings.public_base_url.rstrip('/')}/v1/oauth/callback" diff --git a/apps/api/forge/main.py b/apps/api/forge/main.py index cac69a6..eb36b2f 100644 --- a/apps/api/forge/main.py +++ b/apps/api/forge/main.py @@ -29,6 +29,7 @@ channels, components, connections, + connectors, conversations, embed, embed_public, @@ -271,6 +272,7 @@ def create_app() -> FastAPI: knowledge.router, knowledge.qa_router, traces.router, conversations.router, assistant.router, stats.router, triggers_router.router, channels.router, channels.public, handoff.router, evals.router, pricing.router, mcp_oauth.router, mcp_server.router, mcp_tokens.router, mcp_clients.router, versions.router, + connectors.router, ): app.include_router(r) return app diff --git a/apps/api/forge/models/__init__.py b/apps/api/forge/models/__init__.py index 0071d1d..5e8b830 100644 --- a/apps/api/forge/models/__init__.py +++ b/apps/api/forge/models/__init__.py @@ -6,6 +6,7 @@ AuthProvider, Channel, Component, + ConnectorInstall, Dataset, EntityVersion, HandoffRequest, @@ -38,5 +39,5 @@ "Tenant", "User", "Project", "Workflow", "Thread", "Run", "Trace", "Span", "Tool", "ToolSet", "ToolSetMember", "AuthProvider", "Secret", "McpClient", "Agent", "KbSource", "QaPair", "AuditLog", "Trigger", "Channel", "Component", "HandoffRequest", "Dataset", "ModelPrice", "Memory", - "EntityVersion", "EvalRun", "EvalResult", "OAuthClient", + "EntityVersion", "EvalRun", "EvalResult", "OAuthClient", "ConnectorInstall", ] diff --git a/apps/api/forge/models/entities.py b/apps/api/forge/models/entities.py index da5b25a..f10bdea 100644 --- a/apps/api/forge/models/entities.py +++ b/apps/api/forge/models/entities.py @@ -236,6 +236,12 @@ class McpClient(PkTimestamp, Base): headers_ref: Mapped[str | None] = mapped_column(String(200), nullable=True) enabled: Mapped[bool] = mapped_column(Boolean, default=True) disabled_tools: Mapped[list] = mapped_column(JSON, default=list) # remote tool names toggled off in the External MCP tab + # An Auth Provider whose resolved headers are attached to every request to this server. + # This is what lets Forge talk to an OAuth-protected remote MCP server (mcp.slack.com, + # the Google Workspace / Microsoft servers, Notion, Linear, …) instead of only servers + # that accept a static header secret. Nullable: existing rows and open servers are + # unaffected, and `headers_ref` still works (both are merged, provider wins on conflict). + auth_provider_id: Mapped[str | None] = mapped_column(String(36), nullable=True) class Thread(PkTimestamp, Base): @@ -315,6 +321,29 @@ class Trigger(PkTimestamp, Base): enabled: Mapped[bool] = mapped_column(Boolean, default=True) last_fired_at: Mapped[datetime | None] = mapped_column(nullable=True) status: Mapped[str] = mapped_column(String(20), default="active") + # Two INDEPENDENT axes, deliberately - conflating them is what makes automation ownership + # confusing in most tools: + # + # run_as_user_id - WHOSE connected accounts the run uses. + # scope - WHO the trigger belongs to, and therefore who sees it. + # + # A salesperson's own lead-chaser is `user` scope running as them. A platform team's + # prod-monitor is `project` scope, still running as whoever's accounts it needs. The two vary + # separately: a shared team automation can legitimately act through one person's Slack. + # + # WHOSE connected accounts an unattended run uses. A webhook or a schedule has no signed-in + # person, so without this a per-user connector (every catalog connector is one) has no + # identity to resolve a token for and the run fails at the first tool call. Stamped with the + # editor who saved the workflow the trigger came from, and preserved when a colleague later + # edits that workflow - the automation belongs to whoever connected the account behind it, + # not to whoever last fixed a typo. Reassignable from the Triggers screen. + run_as_user_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + # "project" (a team automation everyone sees) | "user" (someone's own, listed only for them). + # Defaults to "project" so a pre-existing trigger keeps behaving exactly as it did; new ones + # take their default from the role of whoever saved the workflow (see TriggerService). + # NOTE: this is ownership and visibility, NOT access control on the event itself. A webhook + # URL is a credential - anyone holding it fires the trigger regardless of scope. + scope: Mapped[str] = mapped_column(String(10), default="project") # Runtime state (e.g. app_event dedupe cursor / seen ids); NOT synced from the node. meta: Mapped[dict] = mapped_column("metadata", JSON, default=dict) @@ -521,3 +550,44 @@ class UserSecurity(PkTimestamp, Base): email_verified_at: Mapped[datetime | None] = mapped_column(nullable=True) totp_secret: Mapped[str | None] = mapped_column(String(64), nullable=True) totp_enabled: Mapped[bool] = mapped_column(Boolean, default=False) + + +class ConnectorInstall(PkTimestamp, Base): + """One installed connector - the receipt for a manifest that was expanded into real rows. + + A connector is NOT a runtime concept: installing one creates an `AuthProvider`, a `ToolSet`, + N `Tool` rows (and an `McpClient` for MCP-backed connectors), and from then on agents, + workflow tool nodes, the MCP server and traces treat them like any other hand-built tool. + Delete `forge/connectors/` and every installed connector keeps working. + + This row exists so the install is REVERSIBLE and UPGRADABLE: it records exactly which rows + the install created (so uninstall removes those and nothing else) and freezes the manifest + that produced them (so a catalog update can be diffed against what's actually deployed + rather than assumed). + """ + + __tablename__ = "connector_installs" + __table_args__ = (UniqueConstraint("tenant_id", "project_id", "slug", name="uq_connector_install_slug"),) + tenant_id: Mapped[str] = mapped_column(String(36), index=True) + project_id: Mapped[str] = mapped_column(String(36), index=True) + slug: Mapped[str] = mapped_column(String(120), index=True) + name: Mapped[str] = mapped_column(String(120)) + version: Mapped[str] = mapped_column(String(40), default="1.0.0") + source: Mapped[str] = mapped_column(String(20), default="catalog") # catalog | custom | url + # Frozen copy of the manifest that produced this install. Survives catalog edits/upgrades, + # so uninstall and "what changed?" stay correct even if the bundled file moved on. + manifest: Mapped[dict] = mapped_column(JSON, default=dict) + auth_provider_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + tool_set_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + mcp_client_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + created_tool_ids: Mapped[list] = mapped_column(JSON, default=list) + created_secret_names: Mapped[list] = mapped_column(JSON, default=list) + # Hosts this install ADDED to the project's egress allow-list - not every host its manifest + # names. A host somebody allow-listed by hand before the connector existed is not this + # install's to take away on uninstall, and the two are indistinguishable after the fact. + created_egress_hosts: Mapped[list] = mapped_column(JSON, default=list) + # needs_setup (credentials missing) | needs_auth (connect flow not run) | connected | error + status: Mapped[str] = mapped_column(String(20), default="needs_setup") + status_detail: Mapped[str | None] = mapped_column(Text, nullable=True) + # "shared" (one project-wide account) | "per_user" (each end user connects their own). + auth_mode: Mapped[str] = mapped_column(String(20), default="shared") diff --git a/apps/api/forge/routers/connectors.py b/apps/api/forge/routers/connectors.py new file mode 100644 index 0000000..bdfb4c8 --- /dev/null +++ b/apps/api/forge/routers/connectors.py @@ -0,0 +1,779 @@ +"""Connectors - browse the catalog, connect your own account, and manage what's installed. + +Three ways in, deliberately: + + * catalog - `POST /{slug}/connect` is the whole flow: it expands the bundled manifest if the + project doesn't have it yet, then hands back the vendor's sign-in URL. + * custom - `POST /custom` installs a pasted/uploaded manifest, so a private connector pack + can live in a company's own git repo instead of a fork of Forge. + * raw MCP - the existing /mcp-clients routes, unchanged. + +Two rules run through every route here: + +CREDENTIALS COME FROM THE DEPLOYMENT. A catalog connector's vendor OAuth app is read from +FORGE_CONNECTOR_OAUTH_APPS and nowhere else - no route on this router accepts one for a catalog +install, and a connector whose group isn't configured reports itself unavailable, naming the env +key, rather than degrading into a form that asks an end user for a client secret. + +ACCOUNTS ARE PERSONAL. Adding a connector creates project-level rows (tools a workflow is built +on), so it stays editor-gated. Connecting is not: each person signs in as themselves, their token +is stored under their own identity, and every status this router reports is answered for the +CALLER - a colleague's mailbox never becomes yours by virtue of sharing a project. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from forge.auth_providers.oauth_flow import ( + OAuthNotConfigured, + build_authorize_url, +) +from forge.auth_providers.oauth_flow import ( + redirect_uri as oauth_redirect_uri, +) +from forge.auth_providers.resolver import AuthResolver +from forge.connectors import catalog as catalog_mod +from forge.connectors.install import ( + ENV_VAR, + ConnectorInstaller, + InstallError, + env_ready, + missing_app_keys, + not_configured_message, + secret_name, +) +from forge.connectors.manifest import ConnectorManifest, ManifestError, McpBackend, parse_manifest +from forge.deps import ( + CurrentUser, + current_tenant_id, + effective_role, + get_current_user, + get_session, + require_role, + role_at_least, +) +from forge.models import AuthProvider, ConnectorInstall, Tool +from forge.secrets.store import SecretNotFound, SecretStore +from forge.services.auth_providers import AuthProviderService +from forge.util.locks import KeyedLocks + +log = logging.getLogger("forge.connectors") + +router = APIRouter(prefix="/v1/projects/{project_id}/connectors", tags=["connectors"]) + + +# --- payloads ------------------------------------------------------------------------------ + +class InstallIn(BaseModel): + # Credential values keyed by the manifest's setup[].key, for the CUSTOM path only - a + # catalog install ignores both fields (its app comes from the environment and its accounts + # are always personal). Secret values are written to the SecretStore immediately and never + # echoed back by any route on this router. + values: dict[str, str] = Field(default_factory=dict) + auth_mode: str = "shared" # shared | per_user (ignored when the manifest forces one) + + +class CustomInstallIn(InstallIn): + manifest: dict[str, Any] + + +class ConnectIn(BaseModel): + # For a per-user connector an editor may connect ON BEHALF OF a specific end user id; + # omitted means "connect as me", which is what the self-service flow sends. + end_user_id: str | None = None + + +class CredentialsIn(BaseModel): + values: dict[str, str] = Field(default_factory=dict) + + +# --- serialization ------------------------------------------------------------------------- + +def _manifest_out(m: ConnectorManifest, installed: ConnectorInstall | None = None, + *, managed: bool = True, group_ready: bool = False) -> dict: + """Catalog-card shape. Never includes credential values. + + `managed` (every catalog entry) means the vendor app comes from the deployment environment + and there is nothing for anyone to type: the card shows a Connect button, or - when the + operator hasn't registered that vendor yet - says exactly which env keys are missing. + Unmanaged is the custom-manifest path, which still renders `auth.setup` as a form. + """ + ready = env_ready(m) if managed else group_ready + return { + "credential_group": m.group, + "managed": managed, + # Ready to connect right now, with nothing asked of the person clicking. + "configured": ready, + "missing_keys": missing_app_keys(m) if managed else [], + "config_env_key": ENV_VAR, + # The EXACT string sent to the vendor as redirect_uri, so what an operator whitelists is + # what the flow uses. It is derived from FORGE_PUBLIC_BASE_URL (this API's public origin), + # which is not the console's origin - guessing it from the browser produces a URL that + # looks right and fails with redirect_uri_mismatch. + "redirect_uri": oauth_redirect_uri({}), + "slug": m.slug, + "name": m.name, + "version": m.version, + "publisher": m.publisher, + "summary": m.summary, + "categories": m.categories, + "roles": m.roles, + "icon": m.icon, + "docs_url": m.docs_url, + "setup_url": m.setup_url, + "type": m.kind_label, # "MCP" | "REST" + "auth": { + "kind": m.auth.kind, + "per_user": m.auth.per_user, + "scopes": m.auth.scopes, + "setup_help": m.auth.setup_help, + "setup": [f.model_dump() for f in m.auth.setup], + }, + "action_count": ( + len(m.backend.actions) if not isinstance(m.backend, McpBackend) else None + ), + "installed": installed is not None, + "status": installed.status if installed else None, + "install_id": installed.id if installed else None, + } + + +async def _live_tool_counts(session: AsyncSession, tenant_id: str, project_id: str, + rows: list[ConnectorInstall]) -> dict[str, int]: + """How many of each install's actions STILL EXIST, keyed by install id. + + A user may have deleted an action from the Tools screen; reporting the install's original + count would then be a lie. Every route that reports a count uses this one - when only the + list route did, deleting two of Gmail's five actions left the gallery correctly showing 3 + while the detail panel (which polls `/{slug}/status`) insisted on 5. + """ + ids = [tid for r in rows for tid in (r.created_tool_ids or [])] + if not ids: + return {r.id: 0 for r in rows} + found = await session.execute( + select(Tool.id).where(Tool.tenant_id == tenant_id, Tool.project_id == project_id, Tool.id.in_(ids)) + ) + alive = {r[0] for r in found.all()} + return {r.id: len([t for t in (r.created_tool_ids or []) if t in alive]) for r in rows} + + +def _install_out(row: ConnectorInstall, *, tool_count: int) -> dict: + manifest = row.manifest or {} + backend = manifest.get("backend") or {} + return { + "id": row.id, + "slug": row.slug, + "name": row.name, + "version": row.version, + "source": row.source, + "status": row.status, + "status_detail": row.status_detail, + "auth_mode": row.auth_mode, + "auth_kind": (manifest.get("auth") or {}).get("kind", "none"), + "type": "MCP" if backend.get("type") == "mcp" else "REST", + "icon": manifest.get("icon"), + "summary": manifest.get("summary", ""), + "docs_url": manifest.get("docs_url"), + "tool_set_id": row.tool_set_id, + "auth_provider_id": row.auth_provider_id, + "mcp_client_id": row.mcp_client_id, + # Required, deliberately: the old default counted `created_tool_ids`, so a caller that + # forgot to pass one silently reported actions the user had deleted. + "tool_count": tool_count, + "needs_connect": row.status == "needs_auth", + } + + +async def _load_install(session: AsyncSession, tenant_id: str, project_id: str, slug: str) -> ConnectorInstall: + row = await ConnectorInstaller.get_install(session, tenant_id, project_id, slug) + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "connector is not installed in this project") + return row + + +async def _target_end_user(request: Request, user: CurrentUser, requested: str | None) -> str: + """Which end user a per-user connect/disconnect acts on. + + Acting on YOURSELF is open to any real logged-in user (that is the point of self-service + per-user credentials). Acting on someone ELSE is an editor action, checked here rather than + via Depends because it is conditional on the request body.""" + if str(user.id).startswith(("apikey:", "service")): + raise HTTPException(status.HTTP_403_FORBIDDEN, "per-user credentials require a user identity") + target = requested or str(user.id) + if target != str(user.id) and not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException(status.HTTP_403_FORBIDDEN, "connecting on behalf of another user requires role 'editor'") + return target + + +# --- catalog ------------------------------------------------------------------------------- + +@router.get("/catalog") +async def list_catalog(project_id: str, session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id)): + """The bundled catalog, each entry flagged with whether this project already has it. + + Reads a directory of JSON files - no network call, no third-party service, no API key. This + route works identically on an air-gapped install. Whether an entry is CONNECTABLE is read + from the deployment's environment, which is also free: no per-connector secret lookups just + to paint a page.""" + installs = {i.slug: i for i in await ConnectorInstaller.list_installs(session, tenant_id, project_id)} + return { + "connectors": [_manifest_out(m, installs.get(m.slug)) for m in catalog_mod.list_manifests()], + "categories": catalog_mod.categories(), + "roles": catalog_mod.roles(), + } + + +@router.get("/catalog/{slug}") +async def get_catalog_entry(project_id: str, slug: str, session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id)): + manifest = catalog_mod.get_manifest(slug) + if manifest is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "connector not found in the catalog") + install = await ConnectorInstaller.get_install(session, tenant_id, project_id, slug) + out = _manifest_out(manifest, install) + if not isinstance(manifest.backend, McpBackend): + out["actions"] = [ + {"name": a.name, "description": a.description, + "method": (a.request or {}).get("method", "GET")} + for a in manifest.backend.actions + ] + else: + out["mcp_url"] = manifest.backend.url + return out + + +# --- installed ----------------------------------------------------------------------------- + +@router.get("") +async def list_installed(project_id: str, session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): + rows = await ConnectorInstaller.list_installs(session, tenant_id, project_id) + if not rows: + return [] + counts = await _live_tool_counts(session, tenant_id, project_id, rows) + # One query for every provider rather than one per install: painting this screen for a + # project with the full catalog installed was a dozen sequential SELECTs before the secret + # reads even started. + provider_ids = [r.auth_provider_id for r in rows if r.auth_provider_id] + providers: dict[str, AuthProvider] = {} + if provider_ids: + found_aps = await session.execute( + select(AuthProvider).where( + AuthProvider.tenant_id == tenant_id, AuthProvider.id.in_(provider_ids) + ) + ) + providers = {ap.id: ap for ap in found_aps.scalars()} + # ...and one read for every token bundle, for the same reason. + bundles = await _bundles_for(tenant_id, project_id, rows, providers, str(user.id)) + + out = [] + for r in rows: + item = _install_out(r, tool_count=counts[r.id]) + # Per-user connectors have no single "connected" answer, so the list resolves it for the + # CALLER. Without this the gallery would show a green tick to everyone the moment one + # colleague connected their own account, and the person clicking Connect would be told + # they were already done. + item["connected"] = await _connected_for( + session, tenant_id, project_id, r, str(user.id), + provider=providers.get(r.auth_provider_id or ""), bundles=bundles, + ) + out.append(item) + return out + + +#: Distinguishes "the bundle secret does not exist" from "it exists and is unusable". Only the +#: first means a key/bearer connector is still connected via its own credential. +_MISSING = object() + + +def _bundle_name_for(row: ConnectorInstall, ap: AuthProvider, user_id: str) -> str: + """The secret holding the OAuth bundle that decides whether this connector is usable by + `user_id` right now - the caller's own for a per-user connector, the project's otherwise.""" + if row.auth_mode == "per_user": + return AuthResolver.bundle_secret_name(ap.id, {"end_user_id": user_id}, ["end_user_id"]) + return AuthResolver.bundle_secret_name(ap.id) + + +async def _connection_state(tenant_id: str, project_id: str, row: ConnectorInstall, + ap: AuthProvider, user_id: str, *, bundle: Any = _MISSING, + prefetched: bool = False) -> dict: + """`{connected, expires_at}` for one caller against one connector's auth provider. + + The single source of truth for "is this usable right now", shared by the list route and the + per-connector status route. The two answer for different scopes but must agree on the rule - + when they drifted, the gallery showed a green tick to someone the detail panel then asked to + sign in. + + `prefetched` says the caller already read the bundle (see `_bundles_for`), so this does no + I/O at all; `bundle` is then the value, or `_MISSING` if that secret does not exist. + """ + if not prefetched: + try: + bundle = await SecretStore().read_ref( + tenant_id=tenant_id, project_id=project_id, + ref=f"secret://proj/{_bundle_name_for(row, ap, user_id)}", + ) + except SecretNotFound: + bundle = _MISSING + except Exception: # noqa: BLE001 - an undecodable bundle is not a usable connection + bundle = None + if row.auth_mode == "per_user": + # A per-user connector has no project-wide answer: a colleague having connected their + # own mailbox says nothing about yours. And no bundle means not connected, full stop - + # there is no per-user equivalent of a shared key sitting in the provider config. + connected = isinstance(bundle, dict) and bool(bundle.get("access_token")) + return {"connected": connected, "expires_at": bundle.get("expires_at") if connected else None} + if bundle is _MISSING: + # Only OAuth stores a token bundle; for a key/bearer connector the credential itself is + # the connection, so an absent bundle is not "disconnected". + return {"connected": ap.kind != "oauth2_authorization_code", "expires_at": None} + if not isinstance(bundle, dict): + return {"connected": False, "expires_at": None} + return {"connected": bool(bundle.get("access_token")), "expires_at": bundle.get("expires_at")} + + +async def _bundles_for(tenant_id: str, project_id: str, rows: list[ConnectorInstall], + providers: dict[str, AuthProvider], user_id: str) -> dict[str, Any]: + """Every token bundle the installed list needs, in one read. + + A project with the full catalog installed painted this screen with a dozen SEQUENTIAL secret + reads - a round trip, a decrypt and an audit write each - and the connect flow calls reload() + on window focus, so it repeated every time someone came back from a consent window. + """ + names = [ + _bundle_name_for(r, ap, user_id) + for r in rows + if (ap := providers.get(r.auth_provider_id or "")) is not None + ] + if not names: + return {} + return await SecretStore().read_refs( + tenant_id=tenant_id, project_id=project_id, + refs=[f"secret://proj/{n}" for n in names], + ) + + +async def _connected_for(session: AsyncSession, tenant_id: str, project_id: str, + row: ConnectorInstall, user_id: str, + *, provider: AuthProvider | None = None, + bundles: dict[str, Any] | None = None) -> bool: + """Whether THIS user can currently act through this connector. + + `provider` and `bundles` let a caller that already batched those reads pass them in; without + them this reads both itself, which is what the single-connector status route wants.""" + if not row.auth_provider_id: + return True + ap = provider or await AuthProviderService.get(session, tenant_id, row.auth_provider_id) + if ap is None: + return False + if bundles is None: + state = await _connection_state(tenant_id, project_id, row, ap, user_id) + else: + name = _bundle_name_for(row, ap, user_id) + state = await _connection_state( + tenant_id, project_id, row, ap, user_id, + bundle=bundles.get(name, _MISSING), prefetched=True, + ) + return bool(state.get("connected")) + + +@router.post("/{slug}/install", status_code=201) +async def install_connector(project_id: str, slug: str, body: InstallIn | None = None, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + _: CurrentUser = Depends(require_role("editor"))): + """Add a catalog connector without connecting to it yet. + + `POST /{slug}/connect` does this implicitly, which is the path the UI takes. This route is + what you want when adding the connector and signing in are done by different people - an + editor wires up the tools, and everyone else connects their own account afterwards. + """ + manifest = catalog_mod.get_manifest(slug) + if manifest is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "connector not found in the catalog") + try: + row = await ConnectorInstaller().install( + session, tenant_id, project_id, manifest, source="catalog", + ) + except InstallError as e: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e)) from e + # Just created: nothing has had a chance to be deleted, so the receipt IS the live count. + return _install_out(row, tool_count=len(row.created_tool_ids or [])) + + +@router.post("/custom", status_code=201) +async def install_custom(project_id: str, body: CustomInstallIn, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + _: CurrentUser = Depends(require_role("editor"))): + """Install a manifest the user supplied. Same validator, same install path, same rows - + a custom connector is not a second-class citizen, it just isn't in the bundled catalog.""" + try: + manifest = parse_manifest(body.manifest) + except ManifestError as e: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, f"Invalid manifest - {e}") from e + try: + row = await ConnectorInstaller().install( + session, tenant_id, project_id, manifest, + values=body.values, auth_mode=body.auth_mode, source="custom", + ) + except InstallError as e: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e)) from e + return _install_out(row, tool_count=len(row.created_tool_ids or [])) + + +@router.get("/examples") +async def list_examples(project_id: str, _: CurrentUser = Depends(require_role("editor"))): + """Ready-made manifests for services that CAN'T be one-click: an API key, a bot token or a + per-tenant subdomain has to come from somewhere, and that somewhere is a person typing it. + + They are offered as starting points for the custom-connector form - open one, adjust it, add + your key - rather than sitting in the gallery pretending to be one click away. + """ + return [ + { + "slug": m.slug, "name": m.name, "summary": m.summary, "icon": m.icon, + "type": m.kind_label, "auth_kind": m.auth.kind, + "needs": [f.label for f in m.auth.setup if f.required], + # The file as authored, so what lands in the form is editable and re-installable. + "manifest": source, + } + for m, source in catalog_mod.list_examples() + ] + + +@router.post("/validate") +async def validate_manifest(project_id: str, body: dict, + _: CurrentUser = Depends(require_role("editor"))): + """Dry-run a manifest so the paste-a-manifest form can show errors before anything is + created. Creates nothing and touches no credentials.""" + try: + manifest = parse_manifest(body.get("manifest") if "manifest" in body else body) + except ManifestError as e: + return {"ok": False, "error": str(e)} + return {"ok": True, "connector": _manifest_out(manifest, managed=False), "hosts": manifest.hosts()} + + +@router.delete("/{slug}", status_code=204) +async def uninstall_connector(project_id: str, slug: str, session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + _: CurrentUser = Depends(require_role("editor"))): + row = await _load_install(session, tenant_id, project_id, slug) + await ConnectorInstaller().uninstall(session, row) + + +# --- credentials + connect ----------------------------------------------------------------- + +@router.put("/{slug}/credentials", status_code=204) +async def set_credentials(project_id: str, slug: str, body: CredentialsIn, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + _: CurrentUser = Depends(require_role("editor"))): + """Update the connector's stored setup values (rotate a client secret, fix a typo'd key). + + Only keys the manifest declares are accepted; an unknown key is ignored rather than written, + so this route can never be used to plant an arbitrary secret under a connector's namespace. + """ + row = await _load_install(session, tenant_id, project_id, slug) + if row.source == "catalog": + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"{row.name}'s credentials belong to the deployment, not the project - rotate the " + f"{ENV_VAR} entry and restart the API. Everyone's own sign-in survives the rotation.", + ) + manifest = parse_manifest(row.manifest or {}) + declared = {f.key: f for f in manifest.auth.setup} + store = SecretStore() + written = list(row.created_secret_names or []) + for key, value in body.values.items(): + field = declared.get(key) + if field is None or not field.secret or not value: + continue + name = secret_name(manifest.group, key) + await store.write(session, tenant_id=tenant_id, project_id=project_id, + name=name, value=value, kind="connector") + if name not in written: + written.append(name) + row.created_secret_names = written + if row.status == "needs_setup": + row.status = "needs_auth" if manifest.auth.kind == "oauth2_authorization_code" else "connected" + await session.commit() + + +@router.post("/{slug}/connect") +async def connect_connector(request: Request, project_id: str, slug: str, body: ConnectIn | None = None, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): + """Begin the OAuth consent for this connector and return the URL to open. + + This is the WHOLE flow from the user's side: one call, one browser round trip, connected. + If the project doesn't have the connector yet it is installed here rather than being a + separate step the person has to know about - "install" is Forge's internal bookkeeping + (create an auth provider, a tool set, a tool per action), not a decision anyone came here + to make. + + For an MCP-backed connector whose manifest opted into discovery, the server's OAuth metadata + is fetched and a client is registered dynamically FIRST - that is what makes a one-click + connect possible against servers that support it, with the static manifest endpoints as the + fallback for those that don't. + """ + row = await ConnectorInstaller.get_install(session, tenant_id, project_id, slug) + if row is None: + row = await _install_on_demand(request, session, tenant_id, project_id, slug, user) + if not row.auth_provider_id: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "this connector needs no connection") + ap = await AuthProviderService.get(session, tenant_id, row.auth_provider_id) + if ap is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "auth provider missing for this connector") + + per_user = bool((ap.config or {}).get("per_user_context_keys")) + if per_user: + target = await _target_end_user(request, user, body.end_user_id if body else None) + context = {"end_user_id": target} + else: + # A SHARED credential is the project's, not the caller's - setting it up is an editor + # action even though connecting your own per-user account is not. + if not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException(status.HTTP_403_FORBIDDEN, "requires role 'editor' or higher") + context = {} + + await _ensure_oauth_endpoints(session, tenant_id, project_id, row, ap) + try: + url = await build_authorize_url(ap, tenant_id=tenant_id, project_id=project_id, context=context) + except OAuthNotConfigured as e: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e)) from e + return {"authorize_url": url, "per_user": per_user} + + +async def _install_on_demand(request: Request, session: AsyncSession, tenant_id: str, project_id: str, + slug: str, user: CurrentUser) -> ConnectorInstall: + """Add a catalog connector to the project as part of connecting to it. + + Adding a connector creates project-level rows (tools a workflow can be built on), so it stays + an editor action. A viewer who is first through the door gets told what to ask for rather + than a 404 - and once an editor has added it, every other person's connect is just their own + sign-in. + """ + manifest = catalog_mod.get_manifest(slug) + if manifest is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "connector is not installed in this project") + if not env_ready(manifest): + raise HTTPException(status.HTTP_400_BAD_REQUEST, not_configured_message(manifest)) + if not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + f"{manifest.name} hasn't been added to this project yet. An editor adds it once, then " + "everyone connects their own account.", + ) + try: + return await ConnectorInstaller().install( + session, tenant_id, project_id, manifest, source="catalog", + ) + except InstallError as e: + # Two people can click Connect on the same connector at the same moment; the loser's + # install is rejected as a duplicate. That is the right outcome for the ROW and the wrong + # one for the person - the connector they asked for now exists, so join it and carry on + # to their sign-in rather than telling them it is "already installed". + existing = await ConnectorInstaller.get_install(session, tenant_id, project_id, slug) + if existing is not None: + return existing + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e)) from e + + +#: Serializes the once-per-install discovery + dynamic client registration. Two people clicking +#: Connect on the same connector in the same second would otherwise both register a client and +#: both write the credential: last writer wins, and the first person's browser is already sitting +#: on an authorize URL carrying a client_id whose secret has just been replaced, so their callback +#: fails at the token exchange with nothing to explain it. +#: +#: IN-PROCESS ONLY, like the OAuth refresh locks in auth_providers/resolver.py. Across scaled +#: `api` replicas the two connects can still collide; the stored-credential re-check below is +#: what keeps that case converging on one registered client rather than flip-flopping. +_discovery_locks = KeyedLocks() + + +async def _ensure_oauth_endpoints(session: AsyncSession, tenant_id: str, project_id: str, + row: ConnectorInstall, ap: AuthProvider) -> None: + """Fill in authorize/token endpoints (and a client registration) for a discovery connector. + + Runs at most once per install: the discovered values are written back onto the provider + config, so the second connect is a plain authorize-URL build with no extra round trips. + """ + if not (ap.config or {}).get("oauth_discover"): + return + lock = await _discovery_locks.acquire_cm(f"{tenant_id}:{project_id}:{ap.id}") + async with lock: + # Re-read INSIDE the lock: a peer may have completed the whole dance while we waited, + # in which case there is nothing left to do and re-registering would replace their app. + await session.refresh(ap) + await _discover_and_register(session, tenant_id, project_id, row, ap) + + +async def _discover_and_register(session: AsyncSession, tenant_id: str, project_id: str, + row: ConnectorInstall, ap: AuthProvider) -> None: + cfg = dict(ap.config or {}) + if cfg.get("authorize_url") and cfg.get("token_url"): + return + manifest = parse_manifest(row.manifest or {}) + if not isinstance(manifest.backend, McpBackend): + return + + from forge.connectors import mcp_auth + + found = await mcp_auth.discover(manifest.backend.url) + if found.get("authorize_url"): + cfg["authorize_url"] = found["authorize_url"] + if found.get("token_url"): + cfg["token_url"] = found["token_url"] + if not cfg.get("authorize_url") or not cfg.get("token_url"): + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"{manifest.name} did not publish OAuth metadata. Add the authorize/token URLs to the " + "connector's Auth Provider, or register an app with the vendor and paste its credentials.", + ) + # RFC 8707 - bind issued tokens to this specific MCP server. + cfg["resource"] = manifest.backend.url + + store = SecretStore() + have_client = False + if cfg.get("client_id_ref"): + try: + have_client = bool(await store.read_ref(tenant_id=tenant_id, project_id=project_id, ref=cfg["client_id_ref"])) + except SecretNotFound: + have_client = False + if not have_client and found.get("registration_url"): + try: + reg = await mcp_auth.register_client( + found["registration_url"], client_name="Forge", + redirect_uri=mcp_auth.callback_url(), + scopes=manifest.auth.scopes or found.get("scopes_supported") or [], + ) + except Exception as e: # noqa: BLE001 - fall through to "paste your own app credentials" + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"Could not register with {manifest.name} automatically ({e}). Create an app with the " + "vendor and paste its Client ID/Secret in the connector's settings.", + ) from e + # Last check before writing. The in-process lock can't see a peer on ANOTHER replica + # that registered while we were talking to the vendor; if one did, keep theirs and + # discard the client we just registered. Both sides then converge on a single stored + # app instead of overwriting each other and stranding whoever is mid-consent. + try: + if await store.read_ref(tenant_id=tenant_id, project_id=project_id, + ref=f"secret://proj/{secret_name(manifest.group, 'client_id')}"): + log.info("connector %s: another registration won the race; keeping it", manifest.slug) + reg = {} + except SecretNotFound: + pass + names = list(row.created_secret_names or []) + for key, value in (("client_id", reg.get("client_id")), ("client_secret", reg.get("client_secret"))): + if not value: + continue + name = secret_name(manifest.group, key) + await store.write(session, tenant_id=tenant_id, project_id=project_id, + name=name, value=value, kind="connector") + if name not in names: + names.append(name) + row.created_secret_names = names + cfg["client_id_ref"] = f"secret://proj/{secret_name(manifest.group, 'client_id')}" + cfg["client_secret_ref"] = f"secret://proj/{secret_name(manifest.group, 'client_secret')}" + if not cfg.get("scope") and (manifest.auth.scopes or found.get("scopes_supported")): + cfg["scope"] = " ".join(manifest.auth.scopes or found.get("scopes_supported") or []) + ap.config = cfg + await session.commit() + + +@router.get("/{slug}/status") +async def connector_status(project_id: str, slug: str, session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): + """Whether this connector is usable right now - and for a per-user connector, whether the + CALLER personally has connected it (which is the only status that matters to them).""" + row = await _load_install(session, tenant_id, project_id, slug) + counts = await _live_tool_counts(session, tenant_id, project_id, [row]) + out = _install_out(row, tool_count=counts[row.id]) + if not row.auth_provider_id: + out["connected"] = True + return out + ap = await AuthProviderService.get(session, tenant_id, row.auth_provider_id) + if ap is None: + out["connected"] = False + return out + state = await _connection_state(tenant_id, project_id, row, ap, str(user.id)) + out["connected"] = bool(state.get("connected")) + out["expires_at"] = state.get("expires_at") + return out + + +@router.post("/{slug}/sync") +async def sync_connector(request: Request, project_id: str, slug: str, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): + """Bring a connector's actions back in line with its manifest. + + Two different operations behind one button, with two different gates: + + * MCP - ask the server what it exposes, using the CALLER's credential because that is the + only one a per-user connector has. Open to anyone who can connect: gating it would leave + whoever actually signed in staring at zero actions, and the rows created are entirely + determined by what the vendor advertises for a connector an editor already added. + * REST - re-apply the bundled manifest over existing Tool rows. That is a project-level + write (it rewrites tool configs everyone's workflows run), so it stays editor-gated even + though the values are deterministic. + """ + row = await _load_install(session, tenant_id, project_id, slug) + # `or {}` on the inner get too: a stored manifest with an explicit null backend would make + # `.get("backend", {})` return None and the next .get() raise, 500ing instead of falling + # through to the safe editor-gated branch. + if ((row.manifest or {}).get("backend") or {}).get("type") != "mcp": + if not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "refreshing this connector's actions rewrites project tools - requires role 'editor'", + ) + context = {"end_user_id": str(user.id)} if row.auth_mode == "per_user" else None + try: + count = await ConnectorInstaller().sync_tools(session, row, context=context) + except Exception as e: # noqa: BLE001 - report, don't 500: the usual cause is "not connected yet" + from forge.tools.mcp import describe_mcp_error + + return {"ok": False, "error": describe_mcp_error(e)} + return {"ok": True, "tool_count": count} + + +@router.post("/{slug}/disconnect", status_code=204) +async def disconnect_connector(request: Request, project_id: str, slug: str, body: ConnectIn | None = None, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): + """Revoke the stored token WITHOUT uninstalling: the tools stay wired into workflows and + agents, they just stop being able to act until reconnected. Uninstall is the destructive one.""" + row = await _load_install(session, tenant_id, project_id, slug) + if not row.auth_provider_id: + return + ap = await AuthProviderService.get(session, tenant_id, row.auth_provider_id) + if ap is None: + return + if row.auth_mode == "per_user": + target = await _target_end_user(request, user, body.end_user_id if body else None) + await AuthProviderService.clear_user_connection(session, tenant_id, project_id, ap, target) + return + if not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException(status.HTTP_403_FORBIDDEN, "requires role 'editor' or higher") + await SecretStore().write( + session, tenant_id=tenant_id, project_id=project_id, + name=AuthResolver.bundle_secret_name(ap.id), value={}, kind="oauth", + ) + row.status = "needs_auth" + await session.commit() diff --git a/apps/api/forge/routers/mcp_clients.py b/apps/api/forge/routers/mcp_clients.py index 8cd97b7..ea14fdb 100644 --- a/apps/api/forge/routers/mcp_clients.py +++ b/apps/api/forge/routers/mcp_clients.py @@ -7,7 +7,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from forge.deps import CurrentUser, current_tenant_id, get_session, require_role +from forge.deps import CurrentUser, current_tenant_id, get_current_user, get_session, require_role from forge.models import McpClient router = APIRouter(prefix="/v1/projects/{project_id}/mcp-clients", tags=["mcp-clients"]) @@ -21,6 +21,9 @@ class McpClientIn(BaseModel): args: dict = {} headers_ref: str | None = None enabled: bool = True + # Attach an Auth Provider instead of (or as well as) a static header secret - this is how a + # server behind OAuth is reached, with refresh and optional per-user credentials. + auth_provider_id: str | None = None class McpClientPatch(BaseModel): @@ -29,21 +32,40 @@ class McpClientPatch(BaseModel): disabled_tools: list | None = None # remote tool names toggled off url: str | None = None headers_ref: str | None = None + auth_provider_id: str | None = None def _out(m: McpClient) -> dict: return {"id": m.id, "name": m.name, "transport": m.transport, "url": m.url, "command": m.command, "args": m.args, "headers_ref": m.headers_ref, - "enabled": m.enabled, "disabled_tools": m.disabled_tools or []} + "enabled": m.enabled, "disabled_tools": m.disabled_tools or [], + "auth_provider_id": m.auth_provider_id} @router.get("") async def list_clients(project_id: str, session: AsyncSession = Depends(get_session), tenant_id: str = Depends(current_tenant_id)): - rows = (await session.execute( + """Registered MCP servers, each flagged with the connector that owns it (if any). + + A connector-owned server already surfaces its tools as a tool set, so anything picking tools + for an agent should offer the SET and skip the server - listing both makes one integration + look like two, and granting either does the same thing. + """ + from forge.models import ConnectorInstall + + rows = list((await session.execute( select(McpClient).where(McpClient.tenant_id == tenant_id, McpClient.project_id == project_id) - )).scalars() - return [_out(m) for m in rows] + )).scalars()) + owned = { + i.mcp_client_id: i.slug + for i in (await session.execute( + select(ConnectorInstall).where( + ConnectorInstall.tenant_id == tenant_id, ConnectorInstall.project_id == project_id, + ) + )).scalars() + if i.mcp_client_id + } + return [{**_out(m), "connector_slug": owned.get(m.id)} for m in rows] @router.post("", status_code=201) @@ -51,7 +73,8 @@ async def create_client(project_id: str, body: McpClientIn, session: AsyncSessio tenant_id: str = Depends(current_tenant_id), _: CurrentUser = Depends(require_role("editor"))): m = McpClient(tenant_id=tenant_id, project_id=project_id, name=body.name, transport=body.transport, - url=body.url, command=body.command, args=body.args, headers_ref=body.headers_ref, enabled=body.enabled) + url=body.url, command=body.command, args=body.args, headers_ref=body.headers_ref, + enabled=body.enabled, auth_provider_id=body.auth_provider_id) session.add(m) await session.commit() await session.refresh(m) @@ -77,31 +100,38 @@ async def update_client(project_id: str, client_id: str, body: McpClientPatch, s m.url = body.url if body.headers_ref is not None: m.headers_ref = body.headers_ref + if body.auth_provider_id is not None: + # "" clears the attachment (back to anonymous / headers_ref only). + m.auth_provider_id = body.auth_provider_id or None await session.commit() await session.refresh(m) # Drop the cached connection so running agents pick up the new config (audit F12). from forge.tools.mcp import invalidate_client - invalidate_client(client_id) + await invalidate_client(client_id) return _out(m) @router.get("/{client_id}/tools") async def list_remote_tools(project_id: str, client_id: str, session: AsyncSession = Depends(get_session), - tenant_id: str = Depends(current_tenant_id)): + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user)): """Connect to the server and list the tools it exposes - drives the 'pick which to add' UI.""" - from forge.tools.mcp import McpUnavailable, discover_tools + from forge.tools.mcp import McpUnavailable, describe_mcp_error, discover_tools row = (await session.execute( select(McpClient).where(McpClient.tenant_id == tenant_id, McpClient.id == client_id) )).scalar_one_or_none() if row is None: raise HTTPException(status.HTTP_404_NOT_FOUND, "mcp client not found") + # A server behind a per-user auth provider only has THIS caller's token to answer with. try: - tools = await discover_tools(row, tenant_id, project_id) + tools = await discover_tools(row, tenant_id, project_id, {"end_user_id": str(user.id)}) except McpUnavailable as e: return {"ok": False, "error": str(e)} except Exception as e: # noqa: BLE001 - surface connect/auth errors to the UI, don't 500 - return {"ok": False, "error": f"Could not connect: {e}"} + # Unwrapped, an anyio task group reports every failure as "unhandled errors in a + # TaskGroup", which hides the 401/DNS/TLS cause the user needs to see. + return {"ok": False, "error": f"Could not connect: {describe_mcp_error(e)}"} return {"ok": True, "tools": tools} @@ -117,5 +147,5 @@ async def delete_client(project_id: str, client_id: str, session: AsyncSession = await session.delete(m) await session.commit() from forge.tools.mcp import invalidate_client - invalidate_client(client_id) + await invalidate_client(client_id) return {"ok": True} diff --git a/apps/api/forge/routers/oauth.py b/apps/api/forge/routers/oauth.py index 69c6182..15df85a 100644 --- a/apps/api/forge/routers/oauth.py +++ b/apps/api/forge/routers/oauth.py @@ -8,11 +8,7 @@ from __future__ import annotations -import base64 -import hashlib -import secrets as _secrets from html import escape -from urllib.parse import urlencode from fastapi import APIRouter, Depends, HTTPException, status from fastapi.responses import HTMLResponse @@ -20,12 +16,17 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from forge.auth_providers.oauth_flow import ( + OAuthNotConfigured, + build_authorize_url, + redirect_uri, + token_request, +) from forge.auth_providers.resolver import AuthResolver -from forge.config import settings from forge.deps import CurrentUser, current_tenant_id, get_session, require_role from forge.models import AuthProvider from forge.secrets.store import SecretNotFound, SecretStore -from forge.security import TokenError, create_state_token, decode_token +from forge.security import TokenError, decode_token from forge.util.http import shared_async_client from forge.util.ssrf import guarded_request @@ -42,10 +43,6 @@ class OAuthStartIn(BaseModel): context: dict | None = None -def _redirect_uri(cfg: dict) -> str: - return cfg.get("redirect_uri") or f"{settings.public_base_url.rstrip('/')}/v1/oauth/callback" - - async def _load(session, tenant_id: str, project_id: str, ap_id: str) -> AuthProvider: ap = ( await session.execute( @@ -70,37 +67,16 @@ async def oauth_start( _: CurrentUser = Depends(require_role("editor")), ): ap = await _load(session, tenant_id, project_id, ap_id) - cfg = ap.config or {} - client_id = await SecretStore().read_ref(tenant_id=tenant_id, project_id=project_id, ref=cfg["client_id_ref"]) if cfg.get("client_id_ref") else None - if not client_id: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "client_id secret not configured") - # PKCE (finding i): bind the authorization code to a per-request verifier so an intercepted - # code can't be redeemed without it. The verifier rides in the SIGNED state (tamper-proof) - # and is echoed back to us in the callback. NOTE: the signed state is readable by the - # browser; for a PUBLIC client (no client_secret) store the verifier server-side instead. - verifier = _secrets.token_urlsafe(64) - challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") - state_claims = {"tid": tenant_id, "pid": project_id, "ap": ap_id, "cv": verifier} - # Per-user connect: carry ONLY the dims that key this provider's per-user bundle, so the - # callback stores the token under the same per-user name resolve/refresh look up. Absent (or - # a non-per-user provider) => the default single-account name, preserving prior behavior. - per_user = cfg.get("per_user_context_keys") or [] - ctx = (body.context if body else None) or {} - user_ctx = {k: ctx[k] for k in per_user if k in ctx} - if user_ctx: - state_claims["ctx"] = user_ctx - state = create_state_token(state_claims) - q = { - "response_type": "code", - "client_id": str(client_id), - "redirect_uri": _redirect_uri(cfg), - "state": state, - "code_challenge": challenge, - "code_challenge_method": "S256", - } - if cfg.get("scope"): - q["scope"] = cfg["scope"] - return {"authorize_url": f"{cfg['authorize_url']}?{urlencode(q)}"} + # PKCE + signed state live in auth_providers.oauth_flow, shared with the Connectors screen + # so both connect paths have identical security properties (finding i). + try: + url = await build_authorize_url( + ap, tenant_id=tenant_id, project_id=project_id, + context=(body.context if body else None) or {}, + ) + except OAuthNotConfigured as e: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(e)) from e + return {"authorize_url": url} @router.get("/v1/oauth/callback", response_class=HTMLResponse) @@ -131,7 +107,10 @@ async def oauth_callback( data = { "grant_type": "authorization_code", "code": code, - "redirect_uri": _redirect_uri(cfg), + # The SAME helper build_authorize_url used. OAuth requires the redirect_uri sent at + # /authorize and at the exchange to match exactly, so the two legs must never be able to + # compute it differently. + "redirect_uri": redirect_uri(cfg), "client_id": str(client_id) if client_id else None, "client_secret": str(client_secret) if client_secret else None, # PKCE proof matching the code_challenge sent at /start (finding i). @@ -140,9 +119,10 @@ async def oauth_callback( # Fetch the token through the SSRF guard (validates the host pre-connect AND re-validates # any redirect hop, with httpx's cross-origin credential stripping) rather than a raw POST # that would follow a redirect to an internal host (audit S8). + form, headers = token_request(cfg, data) r = await guarded_request( shared_async_client(), "POST", cfg["token_url"], - data={k: v for k, v in data.items() if v is not None}, timeout=30, follow_redirects=True, + data=form, headers=headers, timeout=30, follow_redirects=True, ) if r.status_code >= 400: return HTMLResponse( @@ -165,7 +145,49 @@ async def oauth_callback( # provider's token was stored where resolve never looked (finding i / item 5). bundle_name = AuthResolver.bundle_secret_name(ap_id, claims.get("ctx"), cfg.get("per_user_context_keys")) await AuthResolver()._store_bundle(tenant_id, project_id, ap_id, bundle, name=bundle_name) - return HTMLResponse("

✅ Connected

You can close this window and return to Forge.

") + # The per-user dims this consent was for, carried through the signed state. Discovery below + # has to ask the server AS THAT USER - it is the only credential that exists. + note = await _finish_connector_connect(session, tenant_id, project_id, ap_id, claims.get("ctx")) + return HTMLResponse(f"

✅ Connected

You can close this window and return to Forge.{note}

") + + +async def _finish_connector_connect(session, tenant_id: str, project_id: str, ap_id: str, + context: dict | None = None) -> str: + """If this provider belongs to an installed connector, mark it connected and (for an + MCP-backed one) discover its tools now that credentials exist. + + An MCP connector cannot list its tools before consent - the server answers 401 - so install + deliberately leaves the tool set empty and this is where it gets filled. `context` is the end + user whose token was just stored; without it the discovery call would resolve a shared bundle + that a per-user connector never writes and get a second 401, leaving the connector connected + with zero actions. Failures here are reported in the page text but never fail the callback: + the token IS stored, and a user can always re-sync from the Connectors screen. + """ + from forge.connectors.install import ConnectorInstaller + from forge.models import ConnectorInstall + + install = (await session.execute( + select(ConnectorInstall).where( + ConnectorInstall.tenant_id == tenant_id, + ConnectorInstall.project_id == project_id, + ConnectorInstall.auth_provider_id == ap_id, + ) + )).scalar_one_or_none() + if install is None: + return "" + install.status = "connected" + install.status_detail = None + await session.commit() + if not install.mcp_client_id: + return "" + try: + count = await ConnectorInstaller().sync_tools(session, install, context=context) + except Exception as e: # noqa: BLE001 - the connection succeeded; discovery is recoverable + from forge.tools.mcp import describe_mcp_error + + return (" (Connected, but listing actions failed: " + f"{escape(describe_mcp_error(e)[:200])}. Use “Refresh actions”.)") + return f" {count} action(s) are now available." if count else "" @router.get(_PREFIX + "/status") diff --git a/apps/api/forge/routers/triggers.py b/apps/api/forge/routers/triggers.py index 0ea9393..7ff85e5 100644 --- a/apps/api/forge/routers/triggers.py +++ b/apps/api/forge/routers/triggers.py @@ -1,36 +1,212 @@ -"""List a project's triggers (webhook URLs, schedules) for the console.""" +"""A project's triggers (webhook URLs, schedules): who they run as, and who they belong to. + +Two separate questions, deliberately kept apart - conflating them is what makes automation +ownership muddy: + + * `run_as_user_id` - WHOSE connected accounts the run uses. A trigger fires with nobody signed + in, so it carries an identity; that is what lets a scheduled workflow send from a connected + Gmail. `PUT /{id}/run-as` moves it. + * `scope` - WHO the trigger belongs to, and therefore who sees it here. `project` is a team + automation (the prod monitor, the nightly build) that everyone in the project works with; + `user` is someone's own (a salesperson's lead-chaser), listed only for them. + `PUT /{id}/scope` moves it. + +They vary independently on purpose: a shared team automation can legitimately act through one +person's Slack account, and a personal automation can be handed to a colleague without becoming +everybody's. + +SCOPE IS OWNERSHIP, NOT A LOCK. A webhook URL is a credential - anyone holding it fires the +trigger whatever its scope says - and every run is visible in Traces regardless. Scope decides +whose list a trigger appears in and who may change it, nothing more. +""" from __future__ import annotations -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from forge.config import settings -from forge.deps import current_tenant_id, get_session -from forge.models import Trigger +from forge.deps import ( + CurrentUser, + current_tenant_id, + effective_role, + get_current_user, + get_session, + role_at_least, +) +from forge.models import Trigger, User +from forge.services.triggers import SCOPES router = APIRouter(prefix="/v1/projects/{project_id}/triggers", tags=["triggers"]) +class RunAsIn(BaseModel): + # Omitted / null means "me" - the common case, someone claiming an automation they now own. + user_id: str | None = None + + +class ScopeIn(BaseModel): + scope: str # project | user + + +async def _labels(session: AsyncSession, tenant_id: str, user_ids: set[str]) -> dict[str, str]: + """Map user id -> email for display. A missing id (a user who has since been removed) is + deliberately absent, so the UI can say the automation has no working identity rather than + printing a bare uuid that means nothing to anyone.""" + ids = {u for u in user_ids if u} + if not ids: + return {} + rows = (await session.execute( + select(User).where(User.tenant_id == tenant_id, User.id.in_(ids)) + )).scalars() + return {u.id: u.email for u in rows} + + @router.get("") async def list_triggers( + request: Request, project_id: str, session: AsyncSession = Depends(get_session), tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user), ): - rows = (await session.execute( + """Project triggers, plus your own personal ones. + + Someone else's personal trigger is hidden - a colleague's private lead-chaser is noise on + your screen. It is NOT hidden from an admin, though: every run it produces is already in + Traces, so filtering it out of the one screen that explains WHY those runs happen would be + false privacy that only costs the person who has to answer for the project. + """ + rows = list((await session.execute( select(Trigger).where(Trigger.tenant_id == tenant_id, Trigger.project_id == project_id) - )).scalars() + )).scalars()) + me = str(user.id) + oversight = role_at_least(await effective_role(user, request), "admin") base = settings.public_base_url.rstrip("/") + emails = await _labels(session, tenant_id, {t.run_as_user_id for t in rows if t.run_as_user_id}) out = [] for t in rows: + personal = t.scope == "user" + mine = bool(t.run_as_user_id) and t.run_as_user_id == me + if personal and not mine and not oversight: + continue item = { "id": t.id, "workflow_id": t.workflow_id, "node_id": t.node_id, "kind": t.kind, "enabled": t.enabled, "config": t.config, "last_fired_at": t.last_fired_at.isoformat() if t.last_fired_at else None, + "scope": t.scope or "project", + "run_as_user_id": t.run_as_user_id, + # None when unset OR when that user no longer exists - both mean "this trigger has + # no connected accounts to draw on", which is the thing worth showing. + "run_as_email": emails.get(t.run_as_user_id or ""), + "run_as_is_me": mine, + # True only when you are seeing someone else's personal trigger because of your role. + "visible_via_oversight": personal and not mine, } if t.kind == "webhook_in" and t.key: item["webhook_url"] = f"{base}/v1/hooks/{t.key}" out.append(item) return out + + +@router.put("/{trigger_id}/scope") +async def set_scope( + request: Request, + project_id: str, + trigger_id: str, + body: ScopeIn, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user), +): + """Move a trigger between "the team's" and "mine". + + Sharing it with the project is an editor decision: colleagues will see it, depend on it, and + be able to reassign it. Making one personal is open to the person it runs as - it is their + account doing the work - but not to a bystander, because hiding a trigger others rely on + would look exactly like it disappearing. + + Making one personal is deliberately NOT an ordinary editor power. "Personal" means "listed + only for the person it runs as", so an editor doing it to a trigger that runs as someone else + - or as nobody - removes it from their own screen the instant they click, with no control + left to put it back. Only the person it runs as, or an admin (who keeps oversight visibility + either way), can take that step. + """ + if body.scope not in SCOPES: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, f"scope must be one of {list(SCOPES)}") + trigger = (await session.execute( + select(Trigger).where( + Trigger.tenant_id == tenant_id, Trigger.project_id == project_id, Trigger.id == trigger_id, + ) + )).scalar_one_or_none() + if trigger is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "trigger not found") + + role = await effective_role(user, request) + is_editor = role_at_least(role, "editor") + is_admin = role_at_least(role, "admin") + owns_it = bool(trigger.run_as_user_id) and trigger.run_as_user_id == str(user.id) + if body.scope == "project" and not is_editor: + raise HTTPException(status.HTTP_403_FORBIDDEN, "sharing a trigger with the project requires role 'editor'") + if body.scope == "user": + if not trigger.run_as_user_id: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "this trigger doesn't run as anyone yet - set 'Runs as' first, or it would be " + "personal to nobody and listed for nobody", + ) + if not (owns_it or is_admin): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "only the person a trigger runs as (or an admin) can make it personal", + ) + trigger.scope = body.scope + await session.commit() + return {"scope": trigger.scope} + + +@router.put("/{trigger_id}/run-as") +async def set_run_as( + request: Request, + project_id: str, + trigger_id: str, + body: RunAsIn | None = None, + session: AsyncSession = Depends(get_session), + tenant_id: str = Depends(current_tenant_id), + user: CurrentUser = Depends(get_current_user), +): + """Point this trigger at a person's connected accounts. + + Claiming it for YOURSELF is open to any real logged-in user - it only ever narrows the run to + accounts you personally connected. Assigning it to SOMEONE ELSE means their credentials get + used by a workflow they may not have touched, so that is an editor decision. + """ + if str(user.id).startswith(("apikey:", "service")): + raise HTTPException(status.HTTP_403_FORBIDDEN, "run-as requires a user identity") + trigger = (await session.execute( + select(Trigger).where( + Trigger.tenant_id == tenant_id, Trigger.project_id == project_id, Trigger.id == trigger_id, + ) + )).scalar_one_or_none() + if trigger is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "trigger not found") + + target = (body.user_id if body else None) or str(user.id) + if target != str(user.id): + if not role_at_least(await effective_role(user, request), "editor"): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "assigning a trigger to another person's accounts requires role 'editor'", + ) + exists = (await session.execute( + select(User).where(User.tenant_id == tenant_id, User.id == target) + )).scalar_one_or_none() + if exists is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "no such user in this workspace") + + trigger.run_as_user_id = target + await session.commit() + emails = await _labels(session, tenant_id, {target}) + return {"run_as_user_id": target, "run_as_email": emails.get(target)} diff --git a/apps/api/forge/routers/workflows.py b/apps/api/forge/routers/workflows.py index b2f6241..448ded6 100644 --- a/apps/api/forge/routers/workflows.py +++ b/apps/api/forge/routers/workflows.py @@ -18,6 +18,7 @@ WorkflowUpdate, ) from forge.services.portability import PortabilityService +from forge.services.triggers import default_scope_for from forge.services.versions import safe_snapshot from forge.services.workflows import WorkflowService @@ -120,7 +121,10 @@ async def update_executable( wf = await WorkflowService.get(session, tenant_id, workflow_id) if wf is None: raise HTTPException(404, "Workflow not found") - result = await WorkflowService.update_executable(session, wf, body.executable, require_valid=True) + result = await WorkflowService.update_executable( + session, wf, body.executable, require_valid=True, + owner=str(user.id), scope=default_scope_for(user.role), + ) if result.valid: await safe_snapshot(session, "workflow", wf, author=user) return ValidateOut(valid=result.valid, errors=result.errors, warnings=result.warnings) @@ -162,7 +166,12 @@ async def publish_workflow( wf.active_version = (wf.active_version or 1) + 1 await session.commit() await session.refresh(wf) - await WorkflowService._sync_triggers(session, wf) # (re)register webhook/schedule/etc. + # (Re)register webhook/schedule/etc. The publisher becomes the identity an unattended run + # acts as, and (for a NEW trigger) whether it is the team's or their own - unless the + # trigger already has those, which an edit never overwrites. + await WorkflowService._sync_triggers( + session, wf, owner=str(user.id), scope=default_scope_for(user.role), + ) await safe_snapshot(session, "workflow", wf, author=user) return wf @@ -193,6 +202,9 @@ async def save_canvas( wf = await WorkflowService.get(session, tenant_id, workflow_id) if wf is None: raise HTTPException(404, "Workflow not found") - result = await WorkflowService.save_canvas(session, wf, body.canvas, body.executable) + result = await WorkflowService.save_canvas( + session, wf, body.canvas, body.executable, + owner=str(user.id), scope=default_scope_for(user.role), + ) await safe_snapshot(session, "workflow", wf, author=user) return ValidateOut(valid=result.valid, errors=result.errors, warnings=result.warnings) diff --git a/apps/api/forge/schemas/dto.py b/apps/api/forge/schemas/dto.py index 4e208a2..c169aa9 100644 --- a/apps/api/forge/schemas/dto.py +++ b/apps/api/forge/schemas/dto.py @@ -31,6 +31,7 @@ class ProjectCountsOut(BaseModel): knowledge: int auth: int handoffs: int + connectors: int = 0 class ProjectCreate(BaseModel): diff --git a/apps/api/forge/secrets/store.py b/apps/api/forge/secrets/store.py index 27684a1..af226d2 100644 --- a/apps/api/forge/secrets/store.py +++ b/apps/api/forge/secrets/store.py @@ -97,3 +97,52 @@ async def read_ref(self, *, tenant_id: str, project_id: str, ref: str) -> Any: meta={"scheme": scheme}, ) return value + + async def read_refs(self, *, tenant_id: str, project_id: str, refs: list[str]) -> dict[str, Any]: + """Read several refs at once, returning `{name: value}` for the ones that EXIST. + + Same choke point, same audit trail, one round trip. A caller resolving one secret per row + of a list - the connectors screen reads a token bundle per installed connector - otherwise + pays a SELECT, a decrypt and an audit session each, sequentially, on every paint. + + A missing name is simply absent from the result rather than raising, because a batch + caller is asking about several independent things and one of them being absent is an + answer, not an error. Callers that need "missing" to be fatal should use `read_ref`. + """ + names: list[str] = [] + for ref in refs: + scheme, name = self.parse_ref(ref) + if scheme == "vault": # pragma: no cover - enterprise path + raise NotImplementedError("vault:// refs require the Vault adapter (enterprise).") + if name not in names: + names.append(name) + if not names: + return {} + out: dict[str, Any] = {} + async with self._sf() as session: + rows = list((await session.execute( + select(Secret).where( + Secret.tenant_id == tenant_id, Secret.project_id == project_id, + Secret.name.in_(names), + ) + )).scalars()) + now = datetime.utcnow() + touched = False + for secret in rows: + if secret.last_used_at is None or now - secret.last_used_at > self._LAST_USED_WRITE_WINDOW: + secret.last_used_at = now + touched = True + try: + out[secret.name] = json.loads(decrypt(secret.encrypted_value)) + except Exception: # noqa: BLE001 - an undecodable secret reads as absent, not a 500 + continue + if touched: + await session.commit() + await AuditService.log_many([ + { + "tenant_id": tenant_id, "action": "secret.read", "project_id": project_id, + "resource_type": "secret", "resource_id": name, "meta": {"scheme": "secret"}, + } + for name in out + ]) + return out diff --git a/apps/api/forge/services/audit.py b/apps/api/forge/services/audit.py index 14ed86d..0ddaecc 100644 --- a/apps/api/forge/services/audit.py +++ b/apps/api/forge/services/audit.py @@ -89,6 +89,34 @@ async def log( except Exception: # noqa: BLE001 - auditing must never break the request log.exception("audit write failed for action=%s", action) + @staticmethod + async def log_many(entries: list[dict[str, Any]]) -> None: + """Append several records in ONE session. + + Same guarantees as `log` - append-only, own session, never raises - for a caller that + genuinely performs N audited operations at once (a batched secret read). Auditing them + individually would put N session round trips back on a path whose whole point was to + stop doing N of anything. + """ + if not entries: + return + try: + rows = [] + for e in entries: + row = AuditLog( + tenant_id=e["tenant_id"], action=e["action"], actor_id=e.get("actor_id"), + actor_email=e.get("actor_email"), resource_type=e.get("resource_type"), + resource_id=e.get("resource_id"), project_id=e.get("project_id"), + ip=e.get("ip"), status=e.get("status", "ok"), meta=e.get("meta") or {}, + ) + _truncate_string_columns(row) + rows.append(row) + async with SessionLocal() as s: + s.add_all(rows) + await s.commit() + except Exception: # noqa: BLE001 - auditing must never break the request + log.exception("audit batch write failed for %d record(s)", len(entries)) + @staticmethod async def recent(session, tenant_id: str, *, project_id: str | None = None, limit: int = 200) -> list[AuditLog]: q = select(AuditLog).where(AuditLog.tenant_id == tenant_id) diff --git a/apps/api/forge/services/dispatch.py b/apps/api/forge/services/dispatch.py index e1492d9..acd5cd1 100644 --- a/apps/api/forge/services/dispatch.py +++ b/apps/api/forge/services/dispatch.py @@ -49,10 +49,16 @@ async def dispatch_trigger(run_service: RunService, trigger: Trigger, payload, * schedule (re-check + stamp in one txn) before calling this, so it passes stamp=False to avoid a redundant second write; the webhook/inbound paths keep the default True.""" run_input = TriggerService.build_input(trigger, payload) + # Whose accounts this unattended run acts as. A webhook or a schedule has nobody signed in, + # so without an identity here every per-user connector (which is every catalog connector) + # would fail at its first tool call with "the acting user has not connected their credential". + # The trigger carries the editor who set it up, so a scheduled workflow reads THEIR mailbox - + # the same account they connected when building it. + end_user = {"id": trigger.run_as_user_id} if trigger.run_as_user_id else None async with SessionLocal() as s: run = await run_service.create_run( s, tenant_id=trigger.tenant_id, project_id=trigger.project_id, - workflow_id=trigger.workflow_id, input=run_input, + workflow_id=trigger.workflow_id, input=run_input, end_user=end_user, source=_TRIGGER_SOURCE.get(trigger.kind, trigger.kind or "webhook"), ) run_id = run.id diff --git a/apps/api/forge/services/projects.py b/apps/api/forge/services/projects.py index 0315445..b6ed25d 100644 --- a/apps/api/forge/services/projects.py +++ b/apps/api/forge/services/projects.py @@ -15,6 +15,7 @@ AuthProvider, Channel, Component, + ConnectorInstall, Dataset, HandoffRequest, KbSource, @@ -44,6 +45,7 @@ "components": Component, "knowledge": KbSource, "auth": AuthProvider, + "connectors": ConnectorInstall, } @@ -182,6 +184,7 @@ async def delete(session: AsyncSession, project: Project, *, checkpointer: Any = KbSource, QaPair, McpClient, + ConnectorInstall, Channel, Trigger, Dataset, diff --git a/apps/api/forge/services/runtime.py b/apps/api/forge/services/runtime.py index 6f34f99..57bcb7a 100644 --- a/apps/api/forge/services/runtime.py +++ b/apps/api/forge/services/runtime.py @@ -164,8 +164,11 @@ async def build_compile_context( # Pre-load enabled MCP servers' tools (one connect per server) so agent nodes can # attach them - the agent factory is sync, but MCP discovery is async. from forge.models import McpClient - from forge.tools.mcp import server_tools + from forge.tools.mcp import auth_context_from, server_tools + # The run's per-user dims, so a server behind a PER-USER auth provider resolves the calling + # end user's own credential here rather than a shared workspace token. + mcp_auth_context = auth_context_from(ctx) mcp_by_client: dict[str, list] = {} mcp_rows = ( await session.execute( @@ -178,7 +181,7 @@ async def build_compile_context( ).scalars() for m in mcp_rows: try: - mcp_by_client[m.id] = await server_tools(m, tenant_id, project_id) + mcp_by_client[m.id] = await server_tools(m, tenant_id, project_id, mcp_auth_context) except Exception as e: # noqa: BLE001 - an unreachable server must not break the run log.warning("MCP server %s unavailable, skipping its tools: %s", m.name, e) ctx.mcp_tools_by_client = mcp_by_client diff --git a/apps/api/forge/services/triggers.py b/apps/api/forge/services/triggers.py index 9e89fcc..ea1da94 100644 --- a/apps/api/forge/services/triggers.py +++ b/apps/api/forge/services/triggers.py @@ -16,11 +16,67 @@ from forge.models import Trigger from forge.nodes.triggers import TRIGGER_TYPES +#: Scopes a trigger can have. "project" is a team automation everyone sees; "user" is someone's +#: own, listed only for them. +SCOPES = ("project", "user") + +#: `Trigger.run_as_user_id` is String(36) - a uuid. A machine principal's id is not one. +_MAX_OWNER_LEN = 36 + + +def owner_id_for(user_id: str | None) -> str | None: + """The person a trigger runs as, or None when the saver isn't one. + + A workflow can be saved by a machine principal: the static service token (id "service") or a + scoped API key (id "apikey:"). Neither has connected accounts, so neither is a + meaningful run-as identity - and `apikey:` is 43 characters going into a String(36) + column, which Postgres rejects outright. `_sync_triggers` swallows exceptions so a workflow + still saves, meaning the failure would surface as webhooks and schedules silently never being + registered. The dedicated run-as route already refuses these identities; this is the same + rule applied on the path that stamps them implicitly. + """ + if not user_id: + return None + if user_id.startswith(("apikey:", "service")) or len(user_id) > _MAX_OWNER_LEN: + return None + return user_id + + +def default_scope_for(role: str | None) -> str: + """Whether a NEW trigger this person creates belongs to the project or to them. + + Admins and owners are configuring the project - a build pipeline, a prod monitor - so what + they add is the team's. Everyone else is building for themselves (a salesperson's own + lead-chaser), so theirs stays theirs until they deliberately share it. Either way the choice + is only a default: both directions are one click on the Triggers screen. + """ + return "project" if role in ("owner", "admin") else "user" + class TriggerService: @staticmethod - async def sync_from_workflow(session, workflow) -> list[Trigger]: - """Upsert one Trigger per trigger node in the workflow; drop removed ones.""" + async def sync_from_workflow(session, workflow, *, owner: str | None = None, + scope: str | None = None) -> list[Trigger]: + """Upsert one Trigger per trigger node in the workflow; drop removed ones. + + `owner` is the editor doing the save. It becomes the trigger's `run_as_user_id` - the + identity an unattended run acts as - and is only ever stamped on a trigger that doesn't + have one yet. A colleague editing the workflow must NOT silently take ownership: the + automation runs on whoever's Gmail/Slack account is connected behind it, and quietly + repointing that at the last person to touch the canvas would change what the workflow + can see without anyone deciding to. Reassignment is explicit (Triggers screen). + + `scope` is the same story for ownership: applied only when the trigger is NEW. Editing a + workflow must never move a team automation into someone's private list, nor publish + someone's private one to the whole project. + + A save by a machine principal (service token / API key) yields no owner, and a trigger + with no owner is never personal - "listed only for the person it runs as" hides it from + everyone when that person doesn't exist. + """ + owner = owner_id_for(owner) + if not owner: + scope = "project" ex = workflow.executable or {} nodes = [n for n in ex.get("nodes", []) if isinstance(n, dict) and n.get("type") in TRIGGER_TYPES] existing = list((await session.execute( @@ -39,7 +95,8 @@ async def sync_from_workflow(session, workflow) -> list[Trigger]: tenant_id=workflow.tenant_id, project_id=workflow.project_id, workflow_id=workflow.id, node_id=node_id, kind=n["type"], key=uuid.uuid4().hex if n["type"] == "webhook_in" else None, - config=cfg, enabled=True, + config=cfg, enabled=True, run_as_user_id=owner, + scope=(scope if scope in SCOPES else "project"), ) session.add(trig) else: @@ -47,6 +104,10 @@ async def sync_from_workflow(session, workflow) -> list[Trigger]: trig.config = cfg if n["type"] == "webhook_in" and not trig.key: trig.key = uuid.uuid4().hex + # Adopt an owner for a trigger created before this column existed, but never + # replace one that is already set (see the docstring). + if not trig.run_as_user_id and owner: + trig.run_as_user_id = owner out.append(trig) # Remove triggers whose node no longer exists. for t in existing: diff --git a/apps/api/forge/services/workflows.py b/apps/api/forge/services/workflows.py index 6d965dc..a5756bf 100644 --- a/apps/api/forge/services/workflows.py +++ b/apps/api/forge/services/workflows.py @@ -94,19 +94,21 @@ def validate(executable: dict) -> ValidationResult: @staticmethod async def update_executable( - session: AsyncSession, wf: Workflow, executable: dict, *, require_valid: bool = True + session: AsyncSession, wf: Workflow, executable: dict, *, require_valid: bool = True, + owner: str | None = None, scope: str | None = None, ) -> ValidationResult: result = validate_workflow(executable) if result.valid or not require_valid: wf.executable = executable await session.commit() await session.refresh(wf) - await WorkflowService._sync_triggers(session, wf) + await WorkflowService._sync_triggers(session, wf, owner=owner, scope=scope) return result @staticmethod async def save_canvas( - session: AsyncSession, wf: Workflow, canvas: dict, executable: dict + session: AsyncSession, wf: Workflow, canvas: dict, executable: dict, + *, owner: str | None = None, scope: str | None = None, ) -> ValidationResult: """Persist the canvas (UI-owned) + compiled executable, and validate. @@ -118,15 +120,22 @@ async def save_canvas( wf.executable = executable await session.commit() await session.refresh(wf) - await WorkflowService._sync_triggers(session, wf) + await WorkflowService._sync_triggers(session, wf, owner=owner, scope=scope) return result @staticmethod - async def _sync_triggers(session: AsyncSession, wf: Workflow) -> None: - """Mirror the workflow's trigger nodes into Trigger rows (best-effort).""" + async def _sync_triggers(session: AsyncSession, wf: Workflow, *, owner: str | None = None, + scope: str | None = None) -> None: + """Mirror the workflow's trigger nodes into Trigger rows (best-effort). + + `owner` (the editor saving) is stamped on triggers that don't have one yet - it is the + identity an unattended run acts as, so a scheduled workflow can use the connected + accounts of the person who built it. `scope` decides whether a NEW trigger belongs to + the project or to that person. Neither ever overwrites an existing trigger's values. + """ try: from forge.services.triggers import TriggerService - await TriggerService.sync_from_workflow(session, wf) + await TriggerService.sync_from_workflow(session, wf, owner=owner, scope=scope) except Exception: # noqa: BLE001 - trigger sync must not block saving a workflow pass diff --git a/apps/api/forge/tools/mcp.py b/apps/api/forge/tools/mcp.py index d0cd235..4f8ec78 100644 --- a/apps/api/forge/tools/mcp.py +++ b/apps/api/forge/tools/mcp.py @@ -6,47 +6,220 @@ `remote_tool_name` to expose and any `inject_context` keys to fill from the per-user runtime context (so the model never sets secrets like user_id/api_key). +A server may be authenticated two ways, and they compose: + +* `headers_ref` - a static header secret (an API key / PAT). Simple, shared, no refresh. +* `auth_provider_id` - a full Auth Provider. This is what reaches an OAuth-protected remote + MCP server: tokens refresh automatically, and a PER-USER provider resolves a different + credential for each end user, so an agent acts as the calling user rather than through one + shared workspace token. Provider headers are applied per REQUEST (via httpx.Auth) rather + than baked into the connection, so a token that rotates mid-session is picked up without + reconnecting. + MCP discovery is async, so MCP tools are loaded by `load_mcp_tools` from the runtime assembler (not the sync `materialize_tool` path). """ from __future__ import annotations +import asyncio import contextlib +import hashlib import logging import time +from dataclasses import dataclass, field from typing import Any +import httpx from langchain.tools import ToolRuntime from sqlalchemy import select from forge.db.base import SessionLocal -from forge.models import McpClient +from forge.models import AuthProvider, McpClient from forge.secrets.store import SecretStore log = logging.getLogger("forge.mcp") -# Cache MultiServerMCPClient instances per mcp_client_id with a TTL so a dead connection or + +@dataclass +class _Cached: + """One pooled MCP connection, plus the lock that serializes (re)building it. + + The lock lives HERE rather than in a side registry keyed by the same string. The cache is + capped at `_CACHE_MAX`; a separate registry is not, so keying one by a per-user cache key + would grow a lock per (server, person) for the life of the process - reintroducing, in the + lock table, exactly the unbounded growth the cap exists to prevent. + + `created` drives expiry (age of the connection); `used` drives eviction (last time anyone + asked for it). Keeping them apart is what makes eviction least-RECENTLY-USED: a single + timestamp refreshed on use would never expire a hot connection, and one that is not + refreshed evicts the busiest connection first. + """ + + created: float + used: float + client: Any = None + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + +# Cache MultiServerMCPClient instances per cache key with a TTL so a dead connection or # an edited server config is eventually re-established without a process restart (audit F12). -# `invalidate_client` drops one entry immediately (called when the McpClient row changes); +# The key is `mcp_client_id` for a shared server, and `mcp_client_id::` when the +# attached auth provider resolves a DIFFERENT credential per caller - without that suffix one +# caller's authenticated session would be handed to the next caller of the same server. +# `invalidate_client` drops every entry for a server (called when the McpClient row changes); # `close_all` is called on shutdown. -_CLIENT_CACHE: dict[str, tuple[float, Any]] = {} +_CLIENT_CACHE: dict[str, _Cached] = {} _CACHE_TTL = 300.0 # seconds +# Hard ceiling on live connections. The key carries a per-caller suffix and every catalog MCP +# connector is per-user, so the cache grows with DISTINCT PEOPLE, not distinct servers - a +# project where 500 users each connect Notion would otherwise pin 500 transports for the life +# of the process. The TTL alone doesn't bound it: it only replaces an entry when that same key +# is asked for again, so an idle user's connection is never revisited and never released. +_CACHE_MAX = 64 +# A connection leaving the cache is NOT torn down on the spot. `_client_and_tools` hands the +# client back to its caller and `load_mcp_tool` binds agent tools to it, so a client can still +# be in use long after the cache stops tracking it - closing it there aborts a call that is +# already in flight. Retired clients wait out this grace period and are then closed by the +# reaper below. +_CLOSE_GRACE = 30.0 +_RETIRED: list[tuple[float, Any]] = [] +# How often the reaper wakes. Short relative to `_CLOSE_GRACE` so a transport is closed near its +# deadline rather than a whole interval past it. +_REAP_INTERVAL = 10.0 +#: The single background reaper task, or None when nothing is waiting to be closed. +_REAPER: asyncio.Task | None = None + + +async def _aclose(client: Any) -> None: + aclose = getattr(client, "aclose", None) + if aclose is not None: + with contextlib.suppress(Exception): + await aclose() + + +def _retire(client: Any, now: float) -> None: + """Hand a client over for closing once its grace period expires.""" + if client is not None: + _RETIRED.append((now + _CLOSE_GRACE, client)) + _ensure_reaper() + + +def _ensure_reaper() -> None: + """Make sure the background reaper is running. + + Retirement used to be drained only by `_evict`, which runs only inside a cache BUILD. So a + shedding round that retired ten transports was closed promptly only if traffic happened to + miss the cache again: once it settled into a steady state where every call hit a live entry, + those transports stayed open until some key expired or the process shut down - far past the + 30s grace they were given. + + ONE long-lived task, not a task per retirement. Per-retirement deferral is what the earlier + implementation did, and `spawn` REJECTS (and closes) its coroutine at an in-flight ceiling, + which stranded the transport with nothing holding a reference to it. + """ + global _REAPER + if _REAPER is not None and not _REAPER.done(): + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + # Retired with no running loop. Nothing is holding the transport open in that case + # either, and the next retirement inside a loop starts the reaper. + _REAPER = None + return + _REAPER = loop.create_task(_reap_loop()) + + +async def _reap_loop() -> None: + """Close retired transports near their deadline - off the request path, and off the per-key + build lock that `_evict` runs under. + Exits once the queue drains so an idle process isn't holding a timer open; `_retire` starts + it again. `close_all` cancels it and force-drains whatever is left. + """ + while _RETIRED: + await asyncio.sleep(_REAP_INTERVAL) + await _reap(time.monotonic()) + + +async def _reap(now: float, *, force: bool = False) -> None: + """Close retired clients whose grace period has passed (all of them when `force`). + + Called by `_reap_loop` on a timer and by `close_all` at shutdown - NOT from the eviction + path, which must not do network I/O while holding a build lock. + """ + if not _RETIRED: + return + due = [c for deadline, c in _RETIRED if force or now >= deadline] + _RETIRED[:] = [] if force else [(d, c) for d, c in _RETIRED if now < d] + for client in due: + await _aclose(client) + + +def _evictable() -> list[str]: + """Keys eligible for eviction: built, and not currently being rebuilt. -def invalidate_client(client_id: str) -> None: - """Drop a cached MCP client so the next run reconnects with the latest config.""" - _CLIENT_CACHE.pop(client_id, None) + Skipping a locked entry matters - evicting one mid-build would detach the entry its builder + is about to write into, producing a live client that the cache no longer tracks and that + nothing will ever close. + """ + return [k for k, e in _CLIENT_CACHE.items() if e.client is not None and not e.lock.locked()] + + +def _evict(now: float) -> None: + """Release connections nobody is coming back for: anything past its TTL, then the + least-recently-USED entries once the cache is over its ceiling. + + Deliberately SYNCHRONOUS. It runs while `_client_and_tools` holds the entry's build lock, so + anything awaited here blocks a concurrent caller for that same key - and closing a transport + is an unbounded network await. Eviction is bookkeeping: it pops entries and hands the clients + to `_retire`, and the background reaper closes them. That also keeps the grace period + meaningful, since shedding a connection must never abort an in-flight call. + """ + shed = 0 + for key in [k for k in _evictable() if now - _CLIENT_CACHE[k].created > _CACHE_TTL]: + _retire(_CLIENT_CACHE.pop(key).client, now) + shed += 1 + while len(_CLIENT_CACHE) > _CACHE_MAX: + candidates = _evictable() + if not candidates: + break # everything left is mid-build; the next build will trim instead + oldest = min(candidates, key=lambda k: _CLIENT_CACHE[k].used) + _retire(_CLIENT_CACHE.pop(oldest).client, now) + shed += 1 + if shed: + log.debug("mcp cache: retired %d idle connection(s)", shed) + + +async def invalidate_client(client_id: str) -> None: + """Drop every cached connection for a server so the next run reconnects with the latest + config - including all per-caller variants, which share the `::` prefix. + + Closes what it drops IMMEDIATELY, without the retirement grace: this runs when the server's + row was edited or deleted, so the old connection points at configuration that no longer + exists and finishing an in-flight call on it is not something to protect. + """ + for key in [k for k in _CLIENT_CACHE if k == client_id or k.startswith(client_id + "::")]: + entry = _CLIENT_CACHE.pop(key, None) + if entry is not None: + await _aclose(entry.client) async def close_all() -> None: """Best-effort close of every cached MCP client (transports/subprocesses) on shutdown.""" - for _, client in list(_CLIENT_CACHE.values()): - aclose = getattr(client, "aclose", None) - if aclose is not None: - with contextlib.suppress(Exception): - await aclose() + global _REAPER + if _REAPER is not None: + _REAPER.cancel() + with contextlib.suppress(BaseException): + await _REAPER + _REAPER = None + for entry in list(_CLIENT_CACHE.values()): + await _aclose(entry.client) _CLIENT_CACHE.clear() + # `force` ignores the grace period: the process is going away, so there is no in-flight call + # left to protect and an unclosed transport would just leak into shutdown. + await _reap(time.monotonic(), force=True) class McpUnavailable(RuntimeError): @@ -75,7 +248,127 @@ async def _validate_mcp_url(url: str | None) -> None: await validate_url(url, EgressPolicy.from_settings()) -async def _connection_for(client_row: McpClient, tenant_id: str, project_id: str) -> dict: +async def _load_provider(tenant_id: str, provider_id: str) -> AuthProvider | None: + async with SessionLocal() as session: + return (await session.execute( + select(AuthProvider).where(AuthProvider.tenant_id == tenant_id, AuthProvider.id == provider_id) + )).scalar_one_or_none() + + +class _ProviderAuth(httpx.Auth): + """Attach an Auth Provider's resolved headers to every request to an MCP server. + + Resolution happens per REQUEST (the resolver has its own TTL cache, so this is a dict + lookup in the common case). That is what lets a rotated OAuth token take effect without + tearing down the MCP session, and what makes a per-user provider resolve the CALLER's + credential rather than whichever one happened to be live when the session opened. + + On a 401/403 the cached auth is invalidated and the request is retried exactly once - the + same invalidate-on-401 contract the REST tool follows. One retry, not a loop: if a freshly + minted token is also rejected, the credential is wrong and retrying would only amplify a + failing call into several. + """ + + requires_response_body = False + + def __init__(self, resolver, *, tenant_id: str, project_id: str, provider_id: str, context: dict) -> None: + self._resolver = resolver + self._tenant_id = tenant_id + self._project_id = project_id + self._provider_id = provider_id + self._context = context or {} + + async def _apply(self, request: httpx.Request, *, force: bool) -> None: + resolved = await self._resolver.resolve( + tenant_id=self._tenant_id, project_id=self._project_id, + provider_id=self._provider_id, context=self._context, force=force, + ) + for k, v in resolved.headers.items(): + request.headers[k] = v + if resolved.cookies: + jar = "; ".join(f"{k}={v}" for k, v in resolved.cookies.items()) + existing = request.headers.get("Cookie") + request.headers["Cookie"] = f"{existing}; {jar}" if existing else jar + if resolved.params: + request.url = request.url.copy_merge_params(resolved.params) + + async def async_auth_flow(self, request: httpx.Request): + # Snapshot the whole pre-auth request shape so the retry re-applies onto a CLEAN one. + # Params are MERGED and the cookie jar is CONCATENATED, so applying twice to the same + # mutated request would send `?api_key=X&api_key=X` and `Cookie: s=1; s=1`, which some + # servers reject outright - turning a recoverable 401 into a hard failure. Headers do + # overwrite, but only the ones the SECOND resolve returns: restoring them wholesale + # means a provider that changed shape between the attempts (a renamed header_name) + # can't leave its first-attempt header behind alongside the new one. + original_url = request.url + original_headers = request.headers.copy() + await self._apply(request, force=False) + response = yield request + if response.status_code in (401, 403): + request.url = original_url + request.headers = original_headers + await self._apply(request, force=True) + yield request + + +def auth_cache_dims(cfg: dict) -> list[str]: + """The run-context keys that make one caller's resolved credential different from another's. + + This MUST stay a superset of what `AuthResolver.resolve` varies its own cache on, because the + connection pooled here OUTLIVES the request that built it: the `_ProviderAuth` attached to a + pooled client captures the context of whoever opened it, and every later caller sharing the + cache key is authenticated through that snapshot. Any dimension the resolver treats as + caller-specific and this does not is a credential handed to the wrong person. + + Two such dimensions today: + * per_user_context_keys - a stored per-user connection (end_user_id). + * token_ctx_key - an INLINE token forwarded on the run (X-Forge-Context), with a + deployment-wide fallback. The resolver added this to its key for + precisely this reason; omitting it here would defeat that. + """ + from forge.config import settings + + dims = list(cfg.get("per_user_context_keys") or []) + ctx_key = cfg.get("token_ctx_key") or settings.default_token_ctx_key + if ctx_key and ctx_key not in dims: + dims.append(ctx_key) + return dims + + +async def _auth_for(client_row: McpClient, tenant_id: str, project_id: str, context: dict | None): + """(httpx.Auth | None, cache-key suffix) for a server's attached auth provider.""" + if not getattr(client_row, "auth_provider_id", None): + return None, "" + provider = await _load_provider(tenant_id, client_row.auth_provider_id) + if provider is None: + log.warning("MCP server %s references missing auth provider %s", client_row.name, client_row.auth_provider_id) + return None, "" + from forge.auth_providers.resolver import AuthResolver + + context = context or {} + # Only a provider that resolves a DIFFERENT credential per caller needs a per-caller + # connection; a genuinely shared one keeps a single pooled session for the whole project + # (which is the overwhelmingly common case). + dims = auth_cache_dims(provider.config or {}) + suffix = "" + if dims: + fingerprint = "|".join(f"{k}={context.get(k)}" for k in sorted(dims)) + suffix = "::" + hashlib.sha256(fingerprint.encode()).hexdigest()[:16] + auth = _ProviderAuth( + AuthResolver(), tenant_id=tenant_id, project_id=project_id, + provider_id=provider.id, context=context, + ) + return auth, suffix + + +async def _connection_for(client_row: McpClient, tenant_id: str, project_id: str, + context: dict | None = None, auth: httpx.Auth | None = None) -> dict: + """Build the langchain-mcp-adapters connection dict for a server. + + `auth` lets a caller that has ALREADY resolved the provider hand it in - `_auth_for` does a + DB read, and computing the cache-key suffix plus building the connection would otherwise + query the same AuthProvider row twice on the path every agent turn takes. + """ from forge.config import settings transport = client_row.transport or "streamable_http" @@ -108,46 +401,143 @@ async def _connection_for(client_row: McpClient, tenant_id: str, project_id: str conn["headers"] = headers except Exception: # noqa: BLE001 - missing headers secret => connect without pass + # stdio has no HTTP layer to attach auth to; a provider on a stdio server is a config + # mistake rather than something to silently half-apply. + if transport != "stdio": + if auth is None: + auth, _suffix = await _auth_for(client_row, tenant_id, project_id, context) + if auth is not None: + conn["auth"] = auth return conn -async def _client_and_tools(client_row: McpClient, tenant_id: str, project_id: str): +async def _client_and_tools(client_row: McpClient, tenant_id: str, project_id: str, + context: dict | None = None): MultiServerMCPClient = _require_adapters() now = time.monotonic() - entry = _CLIENT_CACHE.get(client_row.id) - if entry is None or (now - entry[0]) > _CACHE_TTL: - conn = await _connection_for(client_row, tenant_id, project_id) - client = MultiServerMCPClient({client_row.name: conn}) - _CLIENT_CACHE[client_row.id] = (now, client) + auth, suffix = await _auth_for(client_row, tenant_id, project_id, context) + key = client_row.id + suffix + # Claim the slot before any await, so two concurrent misses cannot each create an entry (and + # therefore each build a client, one of which is silently replaced and never closed). A plain + # get-then-set with no await between them is atomic under asyncio; the lock inside the entry + # is then the same object for both. + entry = _CLIENT_CACHE.get(key) + if entry is None: + entry = _CLIENT_CACHE[key] = _Cached(created=0.0, used=now) + + def _fresh() -> bool: + return entry.client is not None and (time.monotonic() - entry.created) <= _CACHE_TTL + + if _fresh(): + entry.used = now else: - client = entry[1] + async with entry.lock: + now = time.monotonic() + if _fresh(): + entry.used = now + else: + # Retire (don't close) the expired connection: a caller from before the TTL + # elapsed may still be running against it. + _retire(entry.client, now) + entry.client = None + try: + # Reuse the provider we just resolved instead of making _connection_for + # re-read it. + conn = await _connection_for(client_row, tenant_id, project_id, context, auth=auth) + entry.client = MultiServerMCPClient({client_row.name: conn}) + except BaseException: + # Don't leave an empty entry parked in the cache: it can never be evicted + # (nothing to retire) yet still counts against the ceiling. + if _CLIENT_CACHE.get(key) is entry: + del _CLIENT_CACHE[key] + raise + entry.created = entry.used = now + if _CLIENT_CACHE.get(key) is not entry: + # `invalidate_client` ran while we were connecting and dropped this entry + # (it does not wait for a build - the config it was building against is + # already gone). Hand this call the client we just made, but retire it so + # the detached transport is still closed rather than orphaned. + _retire(entry.client, now) + else: + _evict(now) + client = entry.client tools = await client.get_tools() return client, tools -async def discover_tools(client_row: McpClient, tenant_id: str, project_id: str) -> list[dict]: +async def discover_tools(client_row: McpClient, tenant_id: str, project_id: str, + context: dict | None = None) -> list[dict]: """List the tools an MCP server exposes - [{name, description}]. Connects fresh (not via the execution cache) so the result always reflects the current McpClient config, and drops any stale cached client so the next run reconnects with the latest settings. Raises McpUnavailable / connection errors. """ - _CLIENT_CACHE.pop(client_row.id, None) + await invalidate_client(client_row.id) MultiServerMCPClient = _require_adapters() - conn = await _connection_for(client_row, tenant_id, project_id) + conn = await _connection_for(client_row, tenant_id, project_id, context) client = MultiServerMCPClient({client_row.name: conn}) - tools = await client.get_tools() + # This client is deliberately NOT cached, so nothing else will ever close it. Discovery runs + # on every connect callback and every "Refresh actions" click, so dropping it on return + # would leak one transport per click. + try: + tools = await client.get_tools() + finally: + await _aclose(client) return [{"name": t.name, "description": (getattr(t, "description", "") or "").strip()} for t in tools] -async def server_tools(client_row: McpClient, tenant_id: str, project_id: str) -> list: +def describe_mcp_error(e: BaseException) -> str: + """A message that names what actually failed. + + The MCP client runs its transport inside an anyio task group, so EVERY failure - a 401 from + the server, a DNS miss, a TLS error - reaches the caller wrapped in an ExceptionGroup whose + str() is the famously uninformative "unhandled errors in a TaskGroup (1 sub-exception)". + Printing that in the UI tells a user nothing and tells us nothing either, so unwrap to the + leaf exceptions and name them. + """ + leaves: list[str] = [] + + def _walk(err: BaseException, depth: int = 0) -> None: + inner = getattr(err, "exceptions", None) + if inner and depth < 5: + for sub in inner: + _walk(sub, depth + 1) + return + text = str(err).strip() + leaves.append(f"{type(err).__name__}: {text}" if text else type(err).__name__) + + _walk(e) + # Deduplicate: a task group commonly reports the same underlying error from several tasks. + unique = list(dict.fromkeys(leaves)) + return "; ".join(unique[:3]) or str(e) or type(e).__name__ + + +async def server_tools(client_row: McpClient, tenant_id: str, project_id: str, + context: dict | None = None) -> list: """Native LangChain tools a server exposes, minus the ones toggled off (disabled_tools). - Used to attach a whole MCP server's tools to an agent.""" - _client, tools = await _client_and_tools(client_row, tenant_id, project_id) + Used to attach a whole MCP server's tools to an agent. `context` carries the run's per-user + dims so a per-user auth provider resolves THIS caller's credential.""" + _client, tools = await _client_and_tools(client_row, tenant_id, project_id, context) disabled = set(getattr(client_row, "disabled_tools", None) or []) return [t for t in tools if t.name not in disabled] +def auth_context_from(ctx) -> dict: + """The per-user dims an MCP auth provider keys on, read off a CompileContext. + + Mirrors the lane order the REST tool uses (tools/rest.py): injected run context first, then + the run's end_user identity last so it is authoritative and can't be shadowed by a value a + caller injected.""" + eu = getattr(ctx, "end_user", None) + return { + **(getattr(ctx, "run_context", None) or {}), + "end_user": eu, + "end_user_id": eu.get("id") if isinstance(eu, dict) else None, + "end_user_email": eu.get("email") if isinstance(eu, dict) else None, + } + + def _wrap_with_context_injection(tool, inject_keys: list[str]): """Wrap an MCP StructuredTool so `inject_keys` are filled from runtime.context (per-user secrets the widget/channel supplies) instead of from the model.""" @@ -180,7 +570,7 @@ async def load_mcp_tool(cfg: dict, ctx) -> Any: ).scalar_one_or_none() if row is None: raise McpUnavailable(f"MCP client {cfg.get('mcp_client_id')!r} not found") - _client, tools = await _client_and_tools(row, ctx.tenant_id, ctx.project_id) + _client, tools = await _client_and_tools(row, ctx.tenant_id, ctx.project_id, auth_context_from(ctx)) name = cfg["remote_tool_name"] match = next((t for t in tools if t.name == name), None) if match is None: diff --git a/apps/api/forge/tools/rest.py b/apps/api/forge/tools/rest.py index 50f25c8..fa42912 100644 --- a/apps/api/forge/tools/rest.py +++ b/apps/api/forge/tools/rest.py @@ -26,7 +26,12 @@ from langchain.tools import ToolRuntime from pydantic import Field, create_model -from forge.auth_providers.templates import has_each_directive, render_template, render_value +from forge.auth_providers.templates import ( + DIRECTIVES, + has_structural_directive, + render_template, + render_value, +) from forge.config import settings from forge.tools.projection import cap_payload, project_response from forge.tracing import tool_io @@ -224,6 +229,40 @@ def _collect(fields: list[dict], values: dict, where: str) -> dict: return out +def _declared_content_type(req: dict) -> str: + """The Content-Type the tool config declares, if any. Read from the raw header specs (these + are authored values, not templated per-call ones), lowercased.""" + for h in req.get("headers", []) or []: + if str(h.get("name", "")).lower() == "content-type": + return str(h.get("value", "")).lower() + return "" + + +def _template_is_json(req: dict, tmpl: str) -> bool: + """Whether a `body_template` should have its substituted strings JSON-escaped. + + This has to agree with `_resolve_body_encoding`, which decides how the body is SERIALIZED. + Two independent answers to "is this JSON?" can disagree - a `body_encoding: raw` template + that happens to open with `{` would get JSON-escaped and then sent verbatim, putting literal + backslashes in the payload - so both read the declared encoding first and only fall back to + inspecting the template when the tool declares nothing. + + The fallback is a character test rather than rendering twice and seeing which parses: a JSON + document opens with `{` or `[`, but so does a form-encoded template whose first thing is a + token (`{{input.q}}=1¬e={{input.n}}`), so a leading `{{` disqualifies it. + """ + enc = str(req.get("body_encoding") or "").strip().lower() + if enc in ("json", "form", "multipart", "raw"): + return enc == "json" + ct = _declared_content_type(req) + if "json" in ct: + return True + if ct: + return False # form-urlencoded, multipart, text/plain, xml - none of them JSON-escape + head = tmpl.lstrip() + return head[:1] in ("{", "[") and not head.startswith("{{") + + def _build_body(req: dict, fields: list[dict], values: dict, context: dict | None): """Request body. A free-form `body_template` takes precedence and is interpolated with two namespaces - `{{input.*}}` (the validated tool args + defaults) and `{{ctx.*}}` (run @@ -234,20 +273,26 @@ def _build_body(req: dict, fields: list[dict], values: dict, context: dict | Non tmpl = req.get("body_template") if tmpl: tvars = {"input": values, "ctx": context or {}, "env": settings.tool_vars} - # A `$each` loop directive needs STRUCTURAL rendering (parse the JSON, then walk it with - # render_value) so the produced array is always valid JSON with native types - plain string - # substitution can't build a variable-length array without trailing-comma/quoting bugs. - # Gate on an ACTUAL parsed `$each` directive (a dict key), NOT a substring of the raw text: - # a template that merely mentions "$each" inside a string value must keep the exact + # A `$each` loop (or a `$mime` message) needs STRUCTURAL rendering (parse the JSON, then + # walk it with render_value) so the produced value is always valid JSON with native types + # - plain string substitution can't build a variable-length array without trailing-comma/ + # quoting bugs, nor base64-encode a MIME message. + # Gate on an ACTUAL parsed directive (a dict key), NOT a substring of the raw text: a + # template that merely mentions "$each" inside a string value must keep the exact # string-substitution behavior (structural rendering coerces token types differently). - if "$each" in tmpl: + if any(d in tmpl for d in DIRECTIVES): try: parsed = _json.loads(tmpl) except ValueError: parsed = None - if parsed is not None and has_each_directive(parsed): + if parsed is not None and has_structural_directive(parsed): return render_value(parsed, tvars, allow_each=True, strict_ns=_STRICT_NS) - rendered = render_template(tmpl, tvars, strict_ns=_STRICT_NS) + # Substituted strings are JSON-escaped when the template IS a JSON document, so that + # model-written text containing a newline or a quote can't terminate a JSON string early + # - which is what silently turned a note body into an unparseable body, and then into raw + # text the API rejected. A form-encoded or plain-text template must NOT be escaped. + rendered = render_template(tmpl, tvars, strict_ns=_STRICT_NS, + escape_json=_template_is_json(req, tmpl)) if isinstance(rendered, (dict, list)): return rendered if isinstance(rendered, str): @@ -495,6 +540,23 @@ async def execute_rest( if isinstance(parsed, (list, dict)): values[f["path"]] = parsed + # A field may declare `extract`: a regex that pulls the real value out of something a PERSON + # would paste. Opaque ids (a Google Sheets key, a Notion page id) live inside a URL, and what + # a user actually says is "here's my sheet: https://docs.google.com/spreadsheets/d/1AbC…/edit". + # Without this the model either forwards the URL - which URL-encodes into a 404 - or, worse, + # invents an id from the document's NAME and gets a 404 that looks like a permissions problem. + # No match leaves the value untouched, so a bare id still passes straight through. + for f in fields: + pattern = f.get("extract") + v = values.get(f["path"]) + if not pattern or not isinstance(v, str) or not v: + continue + # Bound the subject before matching: the pattern is authored (trusted) but the value is + # model-supplied, and an unbounded string is what turns a sloppy regex into a stall. + m = re.search(pattern, v[:4096]) + if m: + values[f["path"]] = m.group(1) if m.groups() else m.group(0) + # {{ctx.*}} is honored in the URL itself too (e.g. a base host, or a ?token= carried in run # context); {name} path params are then substituted from `values` as before. url_t = render_template(req["url_template"], ctx_vars, strict_ns=_STRICT_NS) diff --git a/apps/api/migrations/versions/0011_connectors.py b/apps/api/migrations/versions/0011_connectors.py new file mode 100644 index 0000000..33eaa31 --- /dev/null +++ b/apps/api/migrations/versions/0011_connectors.py @@ -0,0 +1,67 @@ +"""Connectors: connector_installs table + mcp_clients.auth_provider_id + +A connector install is the receipt for a manifest that was expanded into ordinary rows +(AuthProvider + ToolSet + Tools + optionally an McpClient). The table records what was +created so uninstall/upgrade are exact rather than guesswork. + +`mcp_clients.auth_provider_id` lets an external MCP server be authenticated with a full Auth +Provider (OAuth with refresh, per-user bundles) instead of only a static `headers_ref` secret - +which is what makes OAuth-protected remote MCP servers reachable. + +`create_all` builds both on fresh dev DBs; this migration covers managed Postgres and +pre-existing tables. Idempotent: each piece is added only when absent. + +Revision ID: 0011_connectors +Revises: 0010_toolset_exposed +Create Date: 2026-08-13 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0011_connectors" +down_revision = "0010_toolset_exposed" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + insp = sa.inspect(op.get_bind()) + tables = set(insp.get_table_names()) + + if "connector_installs" not in tables: + op.create_table( + "connector_installs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.Column("tenant_id", sa.String(36), nullable=False, index=True), + sa.Column("project_id", sa.String(36), nullable=False, index=True), + sa.Column("slug", sa.String(120), nullable=False, index=True), + sa.Column("name", sa.String(120), nullable=False), + sa.Column("version", sa.String(40), nullable=False, server_default="1.0.0"), + sa.Column("source", sa.String(20), nullable=False, server_default="catalog"), + sa.Column("manifest", sa.JSON(), nullable=True), + sa.Column("auth_provider_id", sa.String(36), nullable=True), + sa.Column("tool_set_id", sa.String(36), nullable=True), + sa.Column("mcp_client_id", sa.String(36), nullable=True), + sa.Column("created_tool_ids", sa.JSON(), nullable=True), + sa.Column("created_secret_names", sa.JSON(), nullable=True), + sa.Column("status", sa.String(20), nullable=False, server_default="needs_setup"), + sa.Column("status_detail", sa.Text(), nullable=True), + sa.Column("auth_mode", sa.String(20), nullable=False, server_default="shared"), + sa.UniqueConstraint("tenant_id", "project_id", "slug", name="uq_connector_install_slug"), + ) + + if "mcp_clients" in tables: + cols = {c["name"] for c in insp.get_columns("mcp_clients")} + if "auth_provider_id" not in cols: + op.add_column("mcp_clients", sa.Column("auth_provider_id", sa.String(36), nullable=True)) + + +def downgrade() -> None: + insp = sa.inspect(op.get_bind()) + tables = set(insp.get_table_names()) + if "mcp_clients" in tables and "auth_provider_id" in {c["name"] for c in insp.get_columns("mcp_clients")}: + op.drop_column("mcp_clients", "auth_provider_id") + if "connector_installs" in tables: + op.drop_table("connector_installs") diff --git a/apps/api/migrations/versions/0012_trigger_run_as.py b/apps/api/migrations/versions/0012_trigger_run_as.py new file mode 100644 index 0000000..cd91816 --- /dev/null +++ b/apps/api/migrations/versions/0012_trigger_run_as.py @@ -0,0 +1,40 @@ +"""Triggers: run_as_user_id - whose connected accounts an unattended run uses + +A webhook or a schedule has no signed-in person. Every catalog connector is per-user, so +without an identity on the trigger a scheduled run has no token to resolve and fails at its +first tool call. This column names the user an unattended run acts as - the editor who saved +the workflow, reassignable afterwards. + +NULL is the pre-existing behaviour (no identity), so this is safe to apply to a live DB: nothing +changes until a workflow is next saved, which stamps the owner. + +`create_all` builds it on fresh dev DBs; this migration covers managed Postgres. Idempotent. + +Revision ID: 0012_trigger_run_as +Revises: 0011_connectors +Create Date: 2026-08-14 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0012_trigger_run_as" +down_revision = "0011_connectors" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "triggers" not in set(insp.get_table_names()): + return + cols = {c["name"] for c in insp.get_columns("triggers")} + if "run_as_user_id" not in cols: + op.add_column("triggers", sa.Column("run_as_user_id", sa.String(36), nullable=True)) + + +def downgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "triggers" not in set(insp.get_table_names()): + return + if "run_as_user_id" in {c["name"] for c in insp.get_columns("triggers")}: + op.drop_column("triggers", "run_as_user_id") diff --git a/apps/api/migrations/versions/0013_trigger_scope.py b/apps/api/migrations/versions/0013_trigger_scope.py new file mode 100644 index 0000000..6a746b2 --- /dev/null +++ b/apps/api/migrations/versions/0013_trigger_scope.py @@ -0,0 +1,44 @@ +"""Triggers: scope - a team automation, or someone's own + +Forge projects are shared, but not every automation in one is. A salesperson's lead-chaser is +theirs; a platform team's prod-monitor belongs to the project. `scope` says which, and drives +who the Triggers screen lists it for. + +Existing rows become "project", which is exactly how they behave today - everyone in the project +already sees every trigger - so this migration changes no observable behaviour on its own. + +`create_all` builds it on fresh dev DBs; this migration covers managed Postgres. Idempotent. + +Revision ID: 0013_trigger_scope +Revises: 0012_trigger_run_as +Create Date: 2026-08-14 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0013_trigger_scope" +down_revision = "0012_trigger_run_as" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "triggers" not in set(insp.get_table_names()): + return + if "scope" in {c["name"] for c in insp.get_columns("triggers")}: + return + # server_default backfills every existing row in one statement; the model's own default + # takes over for new rows. + op.add_column( + "triggers", + sa.Column("scope", sa.String(10), nullable=False, server_default="project"), + ) + + +def downgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "triggers" not in set(insp.get_table_names()): + return + if "scope" in {c["name"] for c in insp.get_columns("triggers")}: + op.drop_column("triggers", "scope") diff --git a/apps/api/migrations/versions/0014_connector_egress_hosts.py b/apps/api/migrations/versions/0014_connector_egress_hosts.py new file mode 100644 index 0000000..a4d9671 --- /dev/null +++ b/apps/api/migrations/versions/0014_connector_egress_hosts.py @@ -0,0 +1,43 @@ +"""Connectors: record which egress hosts an install actually added + +`install` appends a connector's hosts to the project's egress allow-list, and uninstall had no +matching removal - so on a deployment running a strict allow-list the list only ever grew, and +hosts stayed reachable with nothing referencing them. + +Removing `manifest.hosts()` on the way out would be wrong the other way: a host somebody +allow-listed by hand before the connector existed is not the install's to take away, and after +the fact the two are indistinguishable. So the install records exactly what it added. + +Existing rows get an empty list, which means "unknown" - uninstall leaves their hosts alone +rather than guessing. + +`create_all` builds it on fresh dev DBs; this migration covers managed Postgres. Idempotent. + +Revision ID: 0014_connector_egress_hosts +Revises: 0013_trigger_scope +Create Date: 2026-08-16 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0014_connector_egress_hosts" +down_revision = "0013_trigger_scope" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "connector_installs" not in set(insp.get_table_names()): + return + if "created_egress_hosts" in {c["name"] for c in insp.get_columns("connector_installs")}: + return + op.add_column("connector_installs", sa.Column("created_egress_hosts", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + insp = sa.inspect(op.get_bind()) + if "connector_installs" not in set(insp.get_table_names()): + return + if "created_egress_hosts" in {c["name"] for c in insp.get_columns("connector_installs")}: + op.drop_column("connector_installs", "created_egress_hosts") diff --git a/apps/api/tests/test_body_template_loop.py b/apps/api/tests/test_body_template_loop.py index aa474f3..0b4c35c 100644 --- a/apps/api/tests/test_body_template_loop.py +++ b/apps/api/tests/test_body_template_loop.py @@ -1,19 +1,22 @@ -"""`$each` loop support in JSON body templates (batch many rows in one call). +"""Structural directives in JSON body templates - `$each` (batch rows) and `$mime` (email). -Before this, one REST tool call could only build a fixed-shape body, so an agent editing N rows -made N calls (slow, context-heavy, and prone to blowing the graph recursion limit). A body -template can now carry a `{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` directive that -expands one list-valued arg into a variable-length JSON array - so N edits go out in ONE request. +Both exist for the same reason: a value the model cannot be asked to produce as text. + +* `{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` expands one list-valued arg into a + variable-length JSON array, so an agent editing N rows makes ONE call instead of N (slow, + context-heavy, and prone to blowing the graph recursion limit). +* `{"$mime": {"to": …, "subject": …, "text": …}}` builds an RFC 2822 message and base64url-encodes + it, because Gmail's send endpoint accepts nothing else and a model cannot base64-encode by hand. Rendering is structural (parse JSON, then walk with render_value): the output is always valid JSON with native types preserved, unlike string-concatenating an array. The path is opt-in on the -"$each" marker, so existing string-substitution templates are untouched. +directive marker, so existing string-substitution templates are untouched. """ from __future__ import annotations import json -from forge.auth_providers.templates import has_each_directive, render_template, render_value +from forge.auth_providers.templates import has_structural_directive, render_template, render_value from forge.tools.rest import _build_body @@ -62,10 +65,10 @@ def test_render_value_each_is_opt_in_only(): assert out == {"$each": [{"x": "a"}], "$as": "row", "$do": {"x": None}} -def test_has_each_directive_ignores_literal_string_value(): +def test_structural_directive_detection_ignores_literal_string_values(): # A directive is a "$each" KEY; the literal text "$each" inside a string value is not one. - assert has_each_directive({"note": "use $each to loop", "qty": "{{input.qty}}"}) is False - assert has_each_directive({"rows": {"$each": "{{x}}", "$do": {}}}) is True + assert has_structural_directive({"note": "use $each to loop", "qty": "{{input.qty}}"}) is False + assert has_structural_directive({"rows": {"$each": "{{x}}", "$do": {}}}) is True def test_build_body_literal_dollar_each_in_string_keeps_string_substitution(): @@ -157,3 +160,179 @@ def test_build_body_without_each_is_unchanged(): body_template = '{ "orderId": "{{input.orderId}}", "lineNums": [ {{input.lineNum}} ] }' body = _build_body({"body_template": body_template}, [], {"orderId": "Q", "lineNum": 7}, {}) assert body == {"orderId": "Q", "lineNums": [7]} + + +# --- $mime: build an RFC 2822 message server-side -------------------------------------------- +# +# Gmail's send endpoint accepts ONLY a base64url-encoded MIME message. Declaring that as a tool +# argument means asking a model to base64-encode by hand; it can't, and the malformed result +# comes back as an opaque HTTP 400. So the model supplies to/subject/body and this does the rest. + +def _decoded(body: dict): + import base64 + import email + + raw = body["raw"] + blob = base64.urlsafe_b64decode(raw) + return blob, email.message_from_bytes(blob) + + +def test_mime_directive_builds_a_decodable_message(): + tmpl = ('{"raw": {"$mime": {"to": "{{input.to}}", "subject": "{{input.subject}}", ' + '"text": "{{input.body}}"}}}') + body = _build_body({"body_template": tmpl}, [], + {"to": "a@example.com", "subject": "Hello", "body": "Line one\nLine two"}, {}) + assert list(body) == ["raw"] + blob, msg = _decoded(body) + assert msg["To"] == "a@example.com" + assert msg["Subject"] == "Hello" + # RFC 2822 wants CRLF everywhere, headers and body alike, and the SMTP policy normalises the + # body's newlines to match. Mail clients render that as ordinary line breaks. + assert msg.get_payload(decode=True).decode().replace("\r\n", "\n").strip() == "Line one\nLine two" + assert b"\r\n" in blob + + +def test_mime_omits_empty_headers_rather_than_sending_them_blank(): + """A model that leaves cc out sends "" - and `Cc: ` on the wire is what makes Gmail answer + 400 rather than simply having no Cc.""" + tmpl = ('{"raw": {"$mime": {"to": "{{input.to}}", "cc": "{{input.cc}}", ' + '"subject": "{{input.subject}}", "text": "{{input.body}}"}}}') + body = _build_body({"body_template": tmpl}, [], + {"to": "a@example.com", "cc": "", "subject": "S", "body": "B"}, {}) + _blob, msg = _decoded(body) + assert msg["Cc"] is None + assert msg["To"] == "a@example.com" + + +def test_mime_handles_address_lists_and_non_ascii_subjects(): + tmpl = ('{"raw": {"$mime": {"to": "{{input.to}}", "subject": "{{input.subject}}", ' + '"text": "{{input.body}}"}}}') + body = _build_body({"body_template": tmpl}, [], + {"to": ["a@example.com", "b@example.com"], "subject": "Update ✅", "body": "x"}, {}) + _blob, msg = _decoded(body) + assert msg["To"] == "a@example.com, b@example.com" + # Non-ASCII must be RFC 2047 encoded, and must decode back to what was asked for. + from email.header import decode_header, make_header + assert str(make_header(decode_header(msg["Subject"]))) == "Update ✅" + + +def test_mime_is_inert_without_the_structural_opt_in(): + """Every other caller (auth token_fetch rules, data-node payloads) must keep treating a + literal "$mime" key as an ordinary key rather than executing it.""" + from forge.auth_providers.templates import render_value + + tmpl = {"raw": {"$mime": {"to": "{{input.to}}"}}} + out = render_value(tmpl, {"input": {"to": "a@example.com"}}) + assert out == {"raw": {"$mime": {"to": "a@example.com"}}} + + +def test_a_literal_dollar_mime_string_does_not_trigger_structural_rendering(): + body_template = '{ "note": "use $mime for email", "qty": {{input.qty}} }' + body = _build_body({"body_template": body_template}, [], {"qty": 3}, {}) + assert body == {"note": "use $mime for email", "qty": 3} + + +# --- interpolating a list/dict into a JSON body ---------------------------------------------- +# +# `str()` on a list yields Python's repr - single quotes, True/None - which is not JSON. The body +# then fails to parse and is sent as raw text, and the API answers 400. It only ever showed up on +# STRING data: `[[1, 2]]` is valid JSON by coincidence, `[['a', 'b']]` is not. + +def test_embedded_list_renders_as_json_not_python_repr(): + body = _build_body({"body_template": '{"values":{{input.rows}}}'}, + [{"path": "rows", "in": "body", "type": "array"}], + {"rows": [["Test Data 1", "Test Data 2"]]}, {}) + assert body == {"values": [["Test Data 1", "Test Data 2"]]} + + +def test_embedded_dict_renders_as_json(): + body = _build_body({"body_template": '{"fields":{{input.fields}}}'}, + [{"path": "fields", "in": "body", "type": "object"}], + {"fields": {"Name": "Ada", "Active": True, "Notes": None}}, {}) + assert body == {"fields": {"Name": "Ada", "Active": True, "Notes": None}} + + +def test_embedded_list_of_numbers_still_works(): + """The case that accidentally passed before, which is why this went unnoticed.""" + body = _build_body({"body_template": '{"values":{{input.rows}}}'}, + [{"path": "rows", "in": "body", "type": "array"}], + {"rows": [[1, 2]]}, {}) + assert body == {"values": [[1, 2]]} + + +def test_embedded_scalars_keep_their_plain_string_form(): + """Only containers change. A bare token in a query string still renders "False"/"0", which is + what that context wants - see test_render_template_embedded_falsy_values_are_not_dropped.""" + assert render_template("on={{input.flag}}", {"input": {"flag": False}}) == "on=False" + assert render_template("qty={{input.qty}}", {"input": {"qty": 0}}) == "qty=0" + + +def test_non_ascii_in_an_interpolated_list_is_not_escaped_away(): + body = _build_body({"body_template": '{"values":{{input.rows}}}'}, + [{"path": "rows", "in": "body", "type": "array"}], + {"rows": [["café", "naïve"]]}, {}) + assert body == {"values": [["café", "naïve"]]} + + +def test_form_encoded_template_starting_with_a_token_is_not_json_escaped(): + """A JSON body opens with `{`, but so does a form template whose first thing is a token. + Escaping that one puts a literal backslash into the form body.""" + body = _build_body({"body_template": "{{input.q}}=1¬e={{input.n}}"}, [], + {"q": 'a"b', "n": "x"}, {}) + assert body == 'a"b=1¬e=x' + + +def test_a_json_template_is_still_escaped(): + body = _build_body({"body_template": '{"q":"{{input.q}}"}'}, [], {"q": 'a"b'}, {}) + assert body == {"q": 'a"b'} + + +def test_a_bare_token_template_keeps_its_native_value(): + """`{{input.payload}}` alone is a whole-string match, so it resolves to the object itself and + never goes near the escaping path - even though it also starts with `{`.""" + body = _build_body({"body_template": "{{input.payload}}"}, [], + {"payload": {"a": 'q"uote', "b": [1, 2]}}, {}) + assert body == {"a": 'q"uote', "b": [1, 2]} + + +def test_a_declared_body_encoding_decides_the_escaping_not_the_first_character(): + """Escaping (`_build_body`) and serialization (`_resolve_body_encoding`) are two answers to + the same question - is this body JSON? - and they must not be able to disagree. A tool that + DECLARES its encoding has already answered; sniffing the template instead would JSON-escape + a raw payload that merely happens to open with `{`, then send it verbatim with the + backslashes in it.""" + tmpl = '{"q":"{{input.q}}"}' + for enc in ("raw", "form"): + body = _build_body({"body_template": tmpl, "body_encoding": enc}, [], {"q": 'a"b'}, {}) + assert body == '{"q":"a"b"}', f"body_encoding={enc} must not be JSON-escaped" + assert _build_body({"body_template": tmpl, "body_encoding": "json"}, [], {"q": 'a"b'}, {}) == {"q": 'a"b'} + + +def test_a_declared_content_type_decides_the_escaping_too(): + """The declared Content-Type is the other place a tool states what it is sending.""" + form = _build_body( + {"body_template": "{{input.q}}=1", + "headers": [{"name": "Content-Type", "value": "application/x-www-form-urlencoded"}]}, + [], {"q": 'a"b'}, {}, + ) + assert form == 'a"b=1' + js = _build_body( + {"body_template": '{"q":"{{input.q}}"}', + "headers": [{"name": "Content-Type", "value": "application/json"}]}, + [], {"q": 'a"b'}, {}, + ) + assert js == {"q": 'a"b'} + + +def test_escaping_and_serialization_agree_on_every_declared_encoding(): + """A guard against the two decisions drifting apart again: whenever the tool declares an + encoding, `_build_body`'s escaping choice must match what `_resolve_body_encoding` will + actually do with the result.""" + from forge.tools.rest import _resolve_body_encoding, _template_is_json + + for enc in ("json", "form", "multipart", "raw"): + req = {"body_template": '{"q":"{{input.q}}"}', "body_encoding": enc} + body = _build_body(req, [], {"q": "ab"}, {}) + assert _template_is_json(req, req["body_template"]) == ( + _resolve_body_encoding(req, {}, body) == "json" + ), f"escaping and serialization disagree for body_encoding={enc}" diff --git a/apps/api/tests/test_catalog_bodies.py b/apps/api/tests/test_catalog_bodies.py new file mode 100644 index 0000000..4fa6863 --- /dev/null +++ b/apps/api/tests/test_catalog_bodies.py @@ -0,0 +1,107 @@ +r'''Every catalog write action must build a valid JSON body from realistic model output. + +Three shipped connectors were broken in ways inspection missed, all the same shape - a template +that produced text the API could not parse, surfacing as an opaque HTTP 400: + + * a list interpolated with str() became Python repr (single quotes), so `[['a','b']]` was not + JSON while `[[1, 2]]` was - the bug only appeared on string data; + * Sheets' append wrapped an already-2-D `values` into a 3-D array; + * a note body containing a newline terminated its JSON string early. + +So this sweeps the whole catalog with deliberately awkward values - newlines, double quotes, +backslashes, multiple recipients, many rows - and asserts the body parses. It is a shape test, +not a mock of the vendor: what it guards is that Forge sends JSON at all. +''' + +from __future__ import annotations + +import pytest + +from forge.connectors.catalog import list_manifests +from forge.connectors.manifest import RestBackend +from forge.tools.rest import _build_body + +#: What a model plausibly produces, chosen to be hostile to naive string interpolation. +SAMPLE: dict = { + "rows": [["a", "b"], ["c", "d"]], + "to": ["a@example.com", "b@example.com"], + "cc": ["c@example.com"], + "bcc": [], + "subject": 'Re: "urgent" plan', + "body": 'Line one\nHe said "hi"\\done', + "comment": "thanks!\nbye", + "fields": {"Name": "Ada", "Count": 3}, + "query": 'acme "corp"', + "limit": 10, + "timestamp": "2026-08-14T10:00:00Z", + "summary": "Sync", + "description": "line one\nline two", + "start": "2026-08-20T14:00:00+05:30", + "end": "2026-08-20T15:00:00+05:30", + "attendees": ["a@example.com", "b@example.com"], + "timeMin": "2026-08-20T00:00:00Z", + "timeMax": "2026-08-21T00:00:00Z", + "calendars": ["primary"], + "title": 'Bug: "crash" on save', + "labels": ["bug", "p1"], + "addLabelIds": ["INBOX"], + "removeLabelIds": ["UNREAD"], +} + + +def _write_actions(): + for m in list_manifests(): + if not isinstance(m.backend, RestBackend): + continue + for a in m.backend.actions: + body_fields = [f for f in a.request.get("fields", []) if f.get("in") == "body"] + if body_fields or a.request.get("body_template"): + yield pytest.param(m.slug, a, id=f"{m.slug}:{a.name}") + + +@pytest.mark.parametrize("slug,action", list(_write_actions())) +def test_write_action_builds_parseable_json(slug: str, action): + body_fields = [f for f in action.request.get("fields", []) if f.get("in") == "body"] + unknown = [f["path"] for f in body_fields if f["path"] not in SAMPLE] + assert not unknown, ( + f"{slug}.{action.name}: no sample for {unknown} - add one so this action stays covered" + ) + args = {f["path"]: SAMPLE[f["path"]] for f in body_fields} + built = _build_body(action.request, action.request.get("fields", []), args, {}) + assert isinstance(built, (dict, list)), ( + f"{slug}.{action.name} produced text, not JSON - the API will reject it: {built!r}" + ) + + +def _action(slug: str, name: str): + m = next(x for x in list_manifests() if x.slug == slug) + return next(a for a in m.backend.actions if a.name == name) + + +def test_awkward_text_survives_the_round_trip_unmangled(): + """Parsing is necessary but not sufficient - the text must also arrive as written.""" + a = _action("hubspot", "hubspot_create_note") + body = _build_body(a.request, a.request["fields"], + {"body": SAMPLE["body"], "timestamp": SAMPLE["timestamp"]}, {}) + assert body["properties"]["hs_note_body"] == SAMPLE["body"] + + +def test_outlook_send_takes_every_recipient_not_just_the_first(): + """toRecipients was a hardcoded single object, so "send this to alice and bob" was impossible.""" + a = _action("outlook", "outlook_send_message") + body = _build_body(a.request, a.request["fields"], + {"to": SAMPLE["to"], "cc": SAMPLE["cc"], "bcc": [], + "subject": "s", "body": "b"}, {}) + msg = body["message"] + assert [r["emailAddress"]["address"] for r in msg["toRecipients"]] == SAMPLE["to"] + assert [r["emailAddress"]["address"] for r in msg["ccRecipients"]] == SAMPLE["cc"] + assert msg["bccRecipients"] == [] + + +def test_calendar_attendees_become_email_objects(): + a = _action("google-calendar", "gcal_create_event") + body = _build_body(a.request, a.request["fields"], + {"summary": "s", "description": "", "start": SAMPLE["start"], + "end": SAMPLE["end"], "attendees": SAMPLE["attendees"]}, {}) + assert body["attendees"] == [{"email": e} for e in SAMPLE["attendees"]] + assert body["start"] == {"dateTime": SAMPLE["start"]} diff --git a/apps/api/tests/test_connectors.py b/apps/api/tests/test_connectors.py new file mode 100644 index 0000000..2bcfda2 --- /dev/null +++ b/apps/api/tests/test_connectors.py @@ -0,0 +1,1459 @@ +"""Connectors: manifest validation, the bundled catalog, and the install/uninstall pipeline. + +The load-bearing claims these tests pin down: + + * loading the catalog performs NO network I/O (the independence guarantee), + * installing a manifest creates ordinary AuthProvider/ToolSet/Tool rows - nothing bespoke, + * uninstalling removes exactly what the install created and nothing else, + * a failed install leaves no orphan rows behind, + * every CATALOG connector is one-click: OAuth, per-user, and credential-free at the point of + use - the deployment's environment is the only place a vendor app can come from, + * a CUSTOM (pasted) manifest keeps the old behaviour, because a service account with an API + key has to be typed by somebody and that is the path where it belongs. +""" + +from __future__ import annotations + +from contextlib import aclosing + +import pytest + +from forge.connectors.catalog import get_manifest, list_examples, list_manifests +from forge.connectors.install import ( + ConnectorInstaller, + InstallError, + env_ready, + group_has_credentials, + missing_app_keys, + secret_name, +) +from forge.connectors.manifest import ManifestError, RestBackend, parse_manifest +from forge.db.base import SessionLocal +from forge.models import AuthProvider, ConnectorInstall, Tool, ToolSet +from forge.secrets.store import SecretStore +from forge.services.tool_sets import ToolSetService + +REST_MANIFEST = { + "format": "forge.connector/1", + "slug": "acme", + "name": "Acme", + "summary": "Acme test connector", + "categories": ["custom"], + "roles": ["Software Engineer"], + "auth": { + "kind": "bearer", + "per_user": "optional", + "setup": [{"key": "token", "label": "API token", "secret": True, "required": True}], + }, + "egress_hosts": ["api.acme.test"], + "backend": { + "type": "rest", + "base_url": "https://api.acme.test/v1", + "actions": [ + { + "name": "acme_list_widgets", + "description": "List widgets.", + "request": { + "method": "GET", + "url_template": "/widgets", + "fields": [{"path": "limit", "in": "query", "type": "integer", "llm_visible": True}], + }, + "response": {"projection_jmespath": "items[].id"}, + }, + { + "name": "acme_get_widget", + "description": "Read one widget.", + "request": { + "method": "GET", + "url_template": "/widgets/{widget_id}", + "fields": [{"path": "widget_id", "in": "path", "type": "string", "required": True}], + }, + }, + ], + }, + "toolset": {"description": "Acme actions"}, +} + + +# --- manifest validation -------------------------------------------------------------------- + +def test_manifest_rejects_unknown_format(): + with pytest.raises(ManifestError): + parse_manifest({**REST_MANIFEST, "format": "forge.connector/9"}) + + +def test_manifest_rejects_action_without_url_template(): + bad = { + **REST_MANIFEST, + "backend": { + "type": "rest", + "actions": [{"name": "x", "request": {"method": "GET"}}], + }, + } + with pytest.raises(ManifestError): + parse_manifest(bad) + + +def test_manifest_rejects_bad_field_location(): + bad = { + **REST_MANIFEST, + "backend": { + "type": "rest", + "actions": [{ + "name": "x", + "request": {"method": "GET", "url_template": "/x", + "fields": [{"path": "a", "in": "nowhere"}]}, + }], + }, + } + with pytest.raises(ManifestError): + parse_manifest(bad) + + +def test_manifest_rejects_duplicate_action_names(): + action = REST_MANIFEST["backend"]["actions"][0] + with pytest.raises(ManifestError): + parse_manifest({**REST_MANIFEST, + "backend": {"type": "rest", "actions": [action, action]}}) + + +def test_manifest_hosts_include_backend_host_even_when_undeclared(): + m = parse_manifest({**REST_MANIFEST, "egress_hosts": []}) + assert "api.acme.test" in m.hosts() + + +# --- bundled catalog ------------------------------------------------------------------------ + +def test_bundled_catalog_all_parse(): + manifests = list_manifests() + assert len(manifests) >= 10, "the bundled catalog should ship a usable set of connectors" + slugs = {m.slug for m in manifests} + for expected in ("slack", "gmail", "outlook", "github"): + assert expected in slugs + + +def test_every_catalog_connector_is_one_click(): + """The gallery's promise, enforced at the catalog level: click Connect, sign in with your own + account, done. A connector that needs a typed key or a shared bot token can't keep that + promise, so it belongs in examples/ - not on a screen that says there are no forms.""" + for m in list_manifests(): + assert m.auth.kind == "oauth2_authorization_code", ( + f"{m.slug}: catalog connectors sign users in; {m.auth.kind} needs a pasted credential" + ) + assert m.auth.per_user == "required", ( + f"{m.slug}: a catalog account is personal, so the manifest must say so" + ) + # Nothing an END USER has to supply. Non-secret setup values are allowed only when the + # manifest carries a default (Microsoft's tenant = "common"), because the install form + # they would otherwise be typed into no longer exists. + for field in m.auth.setup: + if field.secret: + assert field.key in ("client_id", "client_secret"), ( + f"{m.slug}: secret setup key {field.key!r} can't come from the environment" + ) + else: + assert field.default is not None, ( + f"{m.slug}: non-secret setup key {field.key!r} has no default and nobody to ask" + ) + + +#: Vendors that only issue a refresh_token when the authorize URL explicitly asks for one, keyed +#: by the authorize host that identifies them. Anything not listed here either refreshes by +#: default (Airtable, HubSpot), never expires (GitHub), or asks via a scope (Microsoft's +#: `offline_access`) - so this is a list of the vendors where SILENCE means a one-hour connector. +_OFFLINE_ACCESS_REQUIRED = {"accounts.google.com": {"access_type": "offline", "prompt": "consent"}} + + +def test_connectors_that_need_offline_access_ask_for_it(): + """Google issues a refresh_token ONLY when `access_type=offline` is on the authorize URL, and + re-issues it reliably only with `prompt=consent`. + + Without them the stored bundle has `refresh_token=None`, and `AuthResolver._oauth2_auth_code` + gates refresh on that key - so it silently keeps handing out an access token that expired. + Every Gmail / Calendar / Drive / Sheets call 401s about an hour after someone connects, and + the only cure is Disconnect + reconnect. Nothing else in the suite would notice. + """ + checked = 0 + for m in list_manifests(): + host = (m.auth.authorize_url or "").split("/")[2] if "://" in (m.auth.authorize_url or "") else "" + expected = _OFFLINE_ACCESS_REQUIRED.get(host) + if not expected: + continue + checked += 1 + for key, value in expected.items(): + assert m.auth.authorize_params.get(key) == value, ( + f"{m.slug}: {host} only returns a refresh_token when the authorize URL carries " + f"{key}={value}; without it every tool 401s an hour after sign-in" + ) + assert checked >= 4, "the four Google connectors should all be covered by this sweep" + + +async def test_the_google_authorize_url_actually_carries_access_type(): + """The manifest assertion above is necessary but not sufficient - it would still pass if + `_auth_config` stopped copying `authorize_params` onto the provider, or `build_authorize_url` + stopped applying it. So walk the real chain the Connect button walks, and read the query + string that Google would actually receive. + """ + from urllib.parse import parse_qs, urlparse + + from forge.auth_providers.oauth_flow import build_authorize_url + from forge.connectors.install import _auth_config + + manifest = get_manifest("gmail") + cfg = _auth_config(manifest, {}, per_user=True) + ap = AuthProvider(id="ap_gmail_test", tenant_id="t", project_id="p", + name="Gmail", kind=manifest.auth.kind, config=cfg) + + class _Store: + async def read_ref(self, **kw): + return "client-id.apps.googleusercontent.com" + + url = await build_authorize_url(ap, tenant_id="t", project_id="p", secrets=_Store()) + q = parse_qs(urlparse(url).query) + assert q.get("access_type") == ["offline"], f"authorize URL omits access_type: {sorted(q)}" + assert q.get("prompt") == ["consent"], f"authorize URL omits prompt: {sorted(q)}" + # And the extras must not have been able to trample the protocol parameters. + assert q["code_challenge_method"] == ["S256"] + + +def test_no_catalog_action_asks_the_model_to_encode_something(): + """A model cannot base64-encode by hand. It will emit plausible-looking garbage, the API + answers an opaque 400, and the user has no way to tell whose fault it is. + + Gmail's send endpoint is the case that bit us: it takes only a base64url-encoded MIME + message, and the first manifest exposed `raw` as a tool argument. Encoding belongs on the + server (the `$mime` body directive), so no LLM-visible argument may ask for one. + """ + banned = ("base64", "b64", "encoded", "rfc 2822", "rfc2822", "rfc 5322") + for m in list_manifests(): + if not isinstance(m.backend, RestBackend): + continue + for action in m.backend.actions: + for field in action.request.get("fields", []): + if field.get("llm_visible") is False: + continue + blob = f"{field.get('path', '')} {field.get('description', '')}".lower() + hit = next((b for b in banned if b in blob), None) + assert hit is None, ( + f"{m.slug}.{action.name}: argument {field.get('path')!r} asks the model for " + f"{hit!r}-encoded input; encode it server-side instead" + ) + + +def test_gmail_send_takes_the_fields_a_person_would_type(): + """The regression guard for the 400: Gmail send must expose to/subject/body, and build the + MIME message itself.""" + send = next(a for a in get_manifest("gmail").backend.actions if a.name == "gmail_send_message") + args = {f["path"] for f in send.request["fields"]} + assert {"to", "subject", "body"} <= args + assert "raw" not in args + assert "$mime" in (send.request.get("body_template") or "") + + +def test_examples_are_not_in_the_gallery(): + """The bundled key/token connectors are still shipped - just via the custom path, where a + person typing a credential is the expected thing rather than a broken promise.""" + gallery = {m.slug for m in list_manifests()} + examples = {m.slug for m, _ in list_examples()} + assert examples, "the example manifests should still ship" + assert not (gallery & examples), "an example must not also be a gallery entry" + for expected in ("stripe", "slack-api", "custom-rest-api"): + assert expected in examples + + +def test_catalog_load_does_no_network_io(monkeypatch): + """The independence guarantee, enforced: opening the catalog must never touch the network.""" + import socket + + def _boom(*a, **kw): # pragma: no cover - only runs if the guarantee breaks + raise AssertionError("catalog load attempted network I/O") + + monkeypatch.setattr(socket, "socket", _boom) + monkeypatch.setattr(socket, "create_connection", _boom) + from forge.connectors.catalog import reload_catalog + + reload_catalog() + assert list_manifests() + + +def test_every_catalog_action_declares_llm_visible_required_args(): + """A required field the model can't see can never be filled, so the tool would always 400.""" + for m in list_manifests(): + if not isinstance(m.backend, RestBackend): + continue + for action in m.backend.actions: + for field in action.request.get("fields", []): + if field.get("required") and field.get("llm_visible") is False: + assert field.get("default") is not None, ( + f"{m.slug}.{action.name}: field {field['path']} is required and hidden " + "from the model but has no default" + ) + + +# --- install / uninstall -------------------------------------------------------------------- + +async def _install(tenant: str, project: str, manifest_dict: dict | None = None, **kw): + """Install REST_MANIFEST down the CUSTOM path - a pasted manifest with a typed credential, + which is the only path that still accepts one.""" + manifest = parse_manifest(manifest_dict or REST_MANIFEST) + kw.setdefault("source", "custom") + async with SessionLocal() as s: + return await ConnectorInstaller().install( + s, tenant, project, manifest, values={"token": "sekrit"}, **kw + ) + + +async def _install_catalog(tenant: str, project: str, slug: str): + async with SessionLocal() as s: + return await ConnectorInstaller().install( + s, tenant, project, get_manifest(slug), source="catalog" + ) + + +@pytest.fixture +def google_app(monkeypatch): + """The deployment has registered one Google OAuth app - the normal state for a Forge that + actually offers Gmail.""" + from forge.config import settings + + monkeypatch.setattr( + settings, "connector_oauth_apps", + {"google": {"client_id": "deployment-cid", "client_secret": "deployment-csec"}}, + ) + + +@pytest.fixture +def no_apps(monkeypatch): + """A deployment that has registered nothing. + + Pinned explicitly rather than assumed: settings are read from the developer's own .env, so a + machine that HAS configured a vendor would otherwise fail these tests - and, worse, print a + real client secret into the assertion diff.""" + from forge.config import settings + + monkeypatch.setattr(settings, "connector_oauth_apps", {}) + + +async def test_install_creates_auth_provider_toolset_and_tools(): + tenant, project = "t_conn_i", "p_conn_i" + row = await _install(tenant, project) + + assert row.status == "connected" # bearer needs no browser round trip + assert len(row.created_tool_ids) == 2 + + async with SessionLocal() as s: + ap = await s.get(AuthProvider, row.auth_provider_id) + assert ap is not None and ap.kind == "bearer" + # The credential is a ref, never a literal - the same rule bundles follow. + assert ap.config["token_ref"] == f"secret://proj/{secret_name('acme', 'token')}" + + ts = await s.get(ToolSet, row.tool_set_id) + assert ts is not None + members = await ToolSetService.member_ids(s, tenant, ts.id) + assert sorted(members) == sorted(row.created_tool_ids) + + tools = [await s.get(Tool, tid) for tid in row.created_tool_ids] + assert {t.name for t in tools} == {"acme_list_widgets", "acme_get_widget"} + for t in tools: + assert t.kind == "rest_api" + assert t.auth_provider_id == row.auth_provider_id + # base_url composed with the relative action path. + assert t.config["request"]["url_template"].startswith("https://api.acme.test/v1/widgets") + + +async def test_installed_secret_is_readable_by_the_resolver_ref(): + tenant, project = "t_conn_s", "p_conn_s" + await _install(tenant, project) + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('acme', 'token')}" + ) + assert value == "sekrit" + + +async def test_install_is_rejected_twice(): + tenant, project = "t_conn_d", "p_conn_d" + await _install(tenant, project) + with pytest.raises(InstallError): + await _install(tenant, project) + + +async def test_install_requires_declared_credentials(): + tenant, project = "t_conn_r", "p_conn_r" + manifest = parse_manifest(REST_MANIFEST) + async with SessionLocal() as s: + with pytest.raises(InstallError): + await ConnectorInstaller().install(s, tenant, project, manifest, values={}, source="custom") + + +async def test_per_user_mode_sets_the_context_key_the_resolver_reads(): + tenant, project = "t_conn_pu", "p_conn_pu" + row = await _install(tenant, project, auth_mode="per_user") + assert row.auth_mode == "per_user" + async with SessionLocal() as s: + ap = await s.get(AuthProvider, row.auth_provider_id) + assert ap.config["per_user_context_keys"] == ["end_user_id"] + + +async def test_manifest_forcing_per_user_ignores_a_shared_request(): + """`per_user: required` exists so a personal-mailbox connector cannot be installed with one + shared credential by accident.""" + forced = {**REST_MANIFEST, "slug": "acme-pu", + "auth": {**REST_MANIFEST["auth"], "per_user": "required"}} + row = await _install("t_conn_f", "p_conn_f", forced, auth_mode="shared") + assert row.auth_mode == "per_user" + + +async def test_uninstall_removes_exactly_what_was_created(): + tenant, project = "t_conn_u", "p_conn_u" + row = await _install(tenant, project) + tool_ids, ts_id, ap_id = list(row.created_tool_ids), row.tool_set_id, row.auth_provider_id + + # A tool the user built themselves must survive the uninstall. + from forge.services.tools import ToolService + async with SessionLocal() as s: + mine = await ToolService.create(s, tenant, project, name="my_own_tool", kind="builtin", + config={"builtin": "current_time"}) + mine_id = mine.id + + async with SessionLocal() as s: + install = await ConnectorInstaller.get_install(s, tenant, project, "acme") + await ConnectorInstaller().uninstall(s, install) + + async with SessionLocal() as s: + for tid in tool_ids: + assert await s.get(Tool, tid) is None + assert await s.get(ToolSet, ts_id) is None + assert await s.get(AuthProvider, ap_id) is None + assert await ConnectorInstaller.get_install(s, tenant, project, "acme") is None + assert await s.get(Tool, mine_id) is not None, "uninstall deleted an unrelated tool" + + +async def test_uninstall_clears_the_stored_credential(): + tenant, project = "t_conn_c", "p_conn_c" + await _install(tenant, project) + async with SessionLocal() as s: + install = await ConnectorInstaller.get_install(s, tenant, project, "acme") + await ConnectorInstaller().uninstall(s, install) + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('acme', 'token')}" + ) + assert value == "" + + +async def test_failed_install_leaves_no_orphan_rows(): + """A manifest whose tool creation blows up must not leave a dangling auth provider + tool + set behind - a half-installed connector is worse than none.""" + tenant, project = "t_conn_x", "p_conn_x" + manifest = parse_manifest({**REST_MANIFEST, "slug": "acme-boom"}) + + installer = ConnectorInstaller() + + async def _explode(*a, **kw): + raise RuntimeError("boom") + + installer._create_rest_tools = _explode # type: ignore[method-assign] + + async with SessionLocal() as s: + with pytest.raises(RuntimeError): + await installer.install(s, tenant, project, manifest, values={"token": "x"}, source="custom") + + async with SessionLocal() as s: + from sqlalchemy import select + aps = (await s.execute(select(AuthProvider).where(AuthProvider.project_id == project))).scalars().all() + sets = (await s.execute(select(ToolSet).where(ToolSet.project_id == project))).scalars().all() + installs = (await s.execute(select(ConnectorInstall).where(ConnectorInstall.project_id == project))).scalars().all() + assert aps == [] and sets == [] and installs == [] + + +async def test_failed_install_clears_the_credentials_it_created(): + """Rows are not the only thing a half-install leaves behind. + + Every service on the install path commits as it goes, so a failure has no transaction to + unwind - the secrets written in step 1 survive with no install pointing at them. That is not + just litter: `group_has_credentials` reads the store, so an orphaned client id/secret makes + the whole credential group look already-configured to the next install. + """ + tenant, project = "t_conn_secx", "p_conn_secx" + manifest = parse_manifest({**REST_MANIFEST, "slug": "acme-secboom"}) + installer = ConnectorInstaller() + + async def _explode(*a, **kw): + raise RuntimeError("boom") + + installer._create_rest_tools = _explode # type: ignore[method-assign] + async with SessionLocal() as s: + with pytest.raises(RuntimeError): + await installer.install(s, tenant, project, manifest, values={"token": "x"}, source="custom") + + async with SessionLocal() as s: + assert not await group_has_credentials(SecretStore(), tenant, project, manifest), ( + "a failed install must not leave its group looking configured" + ) + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, + ref=f"secret://proj/{secret_name('acme-secboom', 'token')}", + ) + assert not value + + +async def test_a_failed_install_does_not_blank_a_siblings_shared_credential(google_app): + """Gmail and Calendar are one Google OAuth app. Rolling back a failed Calendar install must + clear only what that install CREATED - blanking the shared client secret it merely re-wrote + would sign every Gmail user out on the way past.""" + tenant, project = "t_conn_sib", "p_conn_sib" + await _install_catalog(tenant, project, "gmail") + + installer = ConnectorInstaller() + + async def _explode(*a, **kw): + raise RuntimeError("boom") + + installer._create_rest_tools = _explode # type: ignore[method-assign] + async with SessionLocal() as s: + with pytest.raises(RuntimeError): + await installer.install(s, tenant, project, get_manifest("google-calendar"), source="catalog") + + secret = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_secret')}", + ) + assert secret == "deployment-csec", "the sibling's shared credential must survive the rollback" + + +async def test_a_concurrent_duplicate_install_is_reported_not_raised_raw(): + """The duplicate check is a read and `POST /connect` installs on demand, so two people + clicking Connect at once both pass it. The loser hits the unique constraint; that must come + back as the same InstallError the read produces, not a raw IntegrityError (a 500).""" + tenant, project = "t_conn_race", "p_conn_race" + manifest = parse_manifest({**REST_MANIFEST, "slug": "acme-race"}) + installer = ConnectorInstaller() + + async def _blind(*a, **kw): + return None # both callers "see" no existing install + + async def _group_looks_empty(*a, **kw): + # The real interleaving: BOTH racers check for the group's credentials before either has + # written them, so both count the write as one they created. The loser's rollback then + # holds a list naming the credential the winner is about to depend on. + return False + + installer.get_install = _blind # type: ignore[method-assign] + installer._secret_exists = _group_looks_empty # type: ignore[method-assign] + await _install(tenant, project, {**REST_MANIFEST, "slug": "acme-race"}) + + async with SessionLocal() as s: + # The same credential both times: two racers on the one-click path both read the vendor + # app out of the same environment, so they write identical values. + with pytest.raises(InstallError, match="already installed"): + await installer.install(s, tenant, project, manifest, values={"token": "sekrit"}, source="custom") + + # ...and the loser's half-built rows are gone, so the winner's install is the only one. + async with SessionLocal() as s: + from sqlalchemy import select + installs = (await s.execute( + select(ConnectorInstall).where(ConnectorInstall.project_id == project) + )).scalars().all() + sets = (await s.execute(select(ToolSet).where(ToolSet.project_id == project))).scalars().all() + assert len(installs) == 1 and len(sets) == 1 + + # ...and crucially the WINNER's credentials survive. Both racers found the group empty and + # both wrote it, so the loser's "secrets I created" list names the very credential the + # surviving install now depends on. Clearing it would turn a harmless race into a connector + # that reports "client_id secret is not set" to everyone. + token = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('acme-race', 'token')}", + ) + assert token == "sekrit", "the loser's rollback must not blank the winner's credential" + + +async def test_install_adds_connector_hosts_to_the_project_egress_allow_list(): + tenant, project = "t_conn_e", "p_conn_e" + from forge.models import Project + async with SessionLocal() as s: + s.add(Project(id=project, tenant_id=tenant, name="P", slug="p", config={})) + await s.commit() + + await _install(tenant, project) + + async with SessionLocal() as s: + proj = await s.get(Project, project) + assert "api.acme.test" in (proj.config.get("egress") or {}).get("allow_hosts", []) + + +async def _allow_hosts(project: str) -> list[str]: + from forge.models import Project + + async with SessionLocal() as s: + proj = await s.get(Project, project) + return list((proj.config.get("egress") or {}).get("allow_hosts") or []) + + +async def _new_project(tenant: str, project: str, *, allow_hosts: list[str] | None = None) -> None: + from forge.models import Project + + cfg = {"egress": {"allow_hosts": list(allow_hosts)}} if allow_hosts is not None else {} + async with SessionLocal() as s: + s.add(Project(id=project, tenant_id=tenant, name="P", slug="p", config=cfg)) + await s.commit() + + +async def test_uninstall_takes_back_the_egress_hosts_it_added(): + """Install only ever ADDED to the allow-list, and uninstall had no matching removal - so on a + deployment running a strict allow-list the list only ever grew. + + Evaluate four connectors and remove them, and their API hosts stay permanently reachable with + nothing referencing them: any hand-written tool a project editor adds later can call them, + which is precisely what default-deny exists to prevent. + """ + tenant, project = "t_conn_eu", "p_conn_eu" + await _new_project(tenant, project) + + install = await _install(tenant, project) + assert "api.acme.test" in await _allow_hosts(project) + assert install.created_egress_hosts == ["api.acme.test"], ( + "the install must record what it added, so uninstall can take back exactly that" + ) + + async with SessionLocal() as s: + row = await s.get(ConnectorInstall, install.id) + await ConnectorInstaller().uninstall(s, row) + + assert "api.acme.test" not in await _allow_hosts(project) + + +async def test_uninstall_leaves_a_host_a_person_allow_listed_by_hand(): + """A host already on the list when the connector arrived was put there by somebody, for + something else. Uninstalling the connector is not consent to remove it - and after the fact + the two are indistinguishable unless the install records what it actually added.""" + tenant, project = "t_conn_ep", "p_conn_ep" + await _new_project(tenant, project, allow_hosts=["api.acme.test", "unrelated.test"]) + + install = await _install(tenant, project) + assert install.created_egress_hosts == [], "the host was already allowed; nothing was added" + + async with SessionLocal() as s: + row = await s.get(ConnectorInstall, install.id) + await ConnectorInstaller().uninstall(s, row) + + hosts = await _allow_hosts(project) + assert "api.acme.test" in hosts, "a hand-allow-listed host is not the connector's to remove" + assert "unrelated.test" in hosts + + +async def test_uninstall_keeps_a_host_a_surviving_connector_still_needs(): + """Gmail and Sheets both reach oauth2.googleapis.com. The first one installed records it; the + second records nothing, because it was already allowed. So uninstalling the FIRST must still + leave the host behind - reading receipts alone would revoke it out from under the sibling.""" + tenant, project = "t_conn_es", "p_conn_es" + await _new_project(tenant, project) + + first = {**REST_MANIFEST, "slug": "acme-one", "egress_hosts": ["api.acme.test", "shared.test"]} + # A DIFFERENT backend host, or `hosts()` would report api.acme.test for this one too (it + # always includes the base_url's host) and the sibling would legitimately still need it. + sibling = { + **REST_MANIFEST, + "slug": "acme-two", + "egress_hosts": ["shared.test"], + "backend": {**REST_MANIFEST["backend"], "base_url": "https://api.other.test/v1"}, + } + one = await _install(tenant, project, first) + two = await _install(tenant, project, sibling) + + assert set(one.created_egress_hosts) == {"api.acme.test", "shared.test"} + assert set(two.created_egress_hosts) == {"api.other.test"}, "shared.test was already allowed" + + async with SessionLocal() as s: + row = await s.get(ConnectorInstall, one.id) + await ConnectorInstaller().uninstall(s, row) + + hosts = await _allow_hosts(project) + assert "shared.test" in hosts, "the surviving connector still needs this host" + assert "api.other.test" in hosts + assert "api.acme.test" not in hosts, "the host only the removed connector used should go" + + # ...and the receipt for shared.test moved to the survivor, so removing that one cleans up + # rather than orphaning a host no install remembers adding. + async with SessionLocal() as s: + row = await s.get(ConnectorInstall, two.id) + assert "shared.test" in (row.created_egress_hosts or []), "the receipt must be handed over" + await ConnectorInstaller().uninstall(s, row) + assert await _allow_hosts(project) == [] + + +async def test_a_failed_install_does_not_leave_its_egress_hosts_behind(): + """The allow-list is widened before the install row is written. A failure after that point + leaves hosts allowed with no install pointing at them - the same permanent widening, minus + even a connector to blame it on.""" + tenant, project = "t_conn_ef", "p_conn_ef" + await _new_project(tenant, project) + installer = ConnectorInstaller() + + def _explode(*a, **kw): + raise RuntimeError("boom") + + # Fail AFTER the allow-list is widened - `_initial_status` is read while building the + # ConnectorInstall row, which is the first thing that happens once the hosts are in. + installer._initial_status = _explode # type: ignore[method-assign] + with pytest.raises(RuntimeError): + async with SessionLocal() as s: + await installer.install(s, tenant, project, parse_manifest(REST_MANIFEST), + values={"token": "x"}, source="custom") + + assert "api.acme.test" not in await _allow_hosts(project) + + +async def test_setup_placeholders_are_substituted_into_urls(): + manifest = { + **REST_MANIFEST, + "slug": "acme-tpl", + "auth": { + "kind": "bearer", + "setup": [ + {"key": "token", "label": "Token", "secret": True, "required": True}, + {"key": "site", "label": "Site", "secret": False, "required": False, "default": "eu"}, + ], + }, + "backend": { + "type": "rest", + "base_url": "https://{setup.site}.acme.test/v1", + "actions": REST_MANIFEST["backend"]["actions"][:1], + }, + } + row = await _install("t_conn_t", "p_conn_t", manifest) + async with SessionLocal() as s: + tool = await s.get(Tool, row.created_tool_ids[0]) + # The default filled in for the value the installer left blank. + assert tool.config["request"]["url_template"].startswith("https://eu.acme.test/v1/") + + +async def test_google_family_shares_one_credential_group(): + """Gmail/Calendar/Drive/Sheets are ONE Google Cloud OAuth client. They must resolve to the + same stored secret and the same env key, or an operator registers four apps to offer what + Google considers one integration.""" + slugs = ["gmail", "google-calendar", "google-drive", "google-sheets"] + groups = {get_manifest(s).group for s in slugs} + assert groups == {"google"} + assert get_manifest("outlook").group == "microsoft" + # An unrelated connector keeps its own namespace. + assert get_manifest("github").group == "github" + + +async def test_one_env_entry_covers_the_whole_google_family(google_app): + """The operator registers ONE Google app; Gmail, Calendar, Drive and Sheets all become + connectable from it, sharing a single stored credential to rotate.""" + tenant, project = "t_conn_grp", "p_conn_grp" + for slug in ("gmail", "google-calendar", "google-drive", "google-sheets"): + assert env_ready(get_manifest(slug)), slug + await _install_catalog(tenant, project, "gmail") + row = await _install_catalog(tenant, project, "google-calendar") + async with SessionLocal() as s: + ap = await s.get(AuthProvider, row.auth_provider_id) + assert ap.config["client_id_ref"] == f"secret://proj/{secret_name('google', 'client_id')}" + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_id')}" + ) + assert value == "deployment-cid" + + +async def test_uninstalling_one_google_connector_does_not_break_its_siblings(google_app): + """The regression this guard exists for: blanking the shared client secret on uninstall + would silently disable every other Google connector in the project.""" + tenant, project = "t_conn_grp2", "p_conn_grp2" + await _install_catalog(tenant, project, "gmail") + await _install_catalog(tenant, project, "google-calendar") + + async with SessionLocal() as s: + gmail = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().uninstall(s, gmail) + + still_there = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_secret')}" + ) + assert still_there == "deployment-csec", "uninstalling Gmail wiped the shared Google credential" + + # Removing the LAST member of the group does clear it. + async with SessionLocal() as s: + cal = await ConnectorInstaller.get_install(s, tenant, project, "google-calendar") + await ConnectorInstaller().uninstall(s, cal) + cleared = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_secret')}" + ) + assert cleared == "" + + +async def test_deployment_registered_app_is_seeded_into_the_project_store(google_app): + """The one-click path: the operator's app is copied into the project's own encrypted store, + so resolve/refresh/rotate behave exactly as they would for a hand-pasted credential and no + second credential source has to be taught to every downstream path.""" + tenant, project = "t_conn_dep", "p_conn_dep" + row = await _install_catalog(tenant, project, "gmail") + + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_id')}" + ) + assert value == "deployment-cid" + async with SessionLocal() as s: + ap = await s.get(AuthProvider, row.auth_provider_id) + assert ap.config["client_id_ref"] == f"secret://proj/{secret_name('google', 'client_id')}" + + +async def test_catalog_install_ignores_credentials_sent_by_a_caller(google_app): + """There is no supported way to type a credential into a catalog connector, so a client that + sends one anyway must not create a second, invisible source of truth - the deployment's app + is the answer, and `env_ready` has to keep telling the truth about it.""" + tenant, project = "t_conn_dep2", "p_conn_dep2" + async with SessionLocal() as s: + await ConnectorInstaller().install( + s, tenant, project, get_manifest("gmail"), source="catalog", + values={"client_id": "smuggled-cid", "client_secret": "smuggled-csec"}, + ) + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_id')}" + ) + assert value == "deployment-cid" + + +async def test_unconfigured_catalog_connector_refuses_to_install(no_apps): + """Default posture with nothing in the environment: the connector is unavailable and the + error names the env key, rather than degrading into a form for a secret an end user should + never hold.""" + gmail = get_manifest("gmail") + assert not env_ready(gmail) + assert missing_app_keys(gmail) == ["client_id", "client_secret"] + + async with SessionLocal() as s: + with pytest.raises(InstallError) as e: + await ConnectorInstaller().install(s, "t_none", "p_none", gmail, source="catalog") + assert "FORGE_CONNECTOR_OAUTH_APPS" in str(e.value) + assert "google" in str(e.value) + + +async def test_discovery_connectors_need_no_environment_entry(no_apps): + """Slack, Notion, Linear and Atlassian publish OAuth metadata, so Forge registers a client + with them on the fly (RFC 7591). They are one-click on a deployment that has configured + nothing at all - which is what an evaluator sees on first boot.""" + for slug in ("slack", "notion", "linear", "atlassian"): + m = get_manifest(slug) + assert m.auth.discover, slug + assert env_ready(m), f"{slug} should be connectable with no env configuration" + + +async def test_gmail_installs_as_per_user_oauth_awaiting_its_first_sign_in(google_app): + """A real catalog entry, end to end - the shape the Connectors screen actually installs.""" + tenant, project = "t_conn_gm", "p_conn_gm" + manifest = get_manifest("gmail") + row = await _install_catalog(tenant, project, "gmail") + + assert row.auth_mode == "per_user" + # Nobody has signed in yet. The row-level status only ever means "at least one person has + # connected"; what matters to a given user is computed per caller by the router. + assert row.status == "needs_auth" + assert len(row.created_tool_ids) == len(manifest.backend.actions) + async with SessionLocal() as s: + ap = await s.get(AuthProvider, row.auth_provider_id) + assert ap.config["per_user_context_keys"] == ["end_user_id"] + assert ap.config["authorize_url"].startswith("https://accounts.google.com/") + assert "gmail.send" in ap.config["scope"] + + +async def test_a_catalog_connector_is_personal_even_if_asked_to_be_shared(google_app): + """`auth_mode` is a custom-connector control. A catalog connector is a personal account, and + a caller asking for a project-wide one must not get someone's mailbox shared with the team.""" + tenant, project = "t_conn_shared", "p_conn_shared" + async with SessionLocal() as s: + row = await ConnectorInstaller().install( + s, tenant, project, get_manifest("gmail"), source="catalog", auth_mode="shared", + ) + assert row.auth_mode == "per_user" + + +async def test_custom_manifests_keep_the_shared_option_and_the_credential_form(): + """The escape hatch the catalog rules deliberately leave open: an unattended workflow with no + end user still needs a credential, and a pasted manifest is where that is configured.""" + tenant, project = "t_conn_custom", "p_conn_custom" + row = await _install(tenant, project, auth_mode="shared") + assert row.auth_mode == "shared" + value = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('acme', 'token')}" + ) + assert value == "sekrit" + # Group sharing still applies on the custom path, where credentials are typed. + assert not await group_has_credentials(SecretStore(), "t_x", "p_x", parse_manifest(REST_MANIFEST)) + + +# --- the API surface a person actually clicks ----------------------------------------------- +# +# One call, one browser round trip, connected - and "connected" means connected FOR YOU. These +# pin down the two claims the Connectors screen makes to whoever is looking at it. + +async def _editor_client(): + """An httpx client authenticated as a freshly registered (owner-role) user, plus a project. + + Returned already in use, so callers close it with `aclosing` rather than `async with` - an + httpx client can only be entered once, and this one has made its first request already. + """ + import uuid + + import httpx + + from forge.main import create_app + + c = httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test") + reg = await c.post("/v1/auth/register", + json={"email": f"u{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + assert reg.status_code == 201, reg.text + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + pid = (await c.post("/v1/projects", json={"name": "Conn", "slug": f"conn-{uuid.uuid4().hex[:8]}"})).json()["id"] + return c, pid + + +async def _client_for(user_id: str, tenant: str, role: str): + import httpx + + from forge.main import create_app + from forge.security import create_access_token + + c = httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test") + c.headers["Authorization"] = f"Bearer {create_access_token(user_id=user_id, tenant_id=tenant, role=role)}" + return c + + +async def _make_user(tenant: str, email: str, role: str) -> str: + from forge.models import User + + async with SessionLocal() as s: + u = User(tenant_id=tenant, email=email, role=role, status="active") + s.add(u) + await s.commit() + await s.refresh(u) + return u.id + + +async def _fake_consent(tenant: str, project: str, slug: str, user_id: str) -> None: + """Stand in for a completed browser consent: store that user's bundle where the callback + would have put it.""" + from forge.services.auth_providers import AuthProviderService + + async with SessionLocal() as s: + install = await ConnectorInstaller.get_install(s, tenant, project, slug) + ap = await s.get(AuthProvider, install.auth_provider_id) + await AuthProviderService.set_user_connection( + s, tenant, project, ap, user_id, bundle={"access_token": f"tok-{user_id}"}, + ) + await s.commit() + + +async def test_connect_adds_the_connector_on_the_first_click(google_app): + """The whole point of the screen: nobody has to know an "install" step exists.""" + c, pid = await _editor_client() + async with aclosing(c): + assert (await c.get(f"/v1/projects/{pid}/connectors")).json() == [] + + r = await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + assert r.status_code == 200, r.text + body = r.json() + assert body["per_user"] is True + assert body["authorize_url"].startswith("https://accounts.google.com/") + # PKCE, and the DEPLOYMENT's client id - not something typed on this screen. + assert "code_challenge_method=S256" in body["authorize_url"] + assert "client_id=deployment-cid" in body["authorize_url"] + + installed = (await c.get(f"/v1/projects/{pid}/connectors")).json() + assert [i["slug"] for i in installed] == ["gmail"] + assert installed[0]["auth_mode"] == "per_user" + assert installed[0]["connected"] is False + + +async def test_the_gallery_and_the_detail_panel_agree_on_the_action_count(google_app): + """Deleting an action from the Tools screen must show up everywhere that reports a count. + + `list_installed` counted tools that still exist; `/{slug}/status` fell back to + `len(created_tool_ids)` and happily reported the deleted ones. Delete two of Gmail's actions + and the gallery said 3 while the detail panel - which is what polls `/status` - said 5. + """ + c, pid = await _editor_client() + async with aclosing(c): + await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + + listed = (await c.get(f"/v1/projects/{pid}/connectors")).json()[0] + status_out = (await c.get(f"/v1/projects/{pid}/connectors/gmail/status")).json() + original = listed["tool_count"] + assert original >= 3 and status_out["tool_count"] == original + + from forge.services.tools import ToolService + + me = (await c.get("/v1/auth/me")).json() + async with SessionLocal() as s: + install = await ConnectorInstaller.get_install(s, me["tenant_id"], pid, "gmail") + for tool_id in list(install.created_tool_ids)[:2]: + tool = await s.get(Tool, tool_id) + await ToolService.delete(s, tool) + + listed = (await c.get(f"/v1/projects/{pid}/connectors")).json()[0] + status_out = (await c.get(f"/v1/projects/{pid}/connectors/gmail/status")).json() + assert listed["tool_count"] == original - 2 + assert status_out["tool_count"] == listed["tool_count"], ( + "the detail panel is still counting actions the user deleted" + ) + + +async def test_the_installed_list_reads_every_bundle_in_one_go(google_app, monkeypatch): + """One secret read for the whole screen, not one per connector. + + `_connected_for` resolved a bundle per install, sequentially - a DB round trip, a decrypt and + an audit write each, all at the store's choke point. A project with the four Google + connectors installed paid that four times on every paint, and the connect flow calls + `reload()` on window focus, so it repeated every time someone came back from a consent + window. The count must not scale with the number of installed connectors. + """ + from forge.secrets.store import SecretStore + + c, pid = await _editor_client() + async with aclosing(c): + slugs = ("gmail", "google-calendar", "google-drive", "google-sheets") + for slug in slugs: + r = await c.post(f"/v1/projects/{pid}/connectors/{slug}/connect", json={}) + assert r.status_code == 200, r.text + + singles, batches = [], [] + real_one, real_many = SecretStore.read_ref, SecretStore.read_refs + + async def _count_one(self, **kw): + singles.append(kw.get("ref")) + return await real_one(self, **kw) + + async def _count_many(self, **kw): + batches.append(list(kw.get("refs") or [])) + return await real_many(self, **kw) + + monkeypatch.setattr(SecretStore, "read_ref", _count_one) + monkeypatch.setattr(SecretStore, "read_refs", _count_many) + + installed = (await c.get(f"/v1/projects/{pid}/connectors")).json() + + assert len(installed) == 4 + assert len(batches) == 1, "the whole list should resolve in a single batched read" + assert len(batches[0]) == 4, "and that read should cover every install" + assert singles == [], f"one-at-a-time bundle reads are back: {singles}" + + +async def test_a_batched_secret_read_still_audits_every_secret(): + """The batch goes through the same choke point. Auditing is not a side effect of reading one + at a time - a read that skipped the trail because it was batched would be a hole in it.""" + from sqlalchemy import select + + from forge.models import AuditLog + from forge.secrets.store import SecretStore + + tenant, project = "t_sec_batch", "p_sec_batch" + store = SecretStore() + async with SessionLocal() as s: + for i in range(3): + await store.write(s, tenant_id=tenant, project_id=project, + name=f"batched_{i}", value=f"v{i}", kind="generic") + + got = await store.read_refs( + tenant_id=tenant, project_id=project, + refs=[f"secret://proj/batched_{i}" for i in range(3)] + ["secret://proj/absent"], + ) + assert got == {"batched_0": "v0", "batched_1": "v1", "batched_2": "v2"} + assert "absent" not in got, "a missing name is absent from the result, not an error" + + async with SessionLocal() as s: + rows = (await s.execute( + select(AuditLog).where(AuditLog.tenant_id == tenant, AuditLog.action == "secret.read") + )).scalars().all() + assert {r.resource_id for r in rows} == {"batched_0", "batched_1", "batched_2"} + + +async def test_unconfigured_connector_is_unavailable_and_names_the_env_key(no_apps): + """No form, no half-working install - the card tells the operator what to register.""" + c, pid = await _editor_client() + async with aclosing(c): + cat = (await c.get(f"/v1/projects/{pid}/connectors/catalog")).json()["connectors"] + gmail = next(x for x in cat if x["slug"] == "gmail") + assert gmail["managed"] is True + assert gmail["configured"] is False + assert gmail["missing_keys"] == ["client_id", "client_secret"] + assert gmail["config_env_key"] == "FORGE_CONNECTOR_OAUTH_APPS" + assert gmail["credential_group"] == "google" + # The callback to whitelist comes from the API's own public base URL. Letting the browser + # guess it from the console origin yields a plausible-looking URL that the vendor then + # rejects as redirect_uri_mismatch - so the server has to be the one saying it. + from forge.config import settings + assert gmail["redirect_uri"] == f"{settings.public_base_url}/v1/oauth/callback" + + r = await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + assert r.status_code == 400 + assert "FORGE_CONNECTOR_OAUTH_APPS" in r.json()["detail"] + + # Slack needs no entry at all - it registers a client dynamically. + slack = next(x for x in cat if x["slug"] == "slack") + assert slack["configured"] is True and slack["missing_keys"] == [] + + +async def test_connected_is_answered_for_the_caller_not_the_project(google_app): + """A colleague signing in to their own mailbox must not show as a green tick on yours.""" + c, pid = await _editor_client() + async with aclosing(c): + me = (await c.get("/v1/auth/me")).json() + tenant, my_id = me["tenant_id"], me["id"] + await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + assert (await c.get(f"/v1/projects/{pid}/connectors")).json()[0]["connected"] is False + + await _fake_consent(tenant, pid, "gmail", my_id) + assert (await c.get(f"/v1/projects/{pid}/connectors")).json()[0]["connected"] is True + assert (await c.get(f"/v1/projects/{pid}/connectors/gmail/status")).json()["connected"] is True + + colleague = await _make_user(tenant, "colleague@example.com", "editor") + c2 = await _client_for(colleague, tenant, "editor") + async with aclosing(c2): + theirs = (await c2.get(f"/v1/projects/{pid}/connectors")).json()[0] + assert theirs["connected"] is False, "one person's Gmail became everyone's" + + +async def test_a_viewer_is_told_who_can_add_a_connector_rather_than_hitting_a_404(google_app): + """Adding a connector creates project-level tools, so it stays an editor action - but the + person who gets there first deserves an instruction, not a dead end.""" + c, pid = await _editor_client() + async with aclosing(c): + tenant = (await c.get("/v1/auth/me")).json()["tenant_id"] + + uid = await _make_user(tenant, "viewer-conn@example.com", "viewer") + c2 = await _client_for(uid, tenant, "viewer") + async with aclosing(c2): + r = await c2.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + assert r.status_code == 403 + assert "editor" in r.json()["detail"] + + # Once an editor has added it, that same viewer connects their OWN account, no extra rights. + await _install_catalog(tenant, pid, "gmail") + c3 = await _client_for(uid, tenant, "viewer") + async with aclosing(c3): + r = await c3.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + assert r.status_code == 200, r.text + assert r.json()["authorize_url"].startswith("https://accounts.google.com/") + await _fake_consent(tenant, pid, "gmail", uid) + assert (await c3.get(f"/v1/projects/{pid}/connectors/gmail/status")).json()["connected"] is True + + +async def test_catalog_credentials_cannot_be_typed_in_through_the_api(google_app): + """Rotation for a catalog connector is "change the env and restart", so the route that + accepts pasted credentials has to say so rather than quietly writing a shadow copy.""" + c, pid = await _editor_client() + async with aclosing(c): + await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + r = await c.put(f"/v1/projects/{pid}/connectors/gmail/credentials", + json={"values": {"client_secret": "smuggled"}}) + assert r.status_code == 400 + assert "FORGE_CONNECTOR_OAUTH_APPS" in r.json()["detail"] + + +async def test_mcp_discovery_asks_as_the_user_who_just_consented(google_app, monkeypatch): + """The bug this exists for: an MCP connector signed in fine, then listed ZERO actions. + + Every catalog connector is per-user, so the only credential that exists after consent is + stored under that person's identity. Discovery that forgets to say who it is resolves a + shared bundle nobody wrote, calls the server unauthenticated, gets a 401, and leaves the + connector "connected" with nothing in it.""" + from forge.connectors.manifest import parse_manifest as _parse + + seen: list[dict | None] = [] + + async def _fake_discover(client_row, tenant_id, project_id, context=None): + seen.append(context) + if not (context or {}).get("end_user_id"): + raise RuntimeError("401 Unauthorized") + return [{"name": "notion_search", "description": "Search."}] + + monkeypatch.setattr("forge.tools.mcp.discover_tools", _fake_discover) + + tenant, project = "t_mcp_ctx", "p_mcp_ctx" + manifest = _parse({ + **get_manifest("notion").model_dump(mode="json"), + "auth": {**get_manifest("notion").model_dump(mode="json")["auth"], "discover": True}, + }) + async with SessionLocal() as s: + install = await ConnectorInstaller().install(s, tenant, project, manifest, source="catalog") + assert install.created_tool_ids == [], "discovery before consent legitimately finds nothing" + assert seen and seen[-1] is None or True # install-time probe has no identity yet + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "notion") + count = await ConnectorInstaller().sync_tools(s, row, context={"end_user_id": "user-1"}) + assert count == 1 + assert seen[-1] == {"end_user_id": "user-1"} + + +async def test_refresh_delivers_a_corrected_manifest_to_an_existing_install(google_app): + """A connector installed last week holds a COPY of the manifest as it was then, so a fix to a + bundled action has to be able to REACH it. Uninstall/reinstall would work but deletes the auth + provider, making everyone who signed in redo it for a change they had nothing to do with.""" + tenant, project = "t_upg", "p_upg" + install = await _install_catalog(tenant, project, "gmail") + tool_ids_before = list(install.created_tool_ids) + + # Rewind this install to a stale manifest: the old Gmail send action, which asked the MODEL + # for a base64url-encoded MIME message and produced an opaque 400. + stale = {**install.manifest} + stale["version"] = "0.9.0" + stale["backend"] = {**stale["backend"], "actions": [ + {**a, "request": {"method": "POST", "url_template": "/messages/send", + "fields": [{"path": "raw", "in": "body", "type": "string", + "required": True, "llm_visible": True}]}} + if a["name"] == "gmail_send_message" else a + for a in stale["backend"]["actions"] + ]} + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + row.manifest = stale + row.version = "0.9.0" + send_tool = next(t for t in [await s.get(Tool, i) for i in tool_ids_before] + if t.name == "gmail_send_message") + send_tool.config = {**send_tool.config, "request": stale["backend"]["actions"][2]["request"]} + await s.commit() + assert {f["path"] for f in send_tool.config["request"]["fields"]} == {"raw"} + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().sync_tools(s, row) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + # Tool ids are preserved, so every workflow node and agent grant keeps working. + assert row.created_tool_ids == tool_ids_before + assert row.version == get_manifest("gmail").version + fixed = next(t for t in [await s.get(Tool, i) for i in row.created_tool_ids] + if t.name == "gmail_send_message") + args = {f["path"] for f in fixed.config["request"]["fields"]} + assert {"to", "subject", "body"} <= args and "raw" not in args + assert "$mime" in fixed.config["request"]["body_template"] + # The auth provider - and therefore everyone's stored sign-in - is untouched. + async with SessionLocal() as s: + assert await s.get(AuthProvider, install.auth_provider_id) is not None + + +async def test_refresh_never_moves_an_install_to_a_new_credential_group(google_app): + """A refresh rewrites the stored manifest, and the stored manifest is where the credential + GROUP is read from. The group is a pointer, not a description: `secret_name(group, ...)` + names the secrets this install actually wrote, and uninstall compares groups to decide + whether a sibling still needs the shared vendor app. If a catalog edit could repoint it, + uninstalling Gmail would blank the Google client secret while Calendar and Sheets are still + using it.""" + tenant, project = "t_upg_grp", "p_upg_grp" + await _install_catalog(tenant, project, "gmail") + await _install_catalog(tenant, project, "google-calendar") + + # A catalog update that regroups Gmail away from the shared Google app. + regrouped = {**get_manifest("gmail").model_dump(mode="json")} + regrouped["version"] = "9.9.9" + regrouped["auth"] = {**regrouped["auth"], "credential_group": "gmail-only"} + + import forge.connectors.catalog as catalog_mod + real = catalog_mod.get_manifest + catalog_mod.get_manifest = lambda slug: ( # type: ignore[assignment] + parse_manifest(regrouped) if slug == "gmail" else real(slug) + ) + try: + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().sync_tools(s, row) + finally: + catalog_mod.get_manifest = real # type: ignore[assignment] + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + assert row.version == "9.9.9", "the upgrade itself must still land" + assert parse_manifest(row.manifest).group == "google", ( + "the credential group must stay where this install's secrets actually are" + ) + # ...and uninstalling Gmail still recognises Calendar as sharing the vendor app. + await ConnectorInstaller().uninstall(s, row) + + secret = await SecretStore().read_ref( + tenant_id=tenant, project_id=project, ref=f"secret://proj/{secret_name('google', 'client_secret')}", + ) + assert secret == "deployment-csec", "the surviving sibling's credential must not be blanked" + + +async def test_refresh_keeps_the_values_this_install_was_configured_with(): + """A refresh must not rebuild URLs from the manifest's DEFAULTS - those are precisely what + the installer overrode, so doing so would silently repoint every tool at the wrong host.""" + manifest = { + **REST_MANIFEST, + "slug": "acme-site", + "auth": { + "kind": "bearer", + "setup": [ + {"key": "token", "label": "Token", "secret": True, "required": True}, + {"key": "site", "label": "Site", "secret": False, "required": False, "default": "eu"}, + ], + }, + "backend": { + "type": "rest", + "base_url": "https://{setup.site}.acme.test/v1", + "actions": REST_MANIFEST["backend"]["actions"][:1], + }, + } + tenant, project = "t_upg3", "p_upg3" + manifest_obj = parse_manifest(manifest) + async with SessionLocal() as s: + row = await ConnectorInstaller().install( + s, tenant, project, manifest_obj, source="custom", + values={"token": "sekrit", "site": "apac"}, + ) + async with SessionLocal() as s: + tool = await s.get(Tool, row.created_tool_ids[0]) + assert tool.config["request"]["url_template"].startswith("https://apac.acme.test/v1/") + install = await ConnectorInstaller.get_install(s, tenant, project, "acme-site") + await ConnectorInstaller().sync_tools(s, install) + async with SessionLocal() as s: + tool = await s.get(Tool, row.created_tool_ids[0]) + assert tool.config["request"]["url_template"].startswith("https://apac.acme.test/v1/"), \ + "refresh rebuilt the URL from the manifest default and lost the configured site" + + +async def test_refresh_keeps_an_action_the_manifest_dropped(): + """Deleting a tool a live workflow still references is a worse failure than carrying a stale + one, so a shrunken manifest leaves the extra row alone.""" + tenant, project = "t_upg2", "p_upg2" + row = await _install(tenant, project) # custom manifest, two actions + keep = list(row.created_tool_ids) + shrunk = {**row.manifest} + shrunk["backend"] = {**shrunk["backend"], "actions": shrunk["backend"]["actions"][:1]} + async with SessionLocal() as s: + install = await ConnectorInstaller.get_install(s, tenant, project, "acme") + install.manifest = shrunk + await s.commit() + await ConnectorInstaller().sync_tools(s, install) + async with SessionLocal() as s: + for tid in keep: + assert await s.get(Tool, tid) is not None + + +async def test_mcp_errors_name_the_actual_failure_not_the_task_group(): + """anyio wraps every MCP transport failure in an ExceptionGroup whose str() is "unhandled + errors in a TaskGroup (1 sub-exception)" - which tells a user nothing about the 401 inside.""" + from forge.tools.mcp import describe_mcp_error + + inner = PermissionError("401 Unauthorized: token is invalid") + group = BaseExceptionGroup("unhandled errors in a TaskGroup", [inner]) + described = describe_mcp_error(group) + assert "401 Unauthorized" in described + assert "TaskGroup" not in described + + # Nested groups (a task group inside a task group) still reach the leaf, and repeats of the + # same error across sibling tasks collapse into one line. + nested = BaseExceptionGroup("outer", [BaseExceptionGroup("inner", [inner, inner])]) + assert describe_mcp_error(nested).count("401 Unauthorized") == 1 + # A plain exception is passed through unharmed. + assert "boom" in describe_mcp_error(RuntimeError("boom")) + + +async def test_examples_route_offers_the_key_based_manifests(): + c, pid = await _editor_client() + async with aclosing(c): + rows = (await c.get(f"/v1/projects/{pid}/connectors/examples")).json() + slugs = {r["slug"] for r in rows} + assert {"stripe", "slack-api", "twilio"} <= slugs + stripe = next(r for r in rows if r["slug"] == "stripe") + assert stripe["needs"], "an example should say what it will ask for" + # The payload is a manifest the custom form can install as-is. + assert parse_manifest(stripe["manifest"]).slug == "stripe" + + +async def test_refresh_does_not_resurrect_a_deliberately_deleted_action(google_app): + """A project that deleted "send email" made a decision. Picking up a manifest fix must not + quietly hand the capability back.""" + from forge.services.tools import ToolService + + tenant, project = "t_upg4", "p_upg4" + install = await _install_catalog(tenant, project, "gmail") + async with SessionLocal() as s: + send = next(t for t in [await s.get(Tool, i) for i in install.created_tool_ids] + if t.name == "gmail_send_message") + gone_id = send.id + await ToolService.delete(s, send) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().sync_tools(s, row) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + live = [t for t in [await s.get(Tool, i) for i in row.created_tool_ids] if t is not None] + assert gone_id not in [t.id for t in live] + assert "gmail_send_message" not in {t.name for t in live}, "refresh restored a removed capability" + # ...while the actions that ARE still there were refreshed as usual. + assert "gmail_search_messages" in {t.name for t in live} + + +async def test_refresh_still_adds_an_action_new_in_the_upgrade(google_app): + """The mirror case: a name the install never created is genuinely new and must appear.""" + tenant, project = "t_upg5", "p_upg5" + install = await _install_catalog(tenant, project, "gmail") + before = len(install.created_tool_ids) + + # Rewind the frozen manifest so one existing action reads as "not created by this install", + # which is exactly the shape of an action added by a later catalog version. + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + trimmed = {**row.manifest} + trimmed["backend"] = {**trimmed["backend"], "actions": [ + a for a in trimmed["backend"]["actions"] if a["name"] != "gmail_list_labels" + ]} + row.manifest = trimmed + listing = next(t for t in [await s.get(Tool, i) for i in row.created_tool_ids] + if t.name == "gmail_list_labels") + row.created_tool_ids = [i for i in row.created_tool_ids if i != listing.id] + from forge.services.tools import ToolService + await ToolService.delete(s, listing) + await s.commit() + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().sync_tools(s, row) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + names = {t.name for t in [await s.get(Tool, i) for i in row.created_tool_ids] if t} + assert "gmail_list_labels" in names + assert len(row.created_tool_ids) == before + + +async def test_refreshing_a_rest_connector_requires_editor(google_app): + """It rewrites project tool configs, which is a project-level write - unlike MCP discovery, + which only asks the vendor what it exposes using the caller's own credential.""" + c, pid = await _editor_client() + async with aclosing(c): + tenant = (await c.get("/v1/auth/me")).json()["tenant_id"] + await c.post(f"/v1/projects/{pid}/connectors/gmail/connect", json={}) + + uid = await _make_user(tenant, "viewer-sync@example.com", "viewer") + c2 = await _client_for(uid, tenant, "viewer") + async with aclosing(c2): + r = await c2.post(f"/v1/projects/{pid}/connectors/gmail/sync") + assert r.status_code == 403 + assert "editor" in r.json()["detail"] + + +async def test_refresh_prunes_ids_of_tools_that_no_longer_exist(google_app): + """Dead ids in the receipt grow the IN clause on every refresh and send uninstall looking + for rows that are already gone.""" + from forge.services.tools import ToolService + + tenant, project = "t_upg6", "p_upg6" + install = await _install_catalog(tenant, project, "gmail") + before = len(install.created_tool_ids) + async with SessionLocal() as s: + doomed = await s.get(Tool, install.created_tool_ids[0]) + doomed_id = doomed.id + await ToolService.delete(s, doomed) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + await ConnectorInstaller().sync_tools(s, row) + + async with SessionLocal() as s: + row = await ConnectorInstaller.get_install(s, tenant, project, "gmail") + assert doomed_id not in row.created_tool_ids + assert len(row.created_tool_ids) == before - 1 diff --git a/apps/api/tests/test_field_extract.py b/apps/api/tests/test_field_extract.py new file mode 100644 index 0000000..e3bd4f5 --- /dev/null +++ b/apps/api/tests/test_field_extract.py @@ -0,0 +1,133 @@ +"""`extract`: pull an opaque id out of the URL a person actually pastes. + +The failure this prevents: Google Sheets addresses a spreadsheet by an opaque key that lives +inside its link, and what a user says is "get testsheet" or "here's my sheet: ". A model +handed only a NAME will confidently pass the name as the id, and the 404 that comes back reads +like a permissions problem rather than "that argument was never knowable". + +`extract` closes the URL half declaratively: paste the link, get the id. The name half is closed +by the field description telling the model to search Drive (or ask) instead of guessing. +""" + +from __future__ import annotations + +import httpx +import pytest + +from forge.connectors.catalog import get_manifest +from forge.connectors.manifest import ManifestError, parse_manifest +from forge.tools.rest import execute_rest + +SHEET_URL = "https://docs.google.com/spreadsheets/d/1AbC-dEf_23/edit#gid=0" + + +def _client(seen: dict) -> httpx.AsyncClient: + def handler(request: httpx.Request) -> httpx.Response: + seen["url"] = str(request.url) + return httpx.Response(200, json={"values": [["ok"]]}) + + return httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + +async def _call(spreadsheet_id: str) -> str: + """Run the real catalog action against a stubbed transport; return the URL it built.""" + m = get_manifest("google-sheets") + action = next(a for a in m.backend.actions if a.name == "sheets_read_range") + request = {**action.request, "url_template": m.backend.base_url + action.request["url_template"]} + seen: dict = {} + async with _client(seen) as client: + await execute_rest( + {"name": "sheets_read_range", "kind": "rest_api", "request": request}, + {"spreadsheet_id": spreadsheet_id, "range": "Sheet1!A1:D50"}, + tenant_id="t_x", project_id="p_x", client=client, + ) + return seen["url"] + + +async def test_a_pasted_sheet_url_becomes_the_spreadsheet_id(): + assert "/spreadsheets/1AbC-dEf_23/values/" in await _call(SHEET_URL) + + +async def test_a_bare_id_passes_straight_through(): + """No match must leave the value alone - the common case is already an id.""" + assert "/spreadsheets/1AbC-dEf_23/values/" in await _call("1AbC-dEf_23") + + +async def test_a_name_is_left_alone_so_the_failure_stays_honest(): + """`extract` deliberately does NOT invent an id from a name. A wrong id that 404s is better + than one that silently reads someone else's sheet, and the description tells the model to + search Drive or ask for the link instead.""" + assert "/spreadsheets/testsheet/values/" in await _call("testsheet") + + +async def test_every_google_id_argument_accepts_a_pasted_link(): + """The regression guard: any opaque-id argument in the Google connectors must either be + extractable from a URL or be discoverable from another action.""" + for slug, field in (("google-sheets", "spreadsheet_id"), ("google-drive", "file_id")): + for action in get_manifest(slug).backend.actions: + for f in action.request.get("fields", []): + if f["path"] == field: + assert f.get("extract"), f"{slug}.{action.name}: {field} can't take a link" + assert "name" in (f.get("description") or "").lower(), ( + f"{slug}.{action.name}: {field} should tell the model a name is not an id" + ) + + +def test_a_broken_extract_regex_is_rejected_at_install_not_at_call_time(): + bad = { + "format": "forge.connector/1", "slug": "bad-extract", "name": "Bad", + "backend": {"type": "rest", "base_url": "https://api.test", "actions": [{ + "name": "x", + "request": {"method": "GET", "url_template": "/{id}", + "fields": [{"path": "id", "in": "path", "extract": "/d/([a-z"}]}, + }]}, + } + with pytest.raises(ManifestError, match="extract"): + parse_manifest(bad) + + +async def test_extract_bounds_the_subject_it_matches(): + """The pattern is authored (trusted) but the value is model-supplied, and an unbounded subject + is what turns a sloppy regex into a stall - so only the first 4KB is searched. + + Only the MATCH is bounded, not the value: a link buried past the cap simply isn't extracted, + and the argument is sent as given. That fails visibly against the API rather than quietly + substituting whatever happened to fall inside the window.""" + url = await _call("x" * 10000 + "/spreadsheets/d/TOO_LATE/edit") + assert "/spreadsheets/TOO_LATE/values/" not in url, "matched past the bound" + assert "/spreadsheets/xxx" in url, "the value should be passed through untouched" + + +# --- Sheets write shape ---------------------------------------------------------------------- +# +# `values` in the Sheets API is ALWAYS a 2-D array (a list of rows). The append action used to +# take a 1-D "cells of the one new row" and wrap it, so asking for 100 rows - where the model +# naturally sends 2-D - produced a 3-D array and a 400. + +def _write_body(action_name: str, args: dict) -> dict: + from forge.tools.rest import _build_body + + a = next(x for x in get_manifest("google-sheets").backend.actions if x.name == action_name) + return _build_body(a.request, a.request["fields"], args, {}) + + +@pytest.mark.parametrize("action", ["sheets_append_row", "sheets_update_range"]) +@pytest.mark.parametrize("rows", [ + [["one row"]], + [["Test Data 1", "Test Data 2"], ["Test Data 5", "Test Data 6"]], + [[f"row {i}", i] for i in range(100)], +]) +def test_sheets_writes_are_always_a_2d_array(action: str, rows: list): + body = _write_body(action, {"rows": rows}) + assert isinstance(body, dict), "body must parse as JSON, not fall through as raw text" + assert body["values"] == rows, "rows must reach the API unchanged - not wrapped, not repr'd" + + +def test_sheets_append_takes_rows_not_a_single_row(): + """The regression guard for 'add random 100 row': one row and many rows are the same shape.""" + a = next(x for x in get_manifest("google-sheets").backend.actions if x.name == "sheets_append_row") + args = {f["path"] for f in a.request["fields"]} + assert "rows" in args and "values" not in args + # Appending must not overwrite whatever sits below the table. + insert = next(f for f in a.request["fields"] if f["path"] == "insertDataOption") + assert insert["default"] == "INSERT_ROWS" diff --git a/apps/api/tests/test_mcp.py b/apps/api/tests/test_mcp.py index 3dddb96..db6480a 100644 --- a/apps/api/tests/test_mcp.py +++ b/apps/api/tests/test_mcp.py @@ -35,7 +35,7 @@ async def test_adapters_import_available(): async def test_load_mcp_tool_finds_remote_tool(monkeypatch): cid = await _make_client() - async def fake_client_and_tools(row, tenant_id, project_id): + async def fake_client_and_tools(row, tenant_id, project_id, context=None): return object(), [_fake_remote_tool("search")] monkeypatch.setattr(mcp_mod, "_client_and_tools", fake_client_and_tools) diff --git a/apps/api/tests/test_mcp_auth.py b/apps/api/tests/test_mcp_auth.py new file mode 100644 index 0000000..9a38502 --- /dev/null +++ b/apps/api/tests/test_mcp_auth.py @@ -0,0 +1,561 @@ +"""MCP client-side auth: an Auth Provider attached to an external MCP server. + +This is what makes an OAuth-protected remote MCP server (mcp.slack.com and friends) reachable. +The properties that matter, and are easy to get wrong: + + * provider headers are applied per REQUEST, so a refreshed token lands without reconnecting, + * a 401 forces exactly one re-resolve and retry (not a loop), + * a PER-USER provider gets its OWN pooled connection per end user - the bug where one user's + authenticated session is handed to the next caller is the whole reason the cache key has a + user-dims suffix. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import httpx +import pytest + +from forge.db.base import SessionLocal +from forge.models import McpClient +from forge.services.auth_providers import AuthProviderService +from forge.tools import mcp as mcp_mod + + +async def _provider(tenant, project, *, per_user: bool, name="srv"): + async with SessionLocal() as s: + cfg = {"token_ref": "secret://proj/tok", "header_name": "Authorization", "prefix": "Bearer "} + if per_user: + cfg["per_user_context_keys"] = ["end_user_id"] + ap = await AuthProviderService.create(s, tenant, project, name=name, kind="bearer", config=cfg) + return ap.id + + +async def _client(tenant, project, ap_id=None, transport="streamable_http"): + async with SessionLocal() as s: + row = McpClient(tenant_id=tenant, project_id=project, name="srv", transport=transport, + url="https://mcp.example/mcp", auth_provider_id=ap_id) + s.add(row) + await s.commit() + await s.refresh(row) + return row + + +class _StubResolver: + """Stands in for AuthResolver: records each resolve and hands back a token.""" + + def __init__(self, token="tok-1"): + self.calls: list[dict] = [] + self.token = token + + async def resolve(self, *, tenant_id, project_id, provider_id, context=None, force=False): + self.calls.append({"context": dict(context or {}), "force": force}) + from forge.auth_providers.resolver import ResolvedAuth + return ResolvedAuth(headers={"Authorization": f"Bearer {self.token}"}) + + +async def test_no_auth_provider_means_no_auth_on_the_connection(): + row = await _client("t_ma_n", "p_ma_n") + conn = await mcp_mod._connection_for(row, "t_ma_n", "p_ma_n") + assert "auth" not in conn + + +async def test_auth_provider_attaches_an_httpx_auth_to_the_connection(): + tenant, project = "t_ma_a", "p_ma_a" + ap_id = await _provider(tenant, project, per_user=False) + row = await _client(tenant, project, ap_id) + conn = await mcp_mod._connection_for(row, tenant, project) + assert isinstance(conn.get("auth"), httpx.Auth) + + +async def test_stdio_transport_never_gets_http_auth(): + """A provider on a stdio server is a config mistake; half-applying it would be worse.""" + tenant, project = "t_ma_s", "p_ma_s" + ap_id = await _provider(tenant, project, per_user=False) + row = await _client(tenant, project, ap_id, transport="stdio") + row.command = "echo" + from forge.config import settings + settings.enable_mcp_stdio = True + try: + conn = await mcp_mod._connection_for(row, tenant, project) + finally: + settings.enable_mcp_stdio = False + assert "auth" not in conn + + +async def test_provider_auth_applies_headers_per_request(): + resolver = _StubResolver() + auth = mcp_mod._ProviderAuth(resolver, tenant_id="t", project_id="p", provider_id="ap", context={}) + request = httpx.Request("POST", "https://mcp.example/mcp") + + flow = auth.async_auth_flow(request) + sent = await flow.__anext__() + assert sent.headers["Authorization"] == "Bearer tok-1" + + # A rotated token is picked up on the NEXT request without rebuilding the connection. + resolver.token = "tok-2" + request2 = httpx.Request("POST", "https://mcp.example/mcp") + flow2 = auth.async_auth_flow(request2) + sent2 = await flow2.__anext__() + assert sent2.headers["Authorization"] == "Bearer tok-2" + + +async def test_401_forces_one_refresh_and_retries_once(): + resolver = _StubResolver() + auth = mcp_mod._ProviderAuth(resolver, tenant_id="t", project_id="p", provider_id="ap", context={}) + request = httpx.Request("POST", "https://mcp.example/mcp") + + flow = auth.async_auth_flow(request) + await flow.__anext__() + resolver.token = "fresh" + retried = await flow.asend(httpx.Response(401, request=request)) + assert retried.headers["Authorization"] == "Bearer fresh" + assert resolver.calls[-1]["force"] is True, "the retry must bypass the resolver's cache" + + # Exactly one retry: a second 401 ends the flow rather than looping. + with pytest.raises(StopAsyncIteration): + await flow.asend(httpx.Response(401, request=request)) + + +async def test_success_does_not_re_resolve(): + resolver = _StubResolver() + auth = mcp_mod._ProviderAuth(resolver, tenant_id="t", project_id="p", provider_id="ap", context={}) + request = httpx.Request("POST", "https://mcp.example/mcp") + flow = auth.async_auth_flow(request) + await flow.__anext__() + with pytest.raises(StopAsyncIteration): + await flow.asend(httpx.Response(200, request=request)) + assert len(resolver.calls) == 1 + + +@pytest.fixture +def no_default_ctx_key(monkeypatch): + """`FORGE_DEFAULT_TOKEN_CTX_KEY` is a deployment-wide setting read from the developer's own + .env, and it legitimately makes EVERY provider caller-specific. Pin it off so these tests + assert the shared-pooling case rather than whatever the local machine is configured for.""" + from forge.config import settings + + monkeypatch.setattr(settings, "default_token_ctx_key", "") + + +async def test_shared_provider_pools_one_connection_for_everyone(no_default_ctx_key): + tenant, project = "t_ma_sh", "p_ma_sh" + ap_id = await _provider(tenant, project, per_user=False) + row = await _client(tenant, project, ap_id) + _a, s1 = await mcp_mod._auth_for(row, tenant, project, {"end_user_id": "u1"}) + _b, s2 = await mcp_mod._auth_for(row, tenant, project, {"end_user_id": "u2"}) + assert s1 == s2 == "" + + +async def test_an_inline_run_token_gets_its_own_pooled_connection(no_default_ctx_key): + """A provider can take its token INLINE from the run context (token_ctx_key) instead of from + a stored per-user connection. The resolver varies its own cache on that key; if the MCP cache + key ignores it, the first caller's connection - carrying the first caller's token in its + attached httpx.Auth - is pooled and handed to everyone else for the whole TTL.""" + tenant, project = "t_ma_inline", "p_ma_inline" + async with SessionLocal() as s: + ap = await AuthProviderService.create( + s, tenant, project, name="inline", kind="bearer", + config={"token_ref": "secret://proj/tok", "token_ctx_key": "user_token"}, + ) + ap_id = ap.id + row = await _client(tenant, project, ap_id) + + _a, s1 = await mcp_mod._auth_for(row, tenant, project, {"user_token": "tok-alice"}) + _b, s2 = await mcp_mod._auth_for(row, tenant, project, {"user_token": "tok-bob"}) + _c, s3 = await mcp_mod._auth_for(row, tenant, project, {"user_token": "tok-alice"}) + + assert s1 and s1 != s2, "two callers' inline tokens must not share one pooled connection" + assert s1 == s3, "the same inline token should reuse its own pooled connection" + + +async def test_the_deployment_wide_inline_token_key_also_splits_the_pool(monkeypatch): + """token_ctx_key has a deployment-wide fallback, and the resolver honours it. So must this.""" + from forge.config import settings + + monkeypatch.setattr(settings, "default_token_ctx_key", "user_token") + tenant, project = "t_ma_dflt", "p_ma_dflt" + ap_id = await _provider(tenant, project, per_user=False) + row = await _client(tenant, project, ap_id) + + _a, s1 = await mcp_mod._auth_for(row, tenant, project, {"user_token": "tok-alice"}) + _b, s2 = await mcp_mod._auth_for(row, tenant, project, {"user_token": "tok-bob"}) + assert s1 and s1 != s2 + + +def test_auth_cache_dims_covers_everything_the_resolver_varies_on(): + """A regression guard with teeth: the resolver builds its cache key from + per_user_context_keys + token_ctx_key. If a future dimension is added there and not here, + the pooled MCP connection starts serving one caller's credential to another.""" + from forge.config import settings + + cfg = {"per_user_context_keys": ["end_user_id"], "token_ctx_key": "user_token"} + dims = set(mcp_mod.auth_cache_dims(cfg)) + + per_user = cfg.get("per_user_context_keys", []) + effective_ctx_key = cfg.get("token_ctx_key") or settings.default_token_ctx_key + resolver_dims = set([*per_user, effective_ctx_key] if effective_ctx_key else per_user) + + assert resolver_dims <= dims, f"MCP pooling ignores caller dimension(s) {resolver_dims - dims}" + + +async def test_per_user_provider_gets_a_distinct_connection_per_end_user(): + tenant, project = "t_ma_pu", "p_ma_pu" + ap_id = await _provider(tenant, project, per_user=True) + row = await _client(tenant, project, ap_id) + _a, s1 = await mcp_mod._auth_for(row, tenant, project, {"end_user_id": "u1"}) + _b, s2 = await mcp_mod._auth_for(row, tenant, project, {"end_user_id": "u2"}) + _c, s3 = await mcp_mod._auth_for(row, tenant, project, {"end_user_id": "u1"}) + assert s1 and s2 and s1 != s2, "two end users must not share one authenticated MCP session" + assert s1 == s3, "the same end user must reuse their own pooled session" + + +async def test_missing_provider_degrades_to_no_auth_rather_than_failing(): + tenant, project = "t_ma_m", "p_ma_m" + row = await _client(tenant, project, "does-not-exist") + auth, suffix = await mcp_mod._auth_for(row, tenant, project, {}) + assert auth is None and suffix == "" + + +class _FakeClient: + """Stands in for a MultiServerMCPClient so we can see whether it gets closed.""" + + def __init__(self) -> None: + self.closed = False + + async def aclose(self) -> None: + self.closed = True + + +def _seed(key: str, client, *, created: float, used: float | None = None) -> None: + mcp_mod._CLIENT_CACHE[key] = mcp_mod._Cached( + created=created, used=used if used is not None else created, client=client + ) + + +async def _reset_cache() -> None: + mcp_mod._CLIENT_CACHE.clear() + mcp_mod._RETIRED.clear() + # Retiring a client starts the background reaper. Leaving it sleeping on a loop that pytest + # is about to close produces a "Task was destroyed but it is pending" warning and lets one + # test's reaper drain the next test's queue. + if mcp_mod._REAPER is not None: + mcp_mod._REAPER.cancel() + with contextlib.suppress(BaseException): + await mcp_mod._REAPER + mcp_mod._REAPER = None + + +@pytest.fixture(autouse=True) +async def _clean_cache(): + """The cache, the retirement queue and the reaper task are module globals; a test that + leaves entries in any of them changes what the next test's eviction sees.""" + await _reset_cache() + yield + await _reset_cache() + + +async def test_invalidate_client_drops_and_closes_every_per_user_variant(): + """Popping an entry without closing it strands a socket nothing will ever reclaim. + + This path closes IMMEDIATELY rather than retiring: it runs when the server row was edited or + deleted, so the connection points at config that no longer exists. + """ + mine = {k: _FakeClient() for k in ("cid", "cid::abc", "cid::def")} + other = _FakeClient() + for k, c in mine.items(): + _seed(k, c, created=0.0) + _seed("other", other, created=0.0) + + await mcp_mod.invalidate_client("cid") + + assert set(mcp_mod._CLIENT_CACHE) == {"other"} + assert all(c.closed for c in mine.values()), "dropped connections must be closed" + assert not other.closed, "an unrelated server's connection must survive" + + +async def test_cache_is_bounded_so_per_user_keys_cannot_grow_without_limit(): + """The cache key carries a per-caller suffix and every catalog MCP connector is per-user, so + the cache grows with distinct PEOPLE. Without a ceiling a busy project pins one live + transport per user for the lifetime of the process.""" + import time + + now = time.monotonic() + clients = [] + for i in range(mcp_mod._CACHE_MAX + 10): + c = _FakeClient() + clients.append(c) + # Ascending `used` so "least recently used" is unambiguous. + _seed(f"cid::{i:04d}", c, created=now, used=now + i) + + mcp_mod._evict(now + 1) + + assert len(mcp_mod._CLIENT_CACHE) == mcp_mod._CACHE_MAX, "the ceiling is honoured immediately" + # Shed connections are NOT closed on the spot - a caller may still be running against one. + assert not any(c.closed for c in clients), "eviction must not abort an in-flight call" + + await mcp_mod._reap(now + 1 + mcp_mod._CLOSE_GRACE) + evicted = [c for c in clients if c.closed] + assert len(evicted) == 10 + assert clients[:10] == evicted, "eviction should drop the least recently used entries first" + + +async def test_eviction_is_least_recently_used_not_oldest_connection(): + """The hot shared connection is the one built first and used constantly. Evicting by build + time would shed it before any of the idle per-user connections that displaced it.""" + import time + + now = time.monotonic() + hot = _FakeClient() + _seed("shared", hot, created=now, used=now + 10_000) # built first, used most recently + idle = [] + for i in range(mcp_mod._CACHE_MAX): + c = _FakeClient() + idle.append(c) + _seed(f"cid::{i:04d}", c, created=now + 1 + i, used=now + i) + + mcp_mod._evict(now + 1) + + assert "shared" in mcp_mod._CLIENT_CACHE, "the busiest connection must not be evicted first" + assert "cid::0000" not in mcp_mod._CLIENT_CACHE + + +async def test_expired_entries_are_retired_and_then_closed(): + import time + + stale = _FakeClient() + fresh = _FakeClient() + now = time.monotonic() + _seed("a", stale, created=now - mcp_mod._CACHE_TTL - 1) + _seed("b", fresh, created=now) + + mcp_mod._evict(now) + + assert set(mcp_mod._CLIENT_CACHE) == {"b"} + assert not stale.closed, "the grace period lets an in-flight call finish" + await mcp_mod._reap(now + mcp_mod._CLOSE_GRACE) + assert stale.closed and not fresh.closed + + +async def test_a_connection_being_rebuilt_is_never_evicted(): + """Evicting a locked entry detaches it from the cache while its builder is still writing + into it - producing a live client nothing tracks and nothing will ever close.""" + import time + + now = time.monotonic() + building = mcp_mod._Cached(created=0.0, used=now) + await building.lock.acquire() + try: + mcp_mod._CLIENT_CACHE["mid-build"] = building + for i in range(mcp_mod._CACHE_MAX + 5): + _seed(f"cid::{i:04d}", _FakeClient(), created=now, used=now + i) + + mcp_mod._evict(now + 1) + + assert mcp_mod._CLIENT_CACHE.get("mid-build") is building + finally: + building.lock.release() + + +async def test_invalidating_a_server_mid_build_does_not_orphan_the_new_connection(): + """`invalidate_client` doesn't wait for an in-flight build, so the builder can finish into an + entry that is no longer in the cache. Nothing would then hold that transport, and nothing + would ever close it.""" + import time + + tenant, project = "t_ma_midb", "p_ma_midb" + row = await _client(tenant, project) + built = _FakeClient() + + class _Adapters: + def __call__(self, _servers): + return built + + async def _connection(*a, **kw): + # The server row is deleted (and the cache invalidated) while we are connecting. + await mcp_mod.invalidate_client(row.id) + return {"url": "https://mcp.example/mcp", "transport": "streamable_http"} + + async def _no_tools(): + return [] + + built.get_tools = _no_tools # type: ignore[attr-defined] + real_adapters, real_connection = mcp_mod._require_adapters, mcp_mod._connection_for + mcp_mod._require_adapters = lambda: _Adapters() # type: ignore[assignment] + mcp_mod._connection_for = _connection # type: ignore[assignment] + try: + await mcp_mod._client_and_tools(row, tenant, project, {}) + finally: + # Restore, don't delete: these are the module's own functions, and `del` would remove + # them outright for every test that runs after this one. + mcp_mod._require_adapters = real_adapters # type: ignore[assignment] + mcp_mod._connection_for = real_connection # type: ignore[assignment] + + assert row.id not in mcp_mod._CLIENT_CACHE, "the invalidation stands" + assert not built.closed, "the caller's own call must still be able to finish" + await mcp_mod._reap(time.monotonic() + mcp_mod._CLOSE_GRACE + 1) + assert built.closed, "the detached connection must still be closed" + + +async def test_close_all_drains_retired_connections_too(): + import time + + now = time.monotonic() + retired = _FakeClient() + live = _FakeClient() + mcp_mod._retire(retired, now) + _seed("a", live, created=now) + + await mcp_mod.close_all() + + assert live.closed and retired.closed, "shutdown must not leave a retired transport open" + assert not mcp_mod._CLIENT_CACHE and not mcp_mod._RETIRED + + +async def test_retired_transports_are_closed_without_a_second_cache_miss(monkeypatch): + """Retirement used to be drained only by `_evict`, which runs only inside a cache BUILD. + + So a burst that pushed the cache over its ceiling retired N transports, and then - if traffic + settled into a steady state where every call hit a live entry - nothing ever built again and + those transports stayed open until a key expired or the process shut down, far past the 30s + grace they were given. Reaping must not depend on someone missing the cache. + + No cache operation of any kind happens after the eviction here: the only thing that can close + these is the background reaper. + """ + import time + + monkeypatch.setattr(mcp_mod, "_REAP_INTERVAL", 0.01) + now = time.monotonic() + shed = [] + for i in range(mcp_mod._CACHE_MAX + 5): + c = _FakeClient() + shed.append(c) + _seed(f"cid::{i:04d}", c, created=now, used=now + i) + + mcp_mod._evict(now) + retired = [c for c in shed if c in [client for _, client in mcp_mod._RETIRED]] + assert len(retired) == 5, "the ceiling should have shed five connections" + assert not any(c.closed for c in retired), "the grace period lets in-flight calls finish" + + # Nothing touches the cache from here on. Advance past the grace period and let the reaper + # run on its own. + monkeypatch.setattr(mcp_mod, "_CLOSE_GRACE", 0.0) + mcp_mod._RETIRED[:] = [(0.0, c) for _, c in mcp_mod._RETIRED] + for _ in range(200): + await asyncio.sleep(0.01) + if all(c.closed for c in retired): + break + + assert all(c.closed for c in retired), ( + "retired transports must be closed on a timer, not only when the next build misses" + ) + assert not mcp_mod._RETIRED + + +async def test_eviction_never_closes_a_transport_while_holding_the_build_lock(): + """`_evict` runs inside `_client_and_tools`'s per-key build lock. Closing a transport there + is an unbounded network await performed while a concurrent caller for that same key is + blocked, which is exactly the cost the retirement queue exists to avoid. + + Enforced structurally rather than by timing: `_evict` is synchronous, so it CANNOT await a + close no matter how it is later edited. + """ + import inspect + + assert not inspect.iscoroutinefunction(mcp_mod._evict), ( + "_evict must stay synchronous - an await here happens under the build lock" + ) + + # And it really does only shed, never close. + import time + + now = time.monotonic() + stale = _FakeClient() + _seed("a", stale, created=now - mcp_mod._CACHE_TTL - 1) + mcp_mod._evict(now) + assert "a" not in mcp_mod._CLIENT_CACHE and not stale.closed + + +async def test_the_reaper_stops_itself_once_the_queue_drains(monkeypatch): + """One long-lived task, and only while there is something to close - an idle process must not + hold a timer open forever, and a retirement after that must start it again.""" + import time + + monkeypatch.setattr(mcp_mod, "_REAP_INTERVAL", 0.01) + monkeypatch.setattr(mcp_mod, "_CLOSE_GRACE", 0.0) + + first = _FakeClient() + mcp_mod._retire(first, time.monotonic()) + assert mcp_mod._REAPER is not None and not mcp_mod._REAPER.done() + + for _ in range(200): + await asyncio.sleep(0.01) + if mcp_mod._REAPER.done(): + break + assert first.closed + assert mcp_mod._REAPER.done(), "the reaper must exit once nothing is waiting to be closed" + + second = _FakeClient() + mcp_mod._retire(second, time.monotonic()) + assert not mcp_mod._REAPER.done(), "a later retirement must start a new reaper" + for _ in range(200): + await asyncio.sleep(0.01) + if second.closed: + break + assert second.closed + + +def test_auth_context_from_puts_end_user_last(): + """end_user_id is authoritative: a caller-injected run_context value must not shadow it, + or one user could resolve another's stored credential.""" + + class _Ctx: + run_context = {"end_user_id": "spoofed", "csrf": "x"} + end_user = {"id": "real-user", "email": "real@acme.com"} + + out = mcp_mod.auth_context_from(_Ctx()) + assert out["end_user_id"] == "real-user" + assert out["csrf"] == "x" + + +async def test_retry_after_401_does_not_double_apply_params_or_cookies(): + """Headers overwrite on re-apply, but params are MERGED and the cookie jar is CONCATENATED. + Applying twice to the same request sends `?api_key=X&api_key=X` and `Cookie: s=1; s=1`, + which some servers reject - turning a recoverable 401 into a hard failure.""" + import httpx + + from forge.auth_providers.resolver import ResolvedAuth + + class _Resolver: + def __init__(self) -> None: + self.calls = 0 + + async def resolve(self, **kw): + self.calls += 1 + return ResolvedAuth( + headers={"X-Auth": f"tok{self.calls}"}, + params={"api_key": "K"}, + cookies={"sess": "S"}, + ) + + resolver = _Resolver() + auth = mcp_mod._ProviderAuth(resolver, tenant_id="t", project_id="p", + provider_id="ap", context={}) + request = httpx.Request("POST", "https://mcp.example.com/mcp?keep=1") + + flow = auth.async_auth_flow(request) + first = await flow.__anext__() + assert first.url.params.get_list("api_key") == ["K"] + try: + second = await flow.asend(httpx.Response(401, request=first)) + except StopAsyncIteration: # pragma: no cover - the retry is expected to happen + raise AssertionError("a 401 should have triggered exactly one retry") from None + + assert second.url.params.get_list("api_key") == ["K"], "auth param was merged twice" + assert second.url.params.get("keep") == "1", "the request's own params must survive" + assert second.headers["Cookie"] == "sess=S", "cookie jar was concatenated twice" + assert second.headers["X-Auth"] == "tok2", "the retry must use the freshly forced credential" + assert resolver.calls == 2 diff --git a/apps/api/tests/test_oauth.py b/apps/api/tests/test_oauth.py index bbbf153..df3dd2a 100644 --- a/apps/api/tests/test_oauth.py +++ b/apps/api/tests/test_oauth.py @@ -101,6 +101,58 @@ async def test_oauth_not_connected_raises(): await AuthResolver().resolve(tenant_id="t_none", project_id="p_none", provider_id="apx", provider=_ap("apx", "t_none", "p_none"), force=True) +class _FakeStore: + async def read_ref(self, **kw): + return "cid-123" + + +async def _authorize_query(cfg: dict) -> dict[str, list[str]]: + from urllib.parse import parse_qs, urlparse + + from forge.auth_providers.oauth_flow import build_authorize_url + + ap = AuthProvider(id="ap_q", tenant_id="t", project_id="p", name="idp", + kind="oauth2_authorization_code", config=cfg) + url = await build_authorize_url(ap, tenant_id="t", project_id="p", secrets=_FakeStore()) + return parse_qs(urlparse(url).query) + + +async def test_authorize_params_is_the_only_way_to_add_authorize_query_params(): + """There is ONE mechanism for vendor-specific authorize parameters. + + There used to be two: a generic `authorize_params` dict, and a top-level special case reading + `cfg["access_type"]` / `cfg["prompt"]`. Nothing ever wrote the top-level spelling, so that + branch was dead on every path a connector takes - while looking authoritative enough that + four Google manifests shipped without asking for offline access. A second spelling that + works in some places and not others is how that recurs, so the top-level one is gone and + this pins it. + """ + q = await _authorize_query({**_CFG, "access_type": "offline", "prompt": "consent"}) + assert "access_type" not in q, ( + "top-level access_type must not be a second, undocumented spelling - use authorize_params" + ) + assert "prompt" not in q + + q = await _authorize_query({**_CFG, "authorize_params": {"access_type": "offline", "user_scope": "chat:write"}}) + assert q["access_type"] == ["offline"] and q["user_scope"] == ["chat:write"] + + +async def test_authorize_params_cannot_trample_the_protocol_parameters(): + """The extras are vendor data, applied last - they must never be able to redirect the + callback somewhere else or drop PKCE down to plain.""" + q = await _authorize_query({ + **_CFG, + "authorize_params": { + "redirect_uri": "https://attacker.example/steal", + "code_challenge_method": "plain", + "state": "forged", + }, + }) + assert q["redirect_uri"] != ["https://attacker.example/steal"] + assert q["code_challenge_method"] == ["S256"] + assert q["state"] != ["forged"] + + async def test_per_user_connect_bundle_is_resolvable(): """Item 5: the connect callback now stores the bundle under the SAME per-user secret name that resolve/refresh read. Before the fix it wrote the default name, so a per-user provider's diff --git a/apps/api/tests/test_projects.py b/apps/api/tests/test_projects.py index d990b2e..cdb9b0c 100644 --- a/apps/api/tests/test_projects.py +++ b/apps/api/tests/test_projects.py @@ -61,13 +61,13 @@ async def test_project_counts_are_scoped_to_project(): counts = await ProjectService.counts(session, tenant_id, proj.id) assert counts == { "workflows": 2, "agents": 1, "tools": 3 + len(BUILTIN_DEFAULTS), - "components": 1, "knowledge": 1, "auth": 1, "handoffs": 1, + "components": 1, "knowledge": 1, "auth": 1, "handoffs": 1, "connectors": 0, } # A fresh project is all zeros except the auto-provisioned platform built-ins (never None). empty = await ProjectService.create(session, tenant_id, name="Empty", slug="empty") assert await ProjectService.counts(session, tenant_id, empty.id) == { "workflows": 0, "agents": 0, "tools": len(BUILTIN_DEFAULTS), "components": 0, "knowledge": 0, "auth": 0, - "handoffs": 0, + "handoffs": 0, "connectors": 0, } diff --git a/apps/api/tests/test_triggers.py b/apps/api/tests/test_triggers.py index 6ee0a5d..06234c8 100644 --- a/apps/api/tests/test_triggers.py +++ b/apps/api/tests/test_triggers.py @@ -2,16 +2,35 @@ from __future__ import annotations +from contextlib import aclosing from datetime import datetime, timedelta +import httpx from langgraph.checkpoint.memory import InMemorySaver from forge.db.base import SessionLocal -from forge.models import Trigger, Workflow +from forge.main import create_app +from forge.models import Trigger, User, Workflow +from forge.security import create_access_token from forge.services.dispatch import dispatch_trigger from forge.services.runs import RunService from forge.services.triggers import TriggerService + +async def _client_for(user_id: str, tenant: str, role: str) -> httpx.AsyncClient: + c = httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test") + c.headers["Authorization"] = f"Bearer {create_access_token(user_id=user_id, tenant_id=tenant, role=role)}" + return c + + +async def _make_user_role(tenant: str, email: str, role: str) -> str: + async with SessionLocal() as s: + u = User(tenant_id=tenant, email=email, role=role, status="active") + s.add(u) + await s.commit() + await s.refresh(u) + return u.id + _WEBHOOK_WF = { "id": "wf_hook", "version": 1, "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}}, @@ -72,3 +91,372 @@ async def test_dispatch_webhook_runs_workflow(): rs = RunService(checkpointer=InMemorySaver()) result = await dispatch_trigger(rs, trigger, {"text": "ping"}) assert result.get("answer") == "Done." and result.get("status") == "done" + + +# --- who an unattended run acts as ---------------------------------------------------------- +# +# Nobody is signed in when a webhook or a schedule fires, but every catalog connector is +# per-user. Without an identity on the trigger, a scheduled workflow has no token to resolve and +# dies at its first tool call - so the trigger carries the person who set it up. + +async def test_sync_stamps_the_editor_as_the_run_as_identity(): + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_owner", project_id="p_owner", name="Owned", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf, owner="user-alice") + assert trig.run_as_user_id == "user-alice" + + +async def test_a_colleague_editing_the_workflow_does_not_take_over_the_accounts(): + """The automation runs on whoever's Gmail is connected behind it. Repointing that at the last + person to touch the canvas would change what the workflow can see without anyone deciding to.""" + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_owner2", project_id="p_owner2", name="Owned2", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + await TriggerService.sync_from_workflow(s, wf, owner="user-alice") + [trig] = await TriggerService.sync_from_workflow(s, wf, owner="user-bob") + assert trig.run_as_user_id == "user-alice" + + +async def test_a_trigger_predating_the_column_adopts_the_next_editor(): + """Triggers that already exist have no owner. The next save claims them, so an upgrade + doesn't leave every existing automation permanently unable to use a connector.""" + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_owner3", project_id="p_owner3", name="Owned3", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [before] = await TriggerService.sync_from_workflow(s, wf) # pre-upgrade shape + assert before.run_as_user_id is None + [after] = await TriggerService.sync_from_workflow(s, wf, owner="user-carol") + assert after.run_as_user_id == "user-carol" + + +async def test_dispatch_binds_the_run_to_the_trigger_owner(): + """The payoff: the run carries an identity, so a per-user auth provider resolves THAT + person's connected credential instead of failing with "no acting user".""" + from forge.models import Run, Thread + + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_ru", project_id="p_ru", name="RunAs", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trigger] = await TriggerService.sync_from_workflow(s, wf, owner="user-dana") + + rs = RunService(checkpointer=InMemorySaver()) + result = await dispatch_trigger(rs, trigger, {"text": "ping"}) + assert result.get("status") == "done" + + async with SessionLocal() as s: + run = await s.get(Run, result["run_id"]) + thread = await s.get(Thread, run.thread_id) + # The dim AuthResolver.bundle_secret_name hashes for a per-user provider. + assert (thread.meta or {}).get("end_user") == {"id": "user-dana"} + assert thread.user_external_id == "user-dana" + + +async def test_run_as_can_be_claimed_by_yourself_but_assigned_only_by_an_editor(): + """Claiming narrows a trigger to accounts YOU connected, so anyone may. Pointing it at + someone else means their credentials get used by a workflow they may never have touched.""" + import uuid + + app = create_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c: + reg = await c.post("/v1/auth/register", + json={"email": f"t{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + me = (await c.get("/v1/auth/me")).json() + tenant, owner_id = me["tenant_id"], me["id"] + pid = (await c.post("/v1/projects", json={"name": "T", "slug": f"trig-{uuid.uuid4().hex[:8]}"})).json()["id"] + + async with SessionLocal() as s: + wf = Workflow(tenant_id=tenant, project_id=pid, name="Hooked", executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf) + tid = trig.id + other = User(tenant_id=tenant, email=f"o{uuid.uuid4().hex[:8]}@example.com", role="viewer", status="active") + s.add(other) + await s.commit() + await s.refresh(other) + other_id = other.id + + rows = (await c.get(f"/v1/projects/{pid}/triggers")).json() + assert rows[0]["run_as_user_id"] is None + assert rows[0]["run_as_email"] is None + + # An editor may hand it to someone else. + r = await c.put(f"/v1/projects/{pid}/triggers/{tid}/run-as", json={"user_id": other_id}) + assert r.status_code == 200, r.text + assert r.json()["run_as_user_id"] == other_id + # ...but not to a user who isn't in the workspace. + assert (await c.put(f"/v1/projects/{pid}/triggers/{tid}/run-as", + json={"user_id": "nope"})).status_code == 404 + + # That viewer can claim it back for themselves without any elevated role. + token = create_access_token(user_id=other_id, tenant_id=tenant, role="viewer") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test") as c2: + c2.headers["Authorization"] = f"Bearer {token}" + assert (await c2.put(f"/v1/projects/{pid}/triggers/{tid}/run-as", json={})).status_code == 200 + row = (await c2.get(f"/v1/projects/{pid}/triggers")).json()[0] + assert row["run_as_is_me"] is True and row["run_as_email"] + # ...but may not push it onto a colleague. + assert (await c2.put(f"/v1/projects/{pid}/triggers/{tid}/run-as", + json={"user_id": owner_id})).status_code == 403 + + +# --- whose automation is it ----------------------------------------------------------------- +# +# Independent of run_as: `scope` says who the trigger BELONGS to. A salesperson's lead-chaser is +# theirs; a platform team's prod monitor is the project's. + +def test_new_triggers_default_to_the_project_for_admins_and_to_the_person_otherwise(): + from forge.services.triggers import default_scope_for + + assert default_scope_for("owner") == "project" + assert default_scope_for("admin") == "project" + assert default_scope_for("editor") == "user" + assert default_scope_for("viewer") == "user" + # An unknown/absent role must not accidentally publish something to the whole team. + assert default_scope_for(None) == "user" + + +async def test_scope_is_set_on_creation_and_never_flipped_by_an_edit(): + """Editing a workflow must not move a team automation into someone's private list, nor + publish someone's private one to everybody.""" + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_scope", project_id="p_scope", name="Scoped", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf, owner="user-alice", scope="user") + assert trig.scope == "user" + # An admin later edits the same workflow - the trigger stays Alice's. + [again] = await TriggerService.sync_from_workflow(s, wf, owner="user-admin", scope="project") + assert again.scope == "user" + + +async def test_pre_existing_triggers_stay_visible_to_everyone(): + """The upgrade must not make a team's existing automations vanish from their screen, so a + trigger synced with no scope is the project's.""" + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_scope2", project_id="p_scope2", name="Legacy", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf) + assert trig.scope == "project" + + +async def test_personal_triggers_are_listed_only_for_their_owner_and_project_admins(): + import uuid + + app = create_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c: + reg = await c.post("/v1/auth/register", + json={"email": f"t{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + me = (await c.get("/v1/auth/me")).json() + tenant, admin_id = me["tenant_id"], me["id"] + pid = (await c.post("/v1/projects", json={"name": "T", "slug": f"sc-{uuid.uuid4().hex[:8]}"})).json()["id"] + + async with SessionLocal() as s: + alice = User(tenant_id=tenant, email=f"alice{uuid.uuid4().hex[:6]}@example.com", role="editor", status="active") + bob = User(tenant_id=tenant, email=f"bob{uuid.uuid4().hex[:6]}@example.com", role="editor", status="active") + s.add_all([alice, bob]) + await s.commit() + await s.refresh(alice) + await s.refresh(bob) + alice_id, bob_id = alice.id, bob.id + + wf = Workflow(tenant_id=tenant, project_id=pid, name="Alice's chaser", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf, owner=alice_id, scope="user") + tid = trig.id + + # The registering user is the workspace owner -> sees it, flagged as not theirs. + rows = (await c.get(f"/v1/projects/{pid}/triggers")).json() + assert [r["id"] for r in rows] == [tid] + assert rows[0]["scope"] == "user" and rows[0]["visible_via_oversight"] is True + + # Alice sees her own. + ca = await _client_for(alice_id, tenant, "editor") + async with aclosing(ca): + rows = (await ca.get(f"/v1/projects/{pid}/triggers")).json() + assert [r["id"] for r in rows] == [tid] + assert rows[0]["run_as_is_me"] is True and rows[0]["visible_via_oversight"] is False + + # Bob, an equal-ranking colleague, does not. + cb = await _client_for(bob_id, tenant, "editor") + async with aclosing(cb): + assert (await cb.get(f"/v1/projects/{pid}/triggers")).json() == [] + + # ...and cannot make someone else's trigger personal-to-nobody or grab it. + r = await cb.put(f"/v1/projects/{pid}/triggers/{tid}/scope", json={"scope": "project"}) + assert r.status_code == 200, "an editor may share a trigger with the project" + # Now that it's the project's, Bob can see it. + assert len((await cb.get(f"/v1/projects/{pid}/triggers")).json()) == 1 + + # Alice can take it back to personal - it is her account doing the work. + ca2 = await _client_for(alice_id, tenant, "editor") + async with aclosing(ca2): + assert (await ca2.put(f"/v1/projects/{pid}/triggers/{tid}/scope", json={"scope": "user"})).status_code == 200 + cb2 = await _client_for(bob_id, tenant, "editor") + async with aclosing(cb2): + assert (await cb2.get(f"/v1/projects/{pid}/triggers")).json() == [] + assert admin_id # (the owner above) + + +async def test_a_viewer_cannot_publish_a_trigger_to_the_whole_project(): + """Sharing makes colleagues see and depend on it, so it is an editor decision.""" + import uuid + + app = create_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c: + reg = await c.post("/v1/auth/register", + json={"email": f"t{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + tenant = (await c.get("/v1/auth/me")).json()["tenant_id"] + pid = (await c.post("/v1/projects", json={"name": "T", "slug": f"sc-{uuid.uuid4().hex[:8]}"})).json()["id"] + async with SessionLocal() as s: + wf = Workflow(tenant_id=tenant, project_id=pid, name="V", executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + owner = await _make_user_role(tenant, f"o{uuid.uuid4().hex[:6]}@example.com", "editor") + [trig] = await TriggerService.sync_from_workflow(s, wf, owner=owner, scope="user") + tid = trig.id + assert trig.scope == "user" + + viewer_id = await _make_user_role(tenant, f"v{uuid.uuid4().hex[:6]}@example.com", "viewer") + cv = await _client_for(viewer_id, tenant, "viewer") + async with aclosing(cv): + assert (await cv.put(f"/v1/projects/{pid}/triggers/{tid}/scope", + json={"scope": "project"})).status_code == 403 + # An unknown scope is rejected outright rather than silently stored. + assert (await cv.put(f"/v1/projects/{pid}/triggers/{tid}/scope", + json={"scope": "everyone"})).status_code == 422 + + +def test_a_machine_principal_is_not_a_run_as_identity(): + """A workflow can be saved by the service token or a scoped API key. Neither has connected + accounts, and `apikey:` is 43 characters going into a String(36) column - which + Postgres rejects, inside a trigger sync whose exceptions are swallowed. The visible symptom + would be webhooks and schedules silently never being registered.""" + from forge.services.triggers import owner_id_for + + assert owner_id_for("service") is None + assert owner_id_for(f"apikey:{'a' * 36}") is None + assert owner_id_for("x" * 37) is None, "anything too long for the column is not an owner" + real = "3f8b1c22-9a1e-4f77-8c31-2b6d5e0a7c44" + assert owner_id_for(real) == real + assert owner_id_for(None) is None + + +async def test_a_workflow_saved_by_a_machine_principal_yields_a_shared_trigger(): + """No owner means nobody to be personal TO: "listed only for the person it runs as" hides a + trigger from everyone when that person doesn't exist. So an unowned trigger stays the + project's, whatever default scope the caller's role suggested.""" + async with SessionLocal() as s: + wf = Workflow(tenant_id="t_trig_svc", project_id="p_trig_svc", name="Svc", + executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow( + s, wf, owner=f"apikey:{'b' * 36}", scope="user", + ) + assert trig.run_as_user_id is None + assert trig.scope == "project", "an ownerless trigger must not be invisible to everyone" + + +async def test_an_editor_cannot_make_someone_elses_trigger_personal(): + """"Personal" means "listed only for the person it runs as". An editor doing it to a trigger + that runs as somebody else removes it from their OWN screen the moment they click, with no + control left to undo it.""" + import uuid + + app = create_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c: + reg = await c.post("/v1/auth/register", + json={"email": f"t{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + tenant = (await c.get("/v1/auth/me")).json()["tenant_id"] + pid = (await c.post("/v1/projects", json={"name": "T", "slug": f"mp-{uuid.uuid4().hex[:8]}"})).json()["id"] + + colleague = await _make_user_role(tenant, f"c{uuid.uuid4().hex[:6]}@example.com", "editor") + editor = await _make_user_role(tenant, f"e{uuid.uuid4().hex[:6]}@example.com", "editor") + async with SessionLocal() as s: + wf = Workflow(tenant_id=tenant, project_id=pid, name="P", executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf, owner=colleague, scope="project") + tid = trig.id + + ce = await _client_for(editor, tenant, "editor") + async with aclosing(ce): + r = await ce.put(f"/v1/projects/{pid}/triggers/{tid}/scope", json={"scope": "user"}) + assert r.status_code == 403, "an editor is not the person this trigger runs as" + # It is still on their screen, which is the point. + listed = (await ce.get(f"/v1/projects/{pid}/triggers")).json() + assert any(t["id"] == tid for t in listed) + + # The person it actually runs as may. + cc = await _client_for(colleague, tenant, "editor") + async with aclosing(cc): + assert (await cc.put(f"/v1/projects/{pid}/triggers/{tid}/scope", + json={"scope": "user"})).status_code == 200 + + +async def test_a_trigger_with_no_owner_cannot_be_made_personal(): + """Personal to nobody is listed for nobody. Refuse it and say what to do instead.""" + import uuid + + app = create_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c: + reg = await c.post("/v1/auth/register", + json={"email": f"t{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"}) + c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}" + tenant = (await c.get("/v1/auth/me")).json()["tenant_id"] + pid = (await c.post("/v1/projects", json={"name": "T", "slug": f"noown-{uuid.uuid4().hex[:8]}"})).json()["id"] + async with SessionLocal() as s: + wf = Workflow(tenant_id=tenant, project_id=pid, name="N", executable=_WEBHOOK_WF, status="active") + s.add(wf) + await s.commit() + await s.refresh(wf) + [trig] = await TriggerService.sync_from_workflow(s, wf) + tid = trig.id + assert trig.run_as_user_id is None + + r = await c.put(f"/v1/projects/{pid}/triggers/{tid}/scope", json={"scope": "user"}) + assert r.status_code == 400 + assert "Runs as" in r.json()["detail"] + + +async def test_an_unowned_trigger_still_runs_for_workflows_that_need_no_identity(): + """Not every workflow touches a per-user connector. One that doesn't must keep firing on a + trigger nobody has claimed - the identity is a requirement of the tools, not of dispatch.""" + wf = await _make_wf("t_noowner", "p_noowner") + async with SessionLocal() as s: + trig = (await s.execute(Trigger.__table__.select().where(Trigger.workflow_id == wf.id))).fetchone() + trigger = await TriggerService.by_key(s, trig.key) + assert trigger.run_as_user_id is None + rs = RunService(checkpointer=InMemorySaver()) + result = await dispatch_trigger(rs, trigger, {"text": "ping"}) + assert result.get("status") == "done" diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index b36131f..a0ea2ce 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -15,7 +15,7 @@ import { TracesScreen } from "@/components/screens/traces"; import { AuthProvidersScreen } from "@/components/screens/auth"; import { SettingsScreen } from "@/components/screens/settings"; import { ConnectScreen } from "@/components/screens/deploy"; -import { McpClientsScreen } from "@/components/screens/mcp"; +import { ConnectorsScreen } from "@/components/screens/connectors"; import { ChannelsScreen, TriggersScreen, DatasetsScreen, HandoffScreen } from "@/components/screens/platform"; import { Icon } from "@/components/icons"; import { api, Agent, ComponentT, DashboardStats, Project, Tool, Workflow } from "@/lib/api"; @@ -29,7 +29,7 @@ const SCREEN_LABEL: Record = { overview: "Overview", workflows: "Workflows", "workflow-canvas": "Support Router", agents: "Agents", "agent-config": "billing_agent", tools: "Tools", "tool-builder": "Tool", components: "Components", "component-builder": "Component", auth: "Auth Providers", knowledge: "Knowledge", playground: "Playground", traces: "Traces", - connect: "Connect", mcp: "External MCP", settings: "Settings", + connect: "Connect", connectors: "Connectors", settings: "Settings", channels: "Channels", triggers: "Triggers", datasets: "Evaluations", handoff: "Agent inbox", embed: "Embed", }; const PARENT: Record = { @@ -44,6 +44,8 @@ function App() { const [projects, setProjects] = useState([]); const [loaded, setLoaded] = useState(false); const [selTool, setSelTool] = useState(null); + // Tool set to focus when the Tools screen opens (set by "Open in Tools" on a connector). + const [focusToolSet, setFocusToolSet] = useState(null); const [selWorkflow, setSelWorkflow] = useState(null); const [selAgent, setSelAgent] = useState(null); const [selComponent, setSelComponent] = useState(null); @@ -81,7 +83,12 @@ function App() { const project = view.project ? cards.find((c) => c.id === view.project) || projects.find((p) => p.id === view.project) : null; const go = (v: View) => setView(v); - const navScreen = (screen: string) => setView((v) => ({ ...v, name: "project", screen })); + const navScreen = (screen: string) => { + // The connector -> tool-set focus is a one-shot hand-off; clear it on any OTHER navigation + // so returning to Tools later shows the whole list rather than a stale filter. + if (screen !== "tools") setFocusToolSet(null); + setView((v) => ({ ...v, name: "project", screen })); + }; async function deleteProject(projectToDelete: { id: string; name: string }, opts?: { skipConfirm?: boolean }) { if (!opts?.skipConfirm && !window.confirm(`Delete project "${projectToDelete.name}"?\n\nThis removes its workflows, agents, tools, auth providers, knowledge, secrets, runs, and traces. This cannot be undone.`)) return; await api.deleteProject(projectToDelete.id); @@ -192,12 +199,14 @@ function App() { case "workflow-canvas": return navScreen("workflows")} onRun={() => navScreen("playground")} onRegisterFlush={(fn) => { canvasFlushRef.current = fn; }} />; case "agents": return { setSelAgent(a); navScreen("agent-config"); }} />; case "agent-config": return navScreen("agents")} />; - case "tools": return { setSelTool(t); navScreen("tool-builder"); }} />; + case "tools": return { setSelTool(t); navScreen("tool-builder"); }} />; case "tool-builder": return navScreen("tools")} />; case "components": return { setSelComponent(c); navScreen("component-builder"); }} />; case "component-builder": return navScreen("components")} />; case "auth": return ; - case "mcp": return ; + // "Open in Tools" from a connector jumps to the Tools screen focused on its tool set, + // so the actions a connector just created are one click from the canvas. + case "connectors": return { setFocusToolSet(setId); navScreen("tools"); }} />; case "knowledge": return ; case "channels": return ; case "triggers": return ; diff --git a/apps/web/components/canvas/AgentConfig.tsx b/apps/web/components/canvas/AgentConfig.tsx index 1651722..70a55d3 100644 --- a/apps/web/components/canvas/AgentConfig.tsx +++ b/apps/web/components/canvas/AgentConfig.tsx @@ -18,6 +18,9 @@ export function AgentConfig({ config, onChange, tools = [], toolSets = [], agent const selectedTools: string[] = config.tools || []; const selectedComponents: string[] = config.components || []; const mwCount = (config.middleware || []).filter((m: MW) => m.enabled !== false).length; + // Servers a person registered by hand. A connector's server is excluded because its actions + // are already grantable above as that connector's tool set. + const rawMcpServers = mcpServers.filter((m) => !m.connector_slug); const toggleTool = (id: string) => set({ tools: selectedTools.includes(id) ? selectedTools.filter((t) => t !== id) : [...selectedTools, id] }); @@ -130,6 +133,9 @@ export function AgentConfig({ config, onChange, tools = [], toolSets = [], agent
toggleOpen(s.id)}>
+ {/* A connector's set carries its icon, so Slack/Gmail/… are recognizable + at a glance rather than reading as anonymous folders. */} + {s.icon && } {s.name} {sel}/{members.length}
@@ -174,7 +180,7 @@ export function AgentConfig({ config, onChange, tools = [], toolSets = [], agent
)} {toolSets.length === 0 && ungrouped.length === 0 && ( -
No tools yet — create some on the Tools screen.
+
No tools yet — install a connector (Slack, Gmail, …) on the Connectors screen, or build one on the Tools screen.
)} @@ -260,10 +266,14 @@ export function AgentConfig({ config, onChange, tools = [], toolSets = [], agent - {mcpServers.length > 0 && ( -
+ {/* Connector-owned servers are deliberately absent: their actions are already offered + above as a tool set, and listing the same integration twice - once as "Notion" the set + and again as "Notion" the server - makes one connection look like two choices that do + the same thing. Only hand-registered servers need a grant here. */} + {rawMcpServers.length > 0 && ( +
- {mcpServers.map((m) => { + {rawMcpServers.map((m) => { const on = (config.mcp_servers || []).includes(m.id); return ( + MCP servers · advanced +
+
+ + ); + } + + return ( +
+
+ + {/* header */} +
+
Connectors
+
+ {searching || q ? ( + setQ(e.target.value)} + onBlur={() => { if (!q) setSearching(false); }} + onKeyDown={(e) => { if (e.key === "Escape") { setQ(""); setSearching(false); } }} /> + ) : ( + + )} +
+ + {addOpen && ( + <> +
setAddOpen(false)} /> +
+ { setAddOpen(false); setCustomOpen(true); }} /> + { setAddOpen(false); setView("mcp"); }} /> +
+ + )} +
+
+
+
+ Connect your own accounts. Each sign-in is personal to you — your colleagues connect theirs. +
+ + {err &&
{err}
} + {note &&
{note}
} + + {/* popular for */} + {popular.length > 0 && ( +
+
+ Popular for + +
+
+ {popular.map((r) => ( +
+ + connect(r.slug)} onOpen={() => setSel(r.slug)} /> +
+ ))} +
+
+ )} + + {/* filter */} +
+ {([["all", "All"], ["connected", "Connected"], ["not", "Not connected"]] as [Filter, string][]).map(([k, label]) => ( + + ))} +
+ + {/* table */} +
+
+
Connector
+
Type
+
Status
+
+ {visible.length === 0 ? ( + + ) : visible.map((r) => ( +
+ +
{r.type === "MCP" ? "MCP server" : "REST API"}
+
+ connect(r.slug)} onOpen={() => setSel(r.slug)} /> +
+
+ ))} +
+ +
+ Connecting creates an auth provider, a tool set, and one tool per action — so its actions show up in{" "} + Tools and can be dropped straight onto a workflow canvas or granted to an agent. The + tools are the project's; the account behind them is yours. + {unavailable > 0 && ( + <> {unavailable} connector{unavailable === 1 ? " isn't" : "s aren't"} set up on this deployment yet — + open one to see which app to register. + )} +
+
+ + {selected && ( + connect(selected.slug)} + onClose={() => setSel(null)} onChanged={reload} + onOpenToolSet={onOpenToolSet} + /> + )} + {customOpen && setCustomOpen(false)} onInstalled={() => { setCustomOpen(false); reload(); }} />} +
+ ); +} + +function MenuItem({ icon, title, sub, onClick }: { icon: string; title: string; sub: string; onClick: () => void }) { + return ( + + ); +} + +/** The right-hand cell. For anything the deployment can actually sign you into, this IS the + * flow - one click straight to the vendor, no intermediate screen. */ +function StatusAction({ row, busy, onConnect, onOpen }: { + row: Row; busy: boolean; onConnect: () => void; onOpen: () => void; +}) { + if (isConnected(row)) { + return ; + } + if (!isAvailable(row)) { + return ; + } + // A pasted connector with a key/token credential has nothing to sign in to - its detail panel + // is where you manage it. + if (row.install && row.auth?.kind !== "oauth2_authorization_code") { + return ; + } + return ( + + ); +} + +/* ---------------------------------------------------------------------------------------- */ + +function ConnectorDetail({ project, slug, row, busy, onConnect, onClose, onChanged, onOpenToolSet }: { + project: any; slug: string; row: Row; busy: boolean; onConnect: () => void; + onClose: () => void; onChanged: () => void; onOpenToolSet?: (setId: string) => void; +}) { + const [detail, setDetail] = useState(null); + const [values, setValues] = useState>({}); + const [work, setWork] = useState(null); + const [err, setErr] = useState(null); + const [note, setNote] = useState(null); + const install = row.install; + + useEffect(() => { + if (row.publisher === "custom") { setDetail(row); return; } + api.connectorDetail(project.id, slug).then(setDetail).catch(() => setDetail(row)); + }, [project.id, slug, row]); + + const spec = detail || row; + const managed = spec.managed !== false; + const available = isAvailable({ ...spec, install }); + const connected = isConnected(row); + const oauth = spec.auth?.kind === "oauth2_authorization_code"; + + const run = async (label: string, fn: () => Promise) => { + setWork(label); setErr(null); setNote(null); + try { await fn(); } catch (e: any) { setErr(e?.message || "Something went wrong"); } finally { setWork(null); } + }; + + return ( + +
+
+ +
+
{spec.summary}
+
+ {spec.type === "MCP" ? "MCP server" : "REST API"} + {connected + ? Connected as you + : install && Not connected} +
+
+
+ + {install?.status_detail &&
{install.status_detail}
} + {err &&
{err}
} + {note &&
{note}
} + + {/* what it can do */} + {spec.actions && spec.actions.length > 0 && ( +
+ {spec.actions.length} actions +
+ {spec.actions.map((a) => ( +
+ {a.method} + + {a.name} + {a.description} + +
+ ))} +
+
+ )} + {spec.type === "MCP" && !install && ( +
+ Actions are discovered from the vendor's MCP server the first time you connect, so the + list stays current as they ship new ones. +
+ )} + {spec.auth?.scopes?.length > 0 && available && ( + +
{spec.auth.scopes.map((s) => {s.replace(/^https:\/\/www\.googleapis\.com\/auth\//, "")})}
+ + )} + + {!available ? ( + + ) : !install || (oauth && !connected) ? ( + <> + {managed && ( +
+ You'll sign in to {vendor(spec.name)} in a new window and approve access. The connection is + yours alone — nobody else in this project can act as you, and you can disconnect at any time. +
+ )} +
+ + {spec.docs_url && API docs } +
+ + ) : null} + + {install && ( + <> +
+ {install.tool_count} action{install.tool_count === 1 ? "" : "s"} + {install.auth_mode === "per_user" ? "Personal connection" : "Shared credential"} + {install.tool_set_id && onOpenToolSet && ( + + )} +
+
+ {connected && oauth && ( + + )} + {spec.type === "MCP" && ( + + )} + {connected && install.auth_kind !== "none" && ( + + )} + +
+ {/* Rotating a credential only applies to a pasted connector - a catalog one's app + lives in the deployment's environment, where its owner rotates it. */} + {install.source !== "catalog" && (spec.auth?.setup || []).some((f) => f.secret) && ( +
+ Update credentials +
+ {(spec.auth?.setup || []).filter((f) => f.secret).map((f) => ( + + setValues((v) => ({ ...v, [f.key]: e.target.value }))} /> + + ))} + +
+
+ )} + + )} +
+ + ); +} + +/** The deployment hasn't registered this vendor. That is an operator's job, in the environment - + * so say precisely what to register and where it goes, rather than showing an end user a form + * for a client secret that shouldn't be theirs to hold. */ +function NotConfigured({ spec }: { spec: ConnectorCatalogEntry }) { + const keys = spec.missing_keys || []; + const envLine = `${spec.config_env_key || "FORGE_CONNECTOR_OAUTH_APPS"}={"${spec.credential_group}":{${keys.map((k) => `"${k}":"…"`).join(",")}}}`; + return ( +
+
Not set up on this Forge deployment yet
+
+ Whoever runs Forge registers one {vendor(spec.name)} OAuth app — with the redirect URI + below — and adds it to the environment. After a restart, everyone here connects their own + {" "}{vendor(spec.name)} account with a single click; nobody types a secret into Forge. +
+ + e.currentTarget.select()} /> + + +
+ {spec.setup_url && Register an app } + {spec.docs_url && API docs } +
+ {spec.auth?.setup_help &&
{spec.auth.setup_help}
} +
+ ); +} + +/** OAuth apps need the exact callback whitelisted; showing it removes the usual first failure. + * The value comes from the SERVER, because the redirect_uri the flow actually sends is built + * from the API's public base URL - not the console's origin the browser happens to be on. A + * guessed URL looks plausible and then fails with redirect_uri_mismatch. */ +function RedirectHint({ url }: { url?: string }) { + if (!url) return null; + return ( + +
+ + +
+ + ); +} + +/* ---------------------------------------------------------------------------------------- */ + +function CustomConnectorModal({ project, onClose, onInstalled }: { project: any; onClose: () => void; onInstalled: () => void }) { + const [text, setText] = useState(""); + const [checked, setChecked] = useState<{ ok: boolean; error?: string; connector?: ConnectorCatalogEntry; hosts?: string[] } | null>(null); + const [values, setValues] = useState>({}); + const [perUser, setPerUser] = useState(false); + const [examples, setExamples] = useState([]); + const [busy, setBusy] = useState(false); + const [err, setErr] = useState(null); + + // Bundled starting points for the services that can't be one-click (an API key or a bot token + // has to come from a person). Best-effort: a viewer without editor rights just doesn't see them. + useEffect(() => { api.connectorExamples(project.id).then(setExamples).catch(() => setExamples([])); }, [project.id]); + + const parsed = useMemo(() => { try { return JSON.parse(text); } catch { return null; } }, [text]); + + const validate = async () => { + setErr(null); + if (!parsed) { setChecked({ ok: false, error: "That isn't valid JSON." }); return; } + try { setChecked(await api.validateConnectorManifest(project.id, parsed)); } + catch (e: any) { setErr(e?.message || "Could not validate"); } + }; + + const install = async () => { + setBusy(true); setErr(null); + try { + await api.installCustomConnector(project.id, { manifest: parsed, values, auth_mode: perUser ? "per_user" : "shared" }); + onInstalled(); + } catch (e: any) { setErr(e?.message || "Install failed"); } finally { setBusy(false); } + }; + + const setup = checked?.connector?.auth?.setup || []; + + return ( + +
+
+ Paste a forge.connector/1 manifest. It installs exactly like a catalog + entry — same validation, same rows — so a private connector pack can live in your own repo instead of a + fork of Forge. This is also where a credential you type yourself belongs: an API key, a bot token, or one + shared service account for the whole project. +
+ {examples.length > 0 && ( + + + + )} + +