diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 5cde97e1..43da2cd8 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: For anything that isn't a clear bug or feature request β€” please use Discussions so the answer helps everyone else too. - name: πŸ“š Documentation & Guides url: https://anythingmcp.com - about: Setup guides for all 188 adapters and every AI client, in seven languages. + about: Setup guides for all 189 adapters and every AI client, in seven languages. - name: πŸ” Security vulnerabilities url: https://github.com/HelpCode-ai/anythingmcp/blob/main/SECURITY.md about: Please do NOT open a public issue. Follow the security disclosure policy. diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 00000000..d2a5ca9e --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,9 @@ +name: AnythingMCP CodeQL config + +paths-ignore: + # Calls one real tool of every keyless adapter, with the URL taken from the + # adapter JSON in this repo. CodeQL reads that as "file data flows into an + # outbound request" (js/file-access-to-http), which is precisely the job of + # the script: the file is our own catalog, not user input, and the script + # runs only in CI and from an operator's shell. + - scripts/probe-keyless.mjs diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f90bd27e..697ebc26 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,6 +31,7 @@ jobs: with: languages: ${{ matrix.language }} queries: security-and-quality + config-file: ./.github/codeql/codeql-config.yml - name: Autobuild uses: github/codeql-action/autobuild@v4 diff --git a/.github/workflows/deploy-cloud.yml b/.github/workflows/deploy-cloud.yml index beea0722..21849cfb 100644 --- a/.github/workflows/deploy-cloud.yml +++ b/.github/workflows/deploy-cloud.yml @@ -23,7 +23,7 @@ jobs: host: ${{ secrets.CLOUD_HOST }} username: root key: ${{ secrets.CLOUD_SSH_KEY }} - source: "docker-compose.cloud.yml,deploy/cloud/Caddyfile,deploy/cloud/db-rest/Dockerfile" + source: "docker-compose.cloud.yml,deploy/cloud/Caddyfile,deploy/motis/Dockerfile,deploy/motis/config.yml,deploy/motis/entrypoint.sh" target: /tmp/anythingmcp-deploy overwrite: true @@ -46,14 +46,16 @@ jobs: docker pull helpcodeai/anythingmcp:${{ inputs.tag }} docker pull helpcodeai/anythingmcp:latest - # Update compose, Caddyfile and the custom db-rest Dockerfile from - # checkout. The Dockerfile is the build context for the db-rest + # Update compose, Caddyfile and the MOTIS build context from + # checkout. deploy/motis is the build context for the motis # service; `docker compose up` builds it if the pinned image tag is - # missing (fresh droplet), otherwise the existing image is reused. + # missing (fresh droplet or a bumped tag), otherwise the existing + # image is reused. cp /tmp/anythingmcp-deploy/docker-compose.cloud.yml ./docker-compose.cloud.yml cp /tmp/anythingmcp-deploy/deploy/cloud/Caddyfile ./Caddyfile - mkdir -p ./deploy/cloud/db-rest - cp /tmp/anythingmcp-deploy/deploy/cloud/db-rest/Dockerfile ./deploy/cloud/db-rest/Dockerfile + mkdir -p ./deploy/motis + cp /tmp/anythingmcp-deploy/deploy/motis/* ./deploy/motis/ + rm -rf ./deploy/cloud/db-rest rm -rf /tmp/anythingmcp-deploy # Keep CRON_SECRET in the server .env in sync with the repo @@ -67,7 +69,7 @@ jobs: fi # Recreate ONLY the app to pick up the new image/config (postgres, - # redis and db-rest keep running β€” no data churn, no needless + # redis and motis keep running β€” no data churn, no needless # downtime), and wait until it reports healthy. The old command # force-recreated the whole stack with no health gate, so a # crash-on-start went unnoticed and could take the DB down with it. diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9704f092..2a7b49ce 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,7 +65,7 @@ jobs: readme-filepath: ./README.md # Docker Hub caps this at 100 characters and shows it in search # results; the repo's was left at the truncated default. - short-description: "188 connectors + any REST/SOAP/GraphQL/SQL as MCP tools for Claude & ChatGPT. Self-hosted, AGPL." + short-description: "189 connectors + any REST/SOAP/GraphQL/SQL as MCP tools for Claude & ChatGPT. Self-hosted, AGPL." - name: Set Docker Hub categories env: diff --git a/.github/workflows/keyless-probe.yml b/.github/workflows/keyless-probe.yml new file mode 100644 index 00000000..2964db68 --- /dev/null +++ b/.github/workflows/keyless-probe.yml @@ -0,0 +1,56 @@ +name: Keyless adapter probe + +# Calls one real tool of every adapter that claims to need no API key, from a +# GitHub runner β€” a datacenter address, like the cloud. The README and the +# banner quote that number; this is what keeps it true. An adapter that only +# works from residential IPs must be marked `selfHostOnly` (hidden from the +# cloud catalog, excluded from the count) rather than advertised. +# +# The Deutsche Bahn adapter needs a MOTIS instance, so the job builds the one +# in deploy/motis and probes through it β€” which also proves that image still +# boots, downloads the feeds and imports them. + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + pull_request: + paths: + - "deploy/motis/**" + - "scripts/probe-keyless.mjs" + - "packages/backend/src/adapters/de/deutsche-bahn.json" + +jobs: + probe: + name: Probe keyless adapters from a datacenter IP + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Build and start MOTIS + run: | + docker build -t anythingmcp-motis:ci deploy/motis + docker run -d --name motis -p 8080:8080 anythingmcp-motis:ci + for i in $(seq 1 60); do + if curl -fsS "http://localhost:8080/api/v1/geocode?text=Berlin&type=STOP" > /dev/null 2>&1; then + echo "MOTIS up after ${i} checks"; break + fi + if [ "$i" = "60" ]; then + echo "::error::MOTIS did not come up"; docker logs motis; exit 1 + fi + sleep 5 + done + + - name: Probe + env: + PROBE_MOTIS_URL: http://localhost:8080 + run: node scripts/probe-keyless.mjs --check --all + + - name: MOTIS logs + if: always() + run: docker logs motis 2>&1 | grep -v "\[debug\]" | tail -40 diff --git a/CITATION.cff b/CITATION.cff index 5f7c01f7..e7a223b3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,7 +13,7 @@ abstract: > server that exposes REST, SOAP/WSDL and GraphQL APIs, SQL/NoSQL databases and other MCP servers as tools for AI clients such as Claude, ChatGPT, Google Gemini, GitHub Copilot and Cursor, without writing code. It ships - 188 pre-built adapters (Deutsche Bahn, DATEV, weclapp, DHL, Shopware, + 189 pre-built adapters (Deutsche Bahn, DATEV, weclapp, DHL, Shopware, Personio, Handelsregister, etc.), a per-workspace knowledge graph served over MCP, per-tool response mapping, OAuth2/RBAC/SSO/SCIM governance, an audit log and a visual tool editor. diff --git a/README.md b/README.md index da503bbb..2a6b415a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@

- AnythingMCP β€” 188 connectors, 26 of them with no API key. Your REST, SOAP/WSDL, GraphQL, SQL and MCP systems become tools for Claude, ChatGPT, Copilot and Gemini. + AnythingMCP β€” 189 connectors, 20 of them with no API key. Your REST, SOAP/WSDL, GraphQL, SQL and MCP systems become tools for Claude, ChatGPT, Copilot and Gemini.

AnythingMCP

Give Claude, ChatGPT and Copilot safe access to the software your company already runs.
- 188 ready adapters, any REST/SOAP/GraphQL/SQL system without code, on your own infrastructure β€” and it learns how your systems connect. + 189 ready adapters, any REST/SOAP/GraphQL/SQL system without code, on your own infrastructure β€” and it learns how your systems connect.

@@ -36,7 +36,7 @@ docker compose up -d # β†’ http://localhost:3000 Three words appear throughout and mean three different things: -- an **adapter** is one of the 188 JSON definitions that ship in this repo β€” DATEV, weclapp, DHL, Deutsche Bahn, Shopware, Personio, Handelsregister and the rest. 26 of them need no API key at all; the others ask for your credentials at import. +- an **adapter** is one of the 189 JSON definitions that ship in this repo β€” DATEV, weclapp, DHL, Deutsche Bahn, Shopware, Personio, Handelsregister and the rest. 20 of them need no API key at all; the others ask for your credentials at import. - a **connector** is an adapter, or your own OpenAPI spec / Postman collection / WSDL / GraphQL endpoint / database, once you have configured it in your workspace. Anything you can point at, in minutes, without writing an MCP server. - an **MCP server** is the URL you hand to Claude. It exposes the connectors you assign to it, and nothing else. @@ -260,7 +260,7 @@ AI clients speak MCP, but your systems speak REST, SOAP, GraphQL and SQL. Writin ## The adapter catalog -188 adapters, exposing 1,800+ tools. **26 need no API key**; the rest ask for your credentials at import and the tools are available immediately. Every one has a setup guide on [anythingmcp.com/guides](https://anythingmcp.com/guides), in seven languages. +189 adapters, exposing 1,800+ tools. **20 need no API key**; the rest ask for your credentials at import and the tools are available immediately. Every one has a setup guide on [anythingmcp.com/guides](https://anythingmcp.com/guides), in seven languages. | Category | Examples | |---|---| diff --git a/deploy/cloud/db-rest/Dockerfile b/deploy/cloud/db-rest/Dockerfile deleted file mode 100644 index 3c0b6a70..00000000 --- a/deploy/cloud/db-rest/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Custom db-rest image for AnythingMCP Cloud. -# -# Two reasons this exists instead of using derhuerst/db-rest:6 directly: -# -# 1. Freshness β€” bump db-vendo-client to the latest patch so we track -# Deutsche Bahn's frequently-changing upstream endpoints/hosts. -# 2. Egress proxy β€” db-vendo-client already honours HTTPS_PROXY (it builds an -# https-proxy-agent from it). The cloud sets HTTPS_PROXY=$CONNECTOR_PROXY_URL -# on this container so all upstream DB calls exit through the Zyte -# web-unblocker, defeating Deutsche Bahn's Akamai block of datacenter IPs -# (the `int.bahn.de` / `app.services-bahn.de` 403 / OPS_BLOCKED responses). -# -# No source changes β€” just a dependency bump on top of the upstream image. -FROM docker.io/derhuerst/db-rest:6 -USER root -WORKDIR /app -RUN npm install db-vendo-client@6.10.12 --no-audit --no-fund diff --git a/deploy/motis/Dockerfile b/deploy/motis/Dockerfile new file mode 100644 index 00000000..77d67801 --- /dev/null +++ b/deploy/motis/Dockerfile @@ -0,0 +1,32 @@ +# MOTIS routing engine for the Deutsche Bahn connector. +# +# The connector used to go through db-rest / db-vendo-client, which scrapes +# bahn.de. Deutsche Bahn blocks datacenter IPs and the library's own README now +# calls those endpoints "very unreliable" and recommends a self-hosted MOTIS. +# This image is that: the official MOTIS binary plus an entrypoint that pulls +# the open GTFS timetable for Germany's trains from gtfs.de (CC BY 4.0), runs +# the import, keeps a GTFS-RT feed polled for live delays and cancellations, +# and re-imports the static feed once a week so it never runs off the end of +# the 30-day window the free feeds cover. +# +# Trains only (long-distance + regional/S-Bahn, ~12 MB of GTFS): the import +# takes about a second and peaks under 400 MB. The full Germany feed with every +# bus and tram is 280 MB and peaks at 7 GB on import, which would not fit next +# to the app on an 8 GB host. See README.md for the feed URLs and how to swap +# them. +FROM ghcr.io/motis-project/motis:2.11.3 + +USER root +COPY config.yml /motis-config/config.yml +COPY entrypoint.sh /entrypoint.sh +RUN chmod 0755 /entrypoint.sh && mkdir -p /data && chown motis:motis /data + +USER motis +WORKDIR /data +EXPOSE 8080 +# The first boot downloads and imports the feeds before the server listens; +# the compose healthchecks give that a longer start_period. +HEALTHCHECK --interval=30s --timeout=5s --start-period=180s --retries=3 \ + CMD wget --quiet --tries=1 --spider http://localhost:8080/ || exit 1 +ENTRYPOINT ["/entrypoint.sh"] +CMD [] diff --git a/deploy/motis/README.md b/deploy/motis/README.md new file mode 100644 index 00000000..87a908b0 --- /dev/null +++ b/deploy/motis/README.md @@ -0,0 +1,75 @@ +# MOTIS for the Deutsche Bahn connector + +The `deutsche-bahn` adapter talks to a [MOTIS](https://github.com/motis-project/motis) +instance (MIT) instead of scraping bahn.de. This directory builds one that is +ready to run: the official MOTIS binary plus an entrypoint that downloads the +open German train timetable, imports it, polls a GTFS-RT feed for live delays +and cancellations, and re-imports the static feed once a week. + +## Why not db-rest / bahn.de + +Until v0.8.1 the connector went through `db-rest` (db-vendo-client), which +calls Deutsche Bahn's undocumented web and app endpoints. Deutsche Bahn blocks +datacenter IP ranges, so from a hosted server every call failed (HTTP 500 / +`OPS_BLOCKED`), and the library's own README now describes those endpoints +as very unreliable and recommends a self-hosted MOTIS. Open data does not get +blocked. + +## What it serves + +| piece | source | licence | +|---|---|---| +| Long-distance trains (ICE, IC, EC, ECE, EN, railjet, night trains) | [gtfs.de `de_fv`](https://gtfs.de/en/feeds/de_fv/) | CC BY 4.0 | +| Regional trains and S-Bahn | [gtfs.de `de_rv`](https://gtfs.de/en/feeds/de_rv/) | CC BY 4.0 | +| Live delays, platform changes, cancellations, service alerts | [gtfs.de GTFS-RT](https://gtfs.de/en/realtime/) `realtime-free.pb` | CC BY-SA 4.0 | + +The free static feeds cover the next 30 days and are re-downloaded every +`MOTIS_REFRESH_DAYS` (default 7). Buses, trams and ferries are deliberately +left out: adding them (`de_full`, 280 MB) takes the import from one second and +under 400 MB of RAM to about 7 GB. + +Attribution: gtfs.de asks that data users name **DELFI e.V.** as the source; +the connector's instructions and the adapter's `docsUrl` do. + +## Running it + +Cloud (`docker-compose.cloud.yml`) runs it as the `motis` service and points +the app at it with `MOTIS_INTERNAL_URL=http://motis:8080`. Self-host: + +```bash +# in .env +COMPOSE_PROFILES=motis +MOTIS_INTERNAL_URL=http://motis:8080 +SSRF_ALLOWED_HOSTS=motis +docker compose up -d +``` + +When `MOTIS_INTERNAL_URL` is set the adapter's `MOTIS_URL` is filled in +automatically at import and hidden from the install form. Without it, the +install form asks for the URL of a MOTIS instance you run elsewhere. + +`http://localhost:8080/` serves the MOTIS UI once the import is done; the +connector uses `/api/v1/geocode`, `/api/v1/stoptimes` and `/api/v1/plan`. + +## Environment + +| variable | default | meaning | +|---|---|---| +| `MOTIS_REFRESH_DAYS` | `7` | re-download and re-import the static feeds after this many days | +| `MOTIS_REFRESH_CHECK_SECONDS` | `3600` | how often the running container checks whether a refresh is due | +| `MOTIS_GTFS_FV_URL` | gtfs.de `fv_free/latest.zip` | long-distance feed | +| `MOTIS_GTFS_RV_URL` | gtfs.de `rv_free/latest.zip` | regional feed | +| `MOTIS_GTFS_RT_URL` | gtfs.de `realtime-free.pb` | GTFS-RT feed polled every 120 s | +| `MOTIS_DATA_DIR` | `/data` | volume with feeds, imported data and the refresh stamp | + +gtfs.de also sells complete feeds with extended route types (so an ICE reports +`HIGHSPEED_RAIL` rather than `REGIONAL_RAIL`) and no 30-day limit; point the +URL variables at those and nothing else changes. + +## Not Transitous + +[Transitous](https://transitous.org) runs the same stack as a public service +and is a good way to try the API, but its policy forbids commercial use and +asks for a User-Agent naming the app and a contact. The adapter therefore +never defaults to it; if you qualify, entering `https://api.transitous.org` as +`MOTIS_URL` works. diff --git a/deploy/motis/config.yml b/deploy/motis/config.yml new file mode 100644 index 00000000..50bbcc0d --- /dev/null +++ b/deploy/motis/config.yml @@ -0,0 +1,42 @@ +# MOTIS import/server configuration, templated by entrypoint.sh: +# __FEEDS__ directory holding the downloaded GTFS zips +# __RT_URL__ GTFS-RT feed (TripUpdates + ServiceAlerts) +# +# Two datasets rather than gtfs.de's single "full" feed: fv (long-distance: +# ICE, IC, EC, ECE, EN, railjet, night trains) and rv (regional rail and +# S-Bahn). Together they are every train in Germany and about 12 MB; the full +# feed adds every bus and tram and needs 7 GB of RAM to import. +# +# No OpenStreetMap file on purpose. MOTIS only needs OSM for street routing, +# map tiles and address geocoding; station-name geocoding works from the +# timetable alone (adr_extend), and that is all the connector asks for. +timetable: + first_day: TODAY + # The free feeds cover the next 30 days; anything beyond that is simply + # absent, so a large window costs nothing and never truncates a feed. + num_days: 365 + railviz: false + with_shapes: false + # The GTFS-RT feed is ~55 MB (it carries every bus in the country); the + # default 30 s download timeout truncated it and MOTIS logged parser errors. + http_timeout: 180 + # Seconds between GTFS-RT polls. The upstream file updates every 10 s, but + # each poll is a 55 MB download, so this is a bandwidth courtesy to gtfs.de + # more than a freshness choice. + update_interval: 120 + datasets: + fv: + path: __FEEDS__/fv.zip + rt: + - url: __RT_URL__ + protocol: gtfsrt + rv: + path: __FEEDS__/rv.zip + rt: + - url: __RT_URL__ + protocol: gtfsrt +geocoding: true +reverse_geocoding: false +street_routing: false +osr_footpath: false +elevators: false diff --git a/deploy/motis/entrypoint.sh b/deploy/motis/entrypoint.sh new file mode 100755 index 00000000..9c619815 --- /dev/null +++ b/deploy/motis/entrypoint.sh @@ -0,0 +1,112 @@ +#!/bin/sh +# Boot MOTIS on the latest gtfs.de train timetable and keep it fresh. +# +# Layout under $MOTIS_DATA_DIR (a volume): +# feeds/ the GTFS zips the current data was built from +# current/ the imported data the server is running on +# staging/ a freshly imported data set waiting to be swapped in +# .imported epoch of the last successful import +# +# The static feeds are re-downloaded and re-imported every $MOTIS_REFRESH_DAYS +# days. The import runs while the old server keeps answering; only the swap +# restarts it (a few seconds, and docker's healthcheck covers it). A failed +# refresh leaves the running data untouched and is retried on the next check. +set -eu + +DATA="${MOTIS_DATA_DIR:-/data}" +FEEDS="$DATA/feeds" +CURRENT="$DATA/current" +STAGING="$DATA/staging" +STAMP="$DATA/.imported" +REFRESH_DAYS="${MOTIS_REFRESH_DAYS:-7}" +CHECK_INTERVAL="${MOTIS_REFRESH_CHECK_SECONDS:-3600}" +FV_URL="${MOTIS_GTFS_FV_URL:-https://download.gtfs.de/germany/fv_free/latest.zip}" +RV_URL="${MOTIS_GTFS_RV_URL:-https://download.gtfs.de/germany/rv_free/latest.zip}" +RT_URL="${MOTIS_GTFS_RT_URL:-https://realtime.gtfs.de/realtime-free.pb}" +UA="anythingmcp-motis/1.0 (+https://anythingmcp.com)" + +log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [entrypoint] $*"; } + +needs_refresh() { + [ -f "$CURRENT/tt.bin" ] || return 0 + [ -f "$STAMP" ] || return 0 + now=$(date +%s) + last=$(cat "$STAMP" 2>/dev/null || echo 0) + [ $((now - last)) -ge $((REFRESH_DAYS * 86400)) ] +} + +# Download both feeds and import them into $STAGING. Never touches $CURRENT. +build_staging() { + tmp="$DATA/feeds.new" + rm -rf "$tmp" "$STAGING" + mkdir -p "$tmp" + log "downloading $FV_URL" + wget -q -U "$UA" -O "$tmp/fv.zip" "$FV_URL" + log "downloading $RV_URL" + wget -q -U "$UA" -O "$tmp/rv.zip" "$RV_URL" + # A truncated or HTML "download" would fail the import; check the zip magic + # first so the log names the real problem. + for f in fv rv; do + if [ "$(head -c 2 "$tmp/$f.zip")" != "PK" ]; then + log "$f.zip is not a zip file (got: $(head -c 40 "$tmp/$f.zip"))" + return 1 + fi + done + sed -e "s#__FEEDS__#$tmp#g" -e "s#__RT_URL__#$RT_URL#g" \ + /motis-config/config.yml > "$DATA/config.import.yml" + log "importing" + /motis import -c "$DATA/config.import.yml" -d "$STAGING" + rm -rf "$FEEDS" + mv "$tmp" "$FEEDS" + log "import finished" +} + +swap_in_staging() { + rm -rf "$DATA/previous" + [ -d "$CURRENT" ] && mv "$CURRENT" "$DATA/previous" + mv "$STAGING" "$CURRENT" + rm -rf "$DATA/previous" + date +%s > "$STAMP" +} + +# First boot, or the data on the volume is stale: build before serving. +if needs_refresh; then + if build_staging; then + swap_in_staging + elif [ -f "$CURRENT/tt.bin" ]; then + log "refresh failed; serving the previous data set" + else + log "no data set to serve" + exit 1 + fi +fi + +while :; do + log "starting server on $CURRENT" + /motis server -d "$CURRENT" & + pid=$! + swap=0 + while kill -0 "$pid" 2>/dev/null; do + sleep "$CHECK_INTERVAL" & + wait $! || true + kill -0 "$pid" 2>/dev/null || break + if needs_refresh; then + if build_staging; then + swap=1 + log "restarting server on the new data set" + kill "$pid" + wait "$pid" || true + swap_in_staging + break + else + # Retried at the next check (hourly by default); the current data + # set keeps serving meanwhile. + log "refresh failed; keeping the current data set" + fi + fi + done + if [ "$swap" -eq 0 ]; then + # The server died on its own; surface that so docker restarts the container. + wait "$pid" && exit 0 || exit $? + fi +done diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index 224e4a35..50eefcf1 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -41,7 +41,7 @@ services: # defaults to 2048, which suits a 4 GB host; the tool registry outgrew # that and crash-looped the instance on 10 Sep. Raise it in the server # .env when the host has the RAM to back it β€” roughly half of total is a - # safe ceiling, since postgres, redis, db-rest and caddy share the box. + # safe ceiling, since postgres, redis, motis and caddy share the box. - NODE_MAX_OLD_SPACE_MB=${NODE_MAX_OLD_SPACE_MB:-2048} - DEPLOYMENT_MODE=cloud - NEXT_PUBLIC_API_URL=https://${DOMAIN} @@ -75,14 +75,15 @@ services: # use_proxy=true route through this; unset = feature off everywhere. - CONNECTOR_PROXY_URL=${CONNECTOR_PROXY_URL:-} - PROXY_RATE_LIMIT_DEFAULT=${PROXY_RATE_LIMIT_DEFAULT:-100} - # Internal self-hosted db-rest. When set (cloud), the Deutsche Bahn - # connector's public base URL (v6.db.transport.rest) is transparently - # routed here. Unset on self-host β†’ connector uses the public API. - - DB_REST_INTERNAL_URL=${DB_REST_INTERNAL_URL:-http://db-rest:3000} - # Allow the SSRF guard to reach the internal db-rest host (it resolves to + # Internal MOTIS instance for the Deutsche Bahn connector. When set, the + # adapter's MOTIS_URL is filled in automatically at import (and hidden + # from the install form), so cloud users never see or choose it. + # Self-hosters leave it unset and enter their own MOTIS URL. + - MOTIS_INTERNAL_URL=${MOTIS_INTERNAL_URL:-http://motis:8080} + # Allow the SSRF guard to reach the internal MOTIS host (it resolves to # a private docker IP, which the guard blocks by default). Harmless on # self-host (no such host on their network). - - SSRF_ALLOWED_HOSTS=${SSRF_ALLOWED_HOSTS:-db-rest} + - SSRF_ALLOWED_HOSTS=${SSRF_ALLOWED_HOSTS:-motis} # Knowledge Graph β€” AI (LLM) features. Off unless KG_LLM_ENABLED=true AND a # provider key are set; the per-workspace toggle still gates actual usage. - KG_LLM_ENABLED=${KG_LLM_ENABLED:-false} @@ -154,49 +155,40 @@ services: retries: 5 restart: unless-stopped - # db-rest β€” self-hosted Deutsche Bahn REST wrapper (derhuerst/db-rest). + # MOTIS β€” routing engine behind the Deutsche Bahn connector, fed by the open + # gtfs.de timetable (CC BY 4.0) plus its GTFS-RT feed for live delays. + # Replaces db-rest, which scraped bahn.de and died when Deutsche Bahn began + # blocking datacenter IPs (see deploy/motis/README.md). # INTERNAL ONLY: deliberately NO `ports:` mapping β†’ reachable solely as - # http://db-rest:3000 from the app over the amcp-cloud_default network, never - # from the internet. The deutsche-bahn connector ships pointing at the public - # v6.db.transport.rest; in cloud the app rewrites the host to this service via - # DB_REST_INTERNAL_URL. Do NOT add a `ports:` entry or a Caddy route. - db-rest: - # Custom image: derhuerst/db-rest:6 + bumped db-vendo-client (see - # deploy/cloud/db-rest/Dockerfile). The tag is version-pinned so the image - # persists across deploys and is only rebuilt when the Dockerfile changes. + # http://motis:8080 from the app over the amcp-cloud_default network, never + # from the internet. Do NOT add a `ports:` entry or a Caddy route. + motis: + # Custom image: official MOTIS + an entrypoint that downloads and imports + # the feeds and refreshes them weekly (deploy/motis). The tag is pinned so + # the image persists across deploys and is only rebuilt when it changes. build: - context: ./deploy/cloud/db-rest - image: anythingmcp-db-rest:6.10.12 - container_name: amcp-cloud-db-rest + context: ./deploy/motis + image: anythingmcp-motis:2.11.3-1 + container_name: amcp-cloud-motis expose: - - "3000" + - "8080" + volumes: + - motis_data:/data environment: - - PORT=3000 - # Reuse the shared Redis for db-rest's response cache (separate DB index). - - REDIS_URL=redis://redis:6379/1 - # Route all upstream Deutsche Bahn calls through the Zyte web-unblocker. - # DB (Akamai) blocks datacenter IPs β€” dbnav/dbweb endpoints return - # OPS_BLOCKED / 403 Forbidden β€” so db-vendo-client (which reads - # HTTPS_PROXY) must egress via a residential/unblocker IP. Unset on - # self-host (CONNECTOR_PROXY_URL empty) β†’ direct calls, as before. - - HTTPS_PROXY=${CONNECTOR_PROXY_URL:-} - # Zyte proxy-mode MITMs TLS, so its cert won't validate against the DB - # host. Only skip verification when a proxy is actually set. This - # container's only external egress is DB (via Zyte) + internal Redis. - - NODE_TLS_REJECT_UNAUTHORIZED=${CONNECTOR_PROXY_URL:+0} - depends_on: - redis: - condition: service_healthy + - MOTIS_REFRESH_DAYS=${MOTIS_REFRESH_DAYS:-7} healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"] + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/"] interval: 30s timeout: 5s retries: 3 - start_period: 20s + # First boot downloads ~12 MB of GTFS and imports it (seconds, not + # minutes); the margin is for a slow gtfs.de. + start_period: 180s restart: unless-stopped volumes: postgres_data: redis_data: + motis_data: caddy_data: caddy_config: diff --git a/docker-compose.yml b/docker-compose.yml index 7e2e36f4..c44870bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,15 @@ services: - ALLOW_OPEN_REGISTRATION=${ALLOW_OPEN_REGISTRATION:-false} # Uncomment to enable Redis (optional β€” used for response caching and rate limiting) # - REDIS_URL=redis://redis:6379 + # Deutsche Bahn connector: URL of a MOTIS instance. Set together with + # COMPOSE_PROFILES=motis to run the bundled one (see deploy/motis); the + # adapter then fills its MOTIS_URL in automatically at import. Leave + # unset to be asked for a URL when installing the adapter. + - MOTIS_INTERNAL_URL=${MOTIS_INTERNAL_URL:-} + # Hosts the SSRF guard may reach even though they resolve to a private + # address. Needed for the bundled MOTIS (`motis`) and any other API you + # run on the same Docker network. + - SSRF_ALLOWED_HOSTS=${SSRF_ALLOWED_HOSTS:-} depends_on: postgres: @@ -101,6 +110,28 @@ services: retries: 5 restart: unless-stopped + # MOTIS routing engine for the Deutsche Bahn connector (optional). + # Activated by COMPOSE_PROFILES=motis in .env, together with + # MOTIS_INTERNAL_URL=http://motis:8080 and SSRF_ALLOWED_HOSTS=motis. Pulls + # the open gtfs.de train timetable on first boot and refreshes it weekly; + # about 12 MB of GTFS, a one-second import, ~100 MB of RAM at rest. + # Details and feed licences: deploy/motis/README.md + motis: + build: + context: ./deploy/motis + image: anythingmcp-motis:2.11.3-1 + profiles: ["motis"] + container_name: ${COMPOSE_PROJECT_NAME:-amcp}-motis + volumes: + - motis_data:/data + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 180s + restart: unless-stopped + # Redis Cache (optional β€” uncomment to enable response caching and rate limiting) # redis: # image: redis:7-alpine @@ -116,6 +147,7 @@ services: volumes: postgres_data: + motis_data: # Imported timetable for the optional MOTIS service caddy_data: # Let's Encrypt certificates (used when Caddy proxy is enabled) caddy_config: # Caddy configuration cache # redis_data: # Uncomment if Redis is enabled diff --git a/docs/assets/banner.png b/docs/assets/banner.png index 528e1aaa..8609f4c7 100644 Binary files a/docs/assets/banner.png and b/docs/assets/banner.png differ diff --git a/docs/assets/social-preview.png b/docs/assets/social-preview.png index d6a195ab..5332da72 100644 Binary files a/docs/assets/social-preview.png and b/docs/assets/social-preview.png differ diff --git a/glama.json b/glama.json index 95065780..c01899d9 100644 --- a/glama.json +++ b/glama.json @@ -3,5 +3,5 @@ "maintainers": [ "keysersoft" ], - "description": "Give Claude, ChatGPT and Copilot safe access to the software your company already runs. 188 pre-built adapters (Deutsche Bahn, DATEV, weclapp, DHL, Shopware, Personio\u2026) plus any REST, SOAP/WSDL, GraphQL or SQL system as MCP tools, no code. Self-hosted, knowledge graph, per-tool response mapping, OAuth2/RBAC/SSO/audit. Open source under AGPL-3.0." + "description": "Give Claude, ChatGPT and Copilot safe access to the software your company already runs. 189 pre-built adapters (Deutsche Bahn, DATEV, weclapp, DHL, Shopware, Personio\u2026) plus any REST, SOAP/WSDL, GraphQL or SQL system as MCP tools, no code. Self-hosted, knowledge graph, per-tool response mapping, OAuth2/RBAC/SSO/audit. Open source under AGPL-3.0." } diff --git a/package.json b/package.json index c875eeda..2cc25fef 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "anythingmcp", - "version": "0.8.1", - "description": "Give Claude, ChatGPT and Copilot safe access to the software your company already runs. 188 pre-built adapters plus any REST, SOAP/WSDL, GraphQL or SQL system as MCP tools, no code. Self-hosted, open source (AGPL-3.0).", + "version": "0.9.0", + "description": "Give Claude, ChatGPT and Copilot safe access to the software your company already runs. 189 pre-built adapters plus any REST, SOAP/WSDL, GraphQL or SQL system as MCP tools, no code. Self-hosted, open source (AGPL-3.0).", "private": true, "license": "AGPL-3.0-only", "engines": { diff --git a/packages/backend/package.json b/packages/backend/package.json index 34689656..3c4a4802 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "@anythingmcp/backend", - "version": "0.8.1", + "version": "0.9.0", "description": "AnythingMCP β€” NestJS Backend + Dynamic MCP Server", "private": true, "license": "AGPL-3.0-only", diff --git a/packages/backend/prisma/migrations/20260915140000_add_product_events/migration.sql b/packages/backend/prisma/migrations/20260915140000_add_product_events/migration.sql new file mode 100644 index 00000000..26729b98 --- /dev/null +++ b/packages/backend/prisma/migrations/20260915140000_add_product_events/migration.sql @@ -0,0 +1,30 @@ +-- ============================================================================= +-- Migration: Add product_events (UI usage events for the activation funnel) +-- ============================================================================= +-- Records what a user does on the pages that lead to the first MCP call: +-- copied the endpoint, opened a client tab, generated a key, left without +-- copying anything. `tool_invocations` cannot host these (tool_id NOT NULL) +-- and `security_events` is an audit trail, not a product log. +-- +-- FKs are ON DELETE SET NULL so deleting a user or organization keeps the +-- aggregate counts intact. +-- ============================================================================= + +CREATE TABLE "product_events" ( + "id" TEXT NOT NULL, + "event" TEXT NOT NULL, + "organization_id" TEXT, + "user_id" TEXT, + "metadata" JSONB, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "product_events_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "product_events_organization_id_created_at_idx" ON "product_events"("organization_id", "created_at"); +CREATE INDEX "product_events_event_created_at_idx" ON "product_events"("event", "created_at"); + +ALTER TABLE "product_events" ADD CONSTRAINT "product_events_organization_id_fkey" + FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "product_events" ADD CONSTRAINT "product_events_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/backend/prisma/schema.prisma b/packages/backend/prisma/schema.prisma index 7cd603e4..870d9db5 100644 --- a/packages/backend/prisma/schema.prisma +++ b/packages/backend/prisma/schema.prisma @@ -42,6 +42,7 @@ model Organization { kgValueSeen KgValueSeen[] kgSkills KgSkillSuggestion[] securityEvents SecurityEvent[] + productEvents ProductEvent[] userRoleAssignments UserRoleAssignment[] identityProviders IdentityProvider[] mcpConnectionGrants McpConnectionGrant[] @@ -158,6 +159,7 @@ model User { identities UserIdentity[] securityEventsAsActor SecurityEvent[] @relation("SecurityEventActor") securityEventsAsTarget SecurityEvent[] @relation("SecurityEventTarget") + productEvents ProductEvent[] memberships OrganizationMember[] connectors Connector[] @@ -1077,6 +1079,33 @@ model SecurityEvent { @@map("security_events") } +// Product-usage events from the UI, e.g. "copied the MCP endpoint", "opened +// the Cursor tab". Separate from security_events (an audit trail) and from +// tool_invocations (needs a tool). Exists because 74% of workspaces that +// attached a connector never sent a single MCP request and nothing recorded +// what they did on the page that tells them how β€” without this the next +// redesign of that page would be as blind as the last one. +model ProductEvent { + id String @id @default(cuid()) + + // Slug, e.g. 'mcp_url_copied'. Allow-listed in ProductEventService. + event String + + organizationId String? @map("organization_id") + organization Organization? @relation(fields: [organizationId], references: [id], onDelete: SetNull) + userId String? @map("user_id") + user User? @relation(fields: [userId], references: [id], onDelete: SetNull) + + // Small, non-sensitive context: which client tab, which server id. + metadata Json? + + createdAt DateTime @default(now()) @map("created_at") + + @@index([organizationId, createdAt]) + @@index([event, createdAt]) + @@map("product_events") +} + // ── OAuth2 Authorization Server ───────────────────────────────────────────── model OAuthClient { diff --git a/packages/backend/src/adapters/adapters.module.ts b/packages/backend/src/adapters/adapters.module.ts index 34cf09c3..ba06d085 100644 --- a/packages/backend/src/adapters/adapters.module.ts +++ b/packages/backend/src/adapters/adapters.module.ts @@ -7,9 +7,10 @@ import { AdaptersService } from './adapters.service'; import { McpServerModule } from '../mcp-server/mcp-server.module'; import { LicenseModule } from '../license/license.module'; import { McpServersModule } from '../mcp-servers/mcp-servers.module'; +import { ConnectorsModule } from '../connectors/connectors.module'; @Module({ - imports: [McpServerModule, LicenseModule, McpServersModule], + imports: [McpServerModule, LicenseModule, McpServersModule, ConnectorsModule], controllers: [AdaptersPublicController, AdaptersController], providers: [AdaptersService], }) diff --git a/packages/backend/src/adapters/adapters.service.ts b/packages/backend/src/adapters/adapters.service.ts index 2701dd03..e7d9e4f6 100644 --- a/packages/backend/src/adapters/adapters.service.ts +++ b/packages/backend/src/adapters/adapters.service.ts @@ -11,6 +11,14 @@ import { ConfigService } from '@nestjs/config'; import { listAdapters, getAdapter, AdapterMeta, AdapterDefinition } from './catalog'; import { hashInstructions } from './catalog-fingerprint'; import { getRequiredSecret } from '../common/secrets.util'; +import { + withOperatorProvided, + withoutOperatorProvided, +} from './cloud-managed-env'; +import { pickProbe } from './probe.util'; +import { ConnectorsService } from '../connectors/connectors.service'; +import { classifyToolExecutionError } from '../connectors/connector-error.util'; +import { applyResponseTransform } from '../connectors/response-transform.util'; @Injectable() export class AdaptersService { @@ -21,6 +29,7 @@ export class AdaptersService { private readonly prisma: PrismaService, private readonly mcpServer: McpServerService, private readonly configService: ConfigService, + private readonly connectors: ConnectorsService, ) { this.encryptionKey = getRequiredSecret( 'ENCRYPTION_KEY', @@ -29,15 +38,34 @@ export class AdaptersService { } listAll(): AdapterMeta[] { - return listAdapters(); + return listAdapters() + .filter((a) => this.isInstallableHere(a)) + .map((a) => ({ + ...a, + requiredEnvVars: withoutOperatorProvided(a.requiredEnvVars) ?? [], + })); } getBySlug(slug: string): AdapterDefinition { const adapter = getAdapter(slug); - if (!adapter) { + if (!adapter || !this.isInstallableHere(adapter)) { throw new NotFoundException(`Adapter "${slug}" not found`); } - return adapter; + return { + ...adapter, + requiredEnvVars: withoutOperatorProvided(adapter.requiredEnvVars) ?? [], + }; + } + + /** + * Adapters that only work from a residential IP (upstreams behind Cloudflare + * or Akamai bot walls, session-token scrapers) are marked `selfHostOnly` in + * the catalog. From the cloud's datacenter address they fail every call, so + * the cloud does not list them at all. Self-host shows them as usual. + */ + private isInstallableHere(adapter: { selfHostOnly?: boolean }): boolean { + if (!adapter.selfHostOnly) return true; + return this.configService.get('DEPLOYMENT_MODE') !== 'cloud'; } async importAdapter( @@ -45,9 +73,17 @@ export class AdaptersService { userId: string, organizationId: string, credentials?: Record, - ): Promise<{ connectorId: string; toolsCreated: number }> { + ): Promise<{ + connectorId: string; + toolsCreated: number; + probe: ImportProbeResult | null; + }> { const adapter = this.getBySlug(slug); + // Values the operator provides for everyone (e.g. the cloud's own MOTIS + // URL) go in here, and override anything the request carried. + credentials = withOperatorProvided(credentials); + // Credentials arrive from the UI verbatim β€” a stray leading/trailing // space (easy to pick up when pasting) would otherwise be encrypted into // authConfig and break auth downstream (e.g. Basic Auth 401s that are @@ -163,7 +199,69 @@ export class AdaptersService { `Imported adapter "${slug}" as connector ${connector.id} with ${toolsCreated} tools`, ); - return { connectorId: connector.id, toolsCreated }; + const probe = await this.runImportProbe(adapter, connector.id); + + return { connectorId: connector.id, toolsCreated, probe }; + } + + /** + * Call one read-only tool of the connector just created, with the + * credentials just entered, and report the outcome to the install form. + * + * Never blocks or undoes the import: a slow or unreachable upstream is + * reported, not treated as a failed install (the connector may need an + * allow-listed IP, or the user may fix a value in the editor). The result is + * whatever the agent would have got, run through the same engine and the + * same response mapping, so "green" here means the first real call will + * work and "red" carries the upstream's own words. + */ + private async runImportProbe( + adapter: AdapterDefinition, + connectorId: string, + ): Promise { + const call = pickProbe(adapter); + if (!call) return null; + const started = Date.now(); + try { + const connector = await this.prisma.connector.findUnique({ + where: { id: connectorId }, + include: { tools: { where: { name: call.toolName } } }, + }); + const tool = connector?.tools[0]; + if (!connector || !tool) return null; + const raw = await this.connectors.executeConnectorCall( + connector, + tool.endpointMapping as any, + call.params, + ); + const shaped = applyResponseTransform(raw, tool.responseMapping as any).value; + return { + ok: true, + toolName: call.toolName, + durationMs: Date.now() - started, + sample: truncateSample(shaped), + }; + } catch (err: any) { + const status: number | undefined = + typeof err?.status === 'number' + ? err.status + : typeof err?.response?.status === 'number' + ? err.response.status + : undefined; + const upstream = String(err?.message ?? err ?? 'unknown error').slice(0, 400); + const { hint } = classifyToolExecutionError({ + status, + authType: adapter.connector.authType, + message: upstream, + }); + return { + ok: false, + toolName: call.toolName, + durationMs: Date.now() - started, + status: status ?? null, + message: `${upstream} ${hint}`.trim(), + }; + } } /** Replace {{VAR}} placeholders in a string with credential values */ @@ -259,3 +357,25 @@ export class AdaptersService { return obj; } } + +export type ImportProbeResult = + | { ok: true; toolName: string; durationMs: number; sample: string } + | { + ok: false; + toolName: string; + durationMs: number; + status: number | null; + message: string; + }; + +/** A short, printable slice of the probe's response for the install form. */ +function truncateSample(value: unknown, max = 600): string { + let text: string; + try { + text = typeof value === 'string' ? value : JSON.stringify(value); + } catch { + text = String(value); + } + if (!text) return ''; + return text.length > max ? `${text.slice(0, max)}…` : text; +} diff --git a/packages/backend/src/adapters/catalog.ts b/packages/backend/src/adapters/catalog.ts index 9527f4b2..70512ba6 100644 --- a/packages/backend/src/adapters/catalog.ts +++ b/packages/backend/src/adapters/catalog.ts @@ -6,6 +6,7 @@ import * as datev from './de/datev.json'; import * as datevSandbox from './de/datev-sandbox.json'; import * as destatisGenesis from './de/destatis-genesis.json'; import * as deutscheBahn from './de/deutsche-bahn.json'; +import * as deutscheBahnTimetables from './de/deutsche-bahn-timetables.json'; import * as dhlTracking from './de/dhl-tracking.json'; import * as dpdGermany from './de/dpd-germany.json'; import * as easybill from './de/easybill.json'; @@ -220,6 +221,16 @@ export interface AdapterMeta { featured?: boolean; /** Higher = ranked earlier in catalog listings. Default 0. */ priority?: number; + /** The upstream only answers residential IPs (bot wall, session-token + * scraper). Listed on self-host, hidden on the cloud, and excluded from the + * advertised "no API key" count. See scripts/probe-keyless.mjs. */ + selfHostOnly?: boolean; + /** A safe, read-only call that proves the connector works: run right after + * import so a wrong credential is reported on the install form instead of + * by the agent days later, and by scripts/probe-keyless.mjs in CI. Params + * may use the literal `__TOMORROW__` for a date. Without one, the first + * GET tool with no required parameters is used. */ + probe?: { tool: string; params?: Record }; } export interface AdapterDefinition extends AdapterMeta { @@ -300,6 +311,7 @@ const RAW_ADAPTERS: AdapterDefinition[] = [ datevSandbox as unknown as AdapterDefinition, destatisGenesis as unknown as AdapterDefinition, deutscheBahn as unknown as AdapterDefinition, + deutscheBahnTimetables as unknown as AdapterDefinition, dhlTracking as unknown as AdapterDefinition, dpdGermany as unknown as AdapterDefinition, easybill as unknown as AdapterDefinition, @@ -510,6 +522,8 @@ export function listAdapters(): AdapterMeta[] { authType: adapter.connector.authType, featured: adapter.featured, priority: adapter.priority, + selfHostOnly: adapter.selfHostOnly, + probe: adapter.probe, })); } diff --git a/packages/backend/src/adapters/cloud-managed-env.spec.ts b/packages/backend/src/adapters/cloud-managed-env.spec.ts new file mode 100644 index 00000000..17f627da --- /dev/null +++ b/packages/backend/src/adapters/cloud-managed-env.spec.ts @@ -0,0 +1,37 @@ +import { + operatorProvidedEnvVars, + withOperatorProvided, + withoutOperatorProvided, +} from './cloud-managed-env'; + +describe('operator-provided adapter env vars', () => { + it('is empty when the operator set nothing', () => { + expect(operatorProvidedEnvVars({})).toEqual({}); + expect(withoutOperatorProvided(['MOTIS_URL'], {})).toEqual(['MOTIS_URL']); + expect(withOperatorProvided(undefined, {})).toBeUndefined(); + }); + + it('maps MOTIS_INTERNAL_URL onto the adapter var, without a trailing slash', () => { + const env = { MOTIS_INTERNAL_URL: 'http://motis:8080/' }; + expect(operatorProvidedEnvVars(env)).toEqual({ MOTIS_URL: 'http://motis:8080' }); + }); + + it('ignores a blank value', () => { + expect(operatorProvidedEnvVars({ MOTIS_INTERNAL_URL: ' ' })).toEqual({}); + }); + + it('hides the provided var from the install form and leaves the rest', () => { + const env = { MOTIS_INTERNAL_URL: 'http://motis:8080' }; + expect(withoutOperatorProvided(['MOTIS_URL', 'OTHER'], env)).toEqual(['OTHER']); + expect(withoutOperatorProvided(undefined, env)).toBeUndefined(); + }); + + it('lets the operator value win over one posted by the user', () => { + // A cloud user must not be able to aim the connector at an arbitrary host + // on the internal network by sending their own MOTIS_URL. + const env = { MOTIS_INTERNAL_URL: 'http://motis:8080' }; + expect( + withOperatorProvided({ MOTIS_URL: 'http://169.254.169.254', X: '1' }, env), + ).toEqual({ MOTIS_URL: 'http://motis:8080', X: '1' }); + }); +}); diff --git a/packages/backend/src/adapters/cloud-managed-env.ts b/packages/backend/src/adapters/cloud-managed-env.ts new file mode 100644 index 00000000..ecb283d1 --- /dev/null +++ b/packages/backend/src/adapters/cloud-managed-env.ts @@ -0,0 +1,57 @@ +/** + * Adapter env vars the operator provides for everyone. + * + * Some adapters need a URL that is an infrastructure decision rather than a + * credential: the Deutsche Bahn adapter needs a MOTIS instance, and the cloud + * runs one next to the app. Asking every cloud user to type + * `http://motis:8080` into an install form would be absurd β€” and a self-hoster + * who enabled the bundled `motis` compose profile is in the same position. + * + * So when the operator sets the matching backend env var, the adapter var is + * filled in at import and hidden from the install form. When it is unset the + * adapter behaves as any other: the form asks for the value. + * + * Keyed on the env var, not on DEPLOYMENT_MODE, so the same mechanism serves + * cloud and a self-host that runs the optional service. + */ +const MANAGED: ReadonlyArray<{ adapterVar: string; envVar: string }> = [ + { adapterVar: 'MOTIS_URL', envVar: 'MOTIS_INTERNAL_URL' }, +]; + +/** The adapter vars the operator has provided, with their values. */ +export function operatorProvidedEnvVars( + env: NodeJS.ProcessEnv = process.env, +): Record { + const out: Record = {}; + for (const { adapterVar, envVar } of MANAGED) { + const value = env[envVar]?.trim(); + if (value) out[adapterVar] = value.replace(/\/+$/, ''); + } + return out; +} + +/** Strip operator-provided vars from an install-form prompt list. */ +export function withoutOperatorProvided( + vars: string[] | undefined, + env: NodeJS.ProcessEnv = process.env, +): string[] | undefined { + if (!vars) return vars; + const provided = operatorProvidedEnvVars(env); + return vars.filter((v) => !(v in provided)); +} + +/** + * Merge operator-provided values into the credentials an import was given. + * The operator's value wins: a user cannot point a cloud connector at a MOTIS + * of their choosing by posting `MOTIS_URL` in the request body, which would + * otherwise turn the connector into an SSRF vector aimed at the internal + * network. + */ +export function withOperatorProvided( + credentials: Record | undefined, + env: NodeJS.ProcessEnv = process.env, +): Record | undefined { + const provided = operatorProvidedEnvVars(env); + if (Object.keys(provided).length === 0) return credentials; + return { ...(credentials ?? {}), ...provided }; +} diff --git a/packages/backend/src/adapters/de/bundesbank.json b/packages/backend/src/adapters/de/bundesbank.json index e05eb4ce..a71fd66d 100644 --- a/packages/backend/src/adapters/de/bundesbank.json +++ b/packages/backend/src/adapters/de/bundesbank.json @@ -7,6 +7,12 @@ "icon": "bundesbank", "docsUrl": "https://api.statistiken.bundesbank.de/doc/index.html", "requiredEnvVars": [], + "probe": { + "tool": "bundesbank_get_exchange_rates", + "params": { + "currency": "USD" + } + }, "connector": { "name": "Bundesbank Statistics", "type": "REST", diff --git a/packages/backend/src/adapters/de/deutsche-bahn-timetables.json b/packages/backend/src/adapters/de/deutsche-bahn-timetables.json new file mode 100644 index 00000000..504f7345 --- /dev/null +++ b/packages/backend/src/adapters/de/deutsche-bahn-timetables.json @@ -0,0 +1,132 @@ +{ + "slug": "deutsche-bahn-timetables", + "name": "Deutsche Bahn Timetables API", + "description": "Deutsche Bahn's official station-board API (DB API Marketplace, CC BY 4.0): find stations by name or EVA number, read the planned hourly timetable of any station, and pull the live change feed β€” delays, platform changes, cancellations and disruption messages β€” straight from DB. Free key, 60 requests per minute. Boards only; use the Deutsche Bahn Fahrplan connector for journey planning.", + "instructions": "This connector calls Deutsche Bahn's official **Timetables API v1** on the DB API Marketplace. It is the source of truth for what is happening at a station right now, published by DB itself (CC BY 4.0). It does not plan journeys β€” pair it with the *Deutsche Bahn Fahrplan* connector for that.\n\n**Getting a key**: create a free account at https://developers.deutschebahn.com, create an application, subscribe it to the *Timetables* API (free plan, 60 requests/minute) and copy the application's Client ID and API Key into `DB_CLIENT_ID` / `DB_API_KEY`.\n\n**Workflow**\n1. `dbt_search_stations` resolves a name (prefix), a DS100 code or an EVA number to the station's `eva` (e.g. `8000107` = Freiburg(Breisgau) Hbf, `8011160` = Berlin Hbf, `8000105` = Frankfurt(Main)Hbf, `8000261` = MΓΌnchen Hbf). Umlauts in a prefix are unreliable β€” search 'Munchen' or 'Muenchen' as well as 'MΓΌnchen'.\n2. `dbt_get_station_board` returns the **planned** timetable of one hour: every train with its `id`, category (`ICE`, `RE`, `S` …), number, planned arrival/departure time and platform, and the route (`from` / `to` are the stop lists as `|`-separated strings).\n3. `dbt_get_changes` returns **all known changes** for the station (updated every 30 s): per train `id`, the changed time (`ct`), changed platform (`cp`), status (`cs`: `c` = cancelled, `a` = added) and messages. `dbt_get_recent_changes` is the same feed limited to the last two minutes β€” poll it instead of the full feed when watching a station.\n4. Match the `id` from the plan with the `id` in the changes; delay in minutes = changed time βˆ’ planned time.\n\n**Times** are `yyMMddHHmm` in **local German time** (`2609151427` = 15 Sep 2026, 14:27). The plan is sliced by hour: `date` is `yyMMdd`, `hour` is `HH` (00–23), both of the current local time in Germany.\n\n**Changes without a matching plan entry** are trains that stop at the station in another hour; fetch that hour's plan if they matter.\n\nLimits: 60 requests per minute on the free plan. Attribution: Deutsche Bahn AG, Timetables API, CC BY 4.0.", + "region": "de", + "category": "transport", + "icon": "db", + "docsUrl": "https://developers.deutschebahn.com/db-api-marketplace/apis/product/timetables", + "requiredEnvVars": ["DB_CLIENT_ID", "DB_API_KEY"], + "connector": { + "name": "Deutsche Bahn Timetables API", + "type": "REST", + "baseUrl": "https://apis.deutschebahn.com/db-api-marketplace/apis/timetables/v1", + "authType": "API_KEY", + "authConfig": { + "headerName": "DB-Api-Key", + "apiKey": "{{DB_API_KEY}}" + }, + "headers": { + "DB-Client-Id": "{{DB_CLIENT_ID}}", + "Accept": "application/xml" + }, + "healthcheckPath": "/station/Berlin%20Hbf" + }, + "tools": [ + { + "name": "dbt_search_stations", + "description": "Find Deutsche Bahn stations by name prefix, DS100 code or EVA number. Returns `name`, `eva` (the station number every other tool takes) and `ds100`. Prefix search: 'Freiburg' lists every station starting with Freiburg; 'Freiburg Hbf' narrows it. A trailing '*' widens the match. Umlauts are unreliable in prefixes β€” try 'Munchen' as well as 'MΓΌnchen'.", + "parameters": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Station name prefix (e.g. 'Freiburg Hbf', 'Berlin Hbf'), DS100 code (e.g. 'RF') or EVA number (e.g. '8000107')." + } + }, + "required": ["pattern"] + }, + "endpointMapping": { + "method": "GET", + "path": "/station/{pattern}" + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "stations.station && to_array(stations.station)[].{name: name, eva: eva, ds100: ds100}" + } + } + }, + { + "name": "dbt_get_station_board", + "description": "Planned timetable of a station for one hour: every train with `id` (match it against dbt_get_changes), `category` (ICE, IC, EC, RE, RB, S …), `number`, `line`, planned `arrival` / `departure` (yyMMddHHmm, local time) and platforms, `from` (previous stops, '|'-separated, origin first) and `to` (next stops, destination last). `date` is yyMMdd and `hour` is HH, both local German time. This is the schedule; live delays and cancellations come from dbt_get_changes.", + "parameters": { + "type": "object", + "properties": { + "eva": { + "type": "string", + "description": "Station EVA number from dbt_search_stations, e.g. '8000107'." + }, + "date": { + "type": "string", + "description": "Date as yyMMdd in local German time, e.g. '260915' for 15 September 2026." + }, + "hour": { + "type": "string", + "description": "Hour as HH (00–23) in local German time, e.g. '14'." + } + }, + "required": ["eva", "date", "hour"] + }, + "endpointMapping": { + "method": "GET", + "path": "/plan/{eva}/{date}/{hour}" + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "{station: timetable.station, trains: timetable.s && to_array(timetable.s)[].{id: id, category: tl.c, number: tl.n, line: dp.l || ar.l, arrival: ar.pt, arrivalPlatform: ar.pp, from: ar.ppth, departure: dp.pt, departurePlatform: dp.pp, to: dp.ppth}}" + } + } + }, + { + "name": "dbt_get_changes", + "description": "All currently known changes at a station, straight from Deutsche Bahn (refreshed every 30 s): per train `id` (matches dbt_get_station_board), `arrival` / `departure` with `changedTime` (yyMMddHHmm local β€” compare with the planned time for the delay), `changedPlatform`, `status` ('c' = cancelled, 'a' = added, 'p' = planned), `changedPath`, and `messages` (disruptions and information with `category`, validity `from` / `to` and a priority). Covers every hour, so entries may refer to trains outside the plan you fetched.", + "parameters": { + "type": "object", + "properties": { + "eva": { + "type": "string", + "description": "Station EVA number from dbt_search_stations." + } + }, + "required": ["eva"] + }, + "endpointMapping": { + "method": "GET", + "path": "/fchg/{eva}" + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "{station: timetable.station, changes: timetable.s && to_array(timetable.s)[].{id: id, arrival: ar && {changedTime: ar.ct, changedPlatform: ar.cp, status: ar.cs, changedPath: ar.cpth}, departure: dp && {changedTime: dp.ct, changedPlatform: dp.cp, status: dp.cs, changedPath: dp.cpth}, messages: m && to_array(m)[].{type: t, category: cat, code: c, from: from, to: to, priority: pr}}}" + } + } + }, + { + "name": "dbt_get_recent_changes", + "description": "Changes at a station from the last two minutes only β€” the same shape as dbt_get_changes, for polling a station you are watching without re-reading the full change feed each time.", + "parameters": { + "type": "object", + "properties": { + "eva": { + "type": "string", + "description": "Station EVA number from dbt_search_stations." + } + }, + "required": ["eva"] + }, + "endpointMapping": { + "method": "GET", + "path": "/rchg/{eva}" + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "{station: timetable.station, changes: timetable.s && to_array(timetable.s)[].{id: id, arrival: ar && {changedTime: ar.ct, changedPlatform: ar.cp, status: ar.cs, changedPath: ar.cpth}, departure: dp && {changedTime: dp.ct, changedPlatform: dp.cp, status: dp.cs, changedPath: dp.cpth}, messages: m && to_array(m)[].{type: t, category: cat, code: c, from: from, to: to, priority: pr}}}" + } + } + } + ] +} diff --git a/packages/backend/src/adapters/de/deutsche-bahn.json b/packages/backend/src/adapters/de/deutsche-bahn.json index 7925e225..a24853f2 100644 --- a/packages/backend/src/adapters/de/deutsche-bahn.json +++ b/packages/backend/src/adapters/de/deutsche-bahn.json @@ -1,38 +1,41 @@ { "slug": "deutsche-bahn", "name": "Deutsche Bahn Fahrplan", - "description": "Query the Deutsche Bahn (DB) timetable via a db-rest instance (the community REST wrapper around DB's public transport APIs). Search stations, look up real-time departures/arrivals, and plan journeys between any two DB stops.", - "instructions": "This connector talks to a **db-rest** instance β€” the community REST wrapper around Deutsche Bahn's public transport APIs. By default it uses the public `v6.db.transport.rest`; AnythingMCP Cloud automatically routes it to an internal self-hosted db-rest for reliability. No authentication required.\n\n**Typical workflow**:\n1. Resolve a station name to a stop `id` (IBNR, e.g. `8000107` = Freiburg(Breisgau) Hbf) with `db_search_locations`.\n2. Pass that `id` to `db_get_departures` / `db_get_arrivals` / `db_get_stop`.\n3. For trip planning, pass `from` and `to` (both IBNR ids) to `db_get_journeys`.\n\n**Field naming** (FPTI / db-rest, English keys):\n- `id` β€” stable stop id (IBNR). Use it as `id` / `from` / `to`.\n- `when` β€” real/estimated time (ISO 8601 WITH timezone). `plannedWhen` β€” scheduled time. `delay` β€” seconds (`0` = on time, `null` = unknown).\n- `platform` / `plannedPlatform`.\n- `direction` β€” where the train is heading (departures); arrivals carry `provenance` instead.\n- `line` β€” `{ name, product }`, product ∈ {nationalExpress (ICE), national (IC/EC), regionalExpress, regional, suburban (S-Bahn), bus, tram, subway, ferry}.\n- `remarks` β€” service messages / disruptions.\n\n**Times** are ISO 8601 **with** timezone offset (e.g. `2026-05-27T08:55:00+02:00`).\n\n**Departures/arrivals**: omit `when` for boards starting now; `duration` (minutes) sets the window.\n\n**Journeys**: pass either `departure` or `arrival` (ISO 8601); omit both for 'now'. `arrival` = arrive-by.", + "description": "German train timetable with live delays: search stations, list departures and arrivals with real-time delay, track and cancellation status, and plan journeys between any two stations. Runs on MOTIS over the open gtfs.de timetable (ICE, IC, EC, regional trains, S-Bahn) β€” no bahn.de scraping, no API key.", + "instructions": "This connector queries a **MOTIS** routing engine loaded with the open German train timetable (gtfs.de, all long-distance and regional trains incl. S-Bahn) and a live GTFS-RT feed for delays, platform changes and cancellations. AnythingMCP Cloud runs its own instance; self-hosters run the bundled `motis` service (`deploy/motis/README.md`) or point `MOTIS_URL` at any MOTIS. No API key.\n\n**Workflow**\n1. `db_search_locations` turns a station name into a stop `id` (e.g. `fv_384020`). Ids change when the timetable is refreshed β€” resolve names in every session, never reuse ids from memory.\n2. `db_get_departures` / `db_get_arrivals` with that `id` for a live board; `db_get_stop` for the station's details.\n3. `db_get_journeys` with two ids plans a trip (station to station; there is no street routing, so coordinates are not accepted).\n\n**Reading the fields**\n- Times are ISO 8601 in **UTC** (`2026-09-15T13:27:00Z`). Germany is UTC+1 (winter) / UTC+2 (summer) β€” convert before quoting a time to the user.\n- `departure` / `arrival` are the live prediction, `scheduledDeparture` / `scheduledArrival` the timetable. Delay in minutes = live βˆ’ scheduled. `realTime: true` means the row carries live data; `false` means schedule only.\n- `cancelled: true` on a departure or leg means that stop or leg is cancelled; `tripCancelled` means the whole train is.\n- `track` is the live platform, `scheduledTrack` the planned one; both may be absent for smaller stops.\n- `line` is the train name as printed on the board (`ICE 43`, `RE 7`, `S1`). The free gtfs.de feed reports every train as mode `REGIONAL_RAIL`; tell ICE/IC/EC from regional trains by the `line` prefix, not by mode.\n\n**Boards**: omit `when` for a board starting now; `n` is the number of rows (default 12, max 100). Big stations have many S-Bahn rows β€” raise `n` when looking for a specific long-distance train.\n\n**Journeys**: `time` is the departure time unless `arrive_by` is true; omit for now. `results` caps the itineraries (default 4). Legs with `mode: WALK` are transfers.\n\nData: DELFI e.V. / gtfs.de (CC BY 4.0), GTFS-RT gtfs.de (CC BY-SA 4.0). Engine: MOTIS (MIT).", "region": "de", "category": "transport", "icon": "db", - "docsUrl": "https://github.com/derhuerst/db-rest", - "requiredEnvVars": [], + "docsUrl": "https://github.com/HelpCode-ai/anythingmcp/blob/main/deploy/motis/README.md", + "requiredEnvVars": [ + "MOTIS_URL" + ], + "probe": { + "tool": "db_search_locations", + "params": { + "query": "Freiburg Hbf" + } + }, "connector": { "name": "Deutsche Bahn Fahrplan", "type": "REST", - "baseUrl": "https://v6.db.transport.rest", + "baseUrl": "{{MOTIS_URL}}", "authType": "NONE", "headers": { - "Accept-Language": "de-DE", "User-Agent": "anythingmcp/1.0 (+https://anythingmcp.com)" - } + }, + "healthcheckPath": "/" }, "tools": [ { "name": "db_search_locations", - "description": "Search Deutsche Bahn stops/stations by name. Returns an array of matches, each with `id` (the stable IBNR id used everywhere else), `name`, `location` (latitude/longitude), and `products` (which services call at this stop). Use the returned `id` as the `id` parameter for db_get_departures, db_get_arrivals, db_get_stop, and as `from`/`to` for db_get_journeys.", + "description": "Search German train stations by name. Returns up to 10 matches with `id` (use it as `id` in db_get_departures / db_get_arrivals / db_get_stop and as `from` / `to` in db_get_journeys), `name`, `lat`, `lon` and `modes`. Ids change when the timetable is refreshed, so resolve the name each session. Search the way a German timetable spells it: 'Freiburg Hbf', 'Berlin Hbf', 'Frankfurt (Main) Hauptbahnhof', 'MΓΌnchen Hbf'.", "parameters": { "type": "object", "properties": { "query": { "type": "string", - "description": "Search term β€” station name, city, abbreviation. Examples: 'Freiburg Hbf', 'Berlin Ostbahnhof', 'MΓΌnchen Hauptbahnhof', 'KΓΆln'." - }, - "limit": { - "type": "number", - "description": "Max results to return (default: 10).", - "default": 10 + "description": "Station name or fragment, e.g. 'Freiburg Hbf', 'Hamburg Altona', 'KΓΆln Messe/Deutz'." } }, "required": [ @@ -41,25 +44,29 @@ }, "endpointMapping": { "method": "GET", - "path": "/locations", + "path": "/api/v1/geocode", "queryParams": { - "query": "$query", - "results": "$limit", - "addresses": "false", - "poi": "false", - "profile": "dbnav" + "text": "$query", + "type": "STOP", + "language": "de" + } + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "[].{id: id, name: name, lat: lat, lon: lon, modes: modes}" } } }, { "name": "db_get_stop", - "description": "Retrieve a single Deutsche Bahn stop by its `id` (IBNR), returning `name`, `location` (latitude/longitude), and the `products` that call there. Useful when you already have the id but need the station metadata.", + "description": "Details of one station by its `id` from db_search_locations: `name`, `lat`, `lon` and the `modes` served there. Use when you already hold an id and need to confirm which station it is.", "parameters": { "type": "object", "properties": { "id": { "type": "string", - "description": "The stop's IBNR / id from db_search_locations (e.g. '8000105' = Frankfurt(Main)Hbf, '8000107' = Freiburg(Breisgau) Hbf, '8011160' = Berlin Hbf)." + "description": "Stop id from db_search_locations, e.g. 'fv_384020'." } }, "required": [ @@ -68,30 +75,37 @@ }, "endpointMapping": { "method": "GET", - "path": "/stops/{id}", + "path": "/api/v1/stoptimes", "queryParams": { - "profile": "dbnav" + "stopId": "$id", + "n": "1" + } + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "place.{id: stopId, name: name, lat: lat, lon: lon, modes: modes}" } } }, { "name": "db_get_departures", - "description": "List upcoming departures from a DB stop with live delays. Returns `departures[]` β€” each entry has `tripId`, `when` (real/estimated ISO time), `plannedWhen`, `delay` (seconds), `platform`/`plannedPlatform`, `direction` (where the train is heading), `line` (`name`, `product`), and `remarks` (disruption messages). `delay` of 0 = on time, `null` = unknown.", + "description": "Live departure board for a station: the next `n` trains with `line` (e.g. 'ICE 43', 'RE 7', 'S1'), `headsign` (where the train is going), `scheduledDeparture` and live `departure` (both ISO 8601 UTC β€” the difference is the delay), `scheduledTrack` / `track`, `realTime` (true = live data), `cancelled` and `tripCancelled`. Omit `when` for departures from now.", "parameters": { "type": "object", "properties": { "id": { "type": "string", - "description": "The stop's IBNR / id (e.g. '8000107' for Freiburg Hbf)." + "description": "Stop id from db_search_locations." }, "when": { "type": "string", - "description": "Optional ISO 8601 reference time (with timezone), e.g. '2026-05-27T08:30:00+02:00'. Omit for departures starting now." + "description": "Optional start of the board as ISO 8601 with timezone or 'Z', e.g. '2026-09-15T08:30:00+02:00'. Omit for now." }, - "duration": { + "n": { "type": "number", - "description": "Optional window in minutes to look ahead (default: 10).", - "default": 10 + "description": "Number of departures to return (default 12, max 100). Raise it at big stations when looking for a specific long-distance train among many S-Bahn rows.", + "default": 12 } }, "required": [ @@ -100,32 +114,39 @@ }, "endpointMapping": { "method": "GET", - "path": "/stops/{id}/departures", + "path": "/api/v1/stoptimes", "queryParams": { - "when": "$when", - "duration": "$duration", - "profile": "dbnav" + "stopId": "$id", + "time": "$when", + "n": "$n", + "arriveBy": "false" + } + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "{stop: place.name, stopId: place.stopId, departures: stopTimes[].{line: routeShortName, headsign: headsign, scheduledDeparture: place.scheduledDeparture, departure: place.departure, scheduledTrack: place.scheduledTrack, track: place.track, realTime: realTime, cancelled: cancelled, tripCancelled: tripCancelled, agency: agencyName}}" } } }, { "name": "db_get_arrivals", - "description": "List upcoming arrivals at a DB stop with live delays. Same response shape as db_get_departures (`arrivals[]`), but each entry carries `provenance` (the train's origin) instead of `direction`, and `when`/`plannedWhen` are arrival times.", + "description": "Live arrival board for a station: the next `n` arriving trains with `line`, `origin` (where the train started), `headsign`, `scheduledArrival` and live `arrival` (ISO 8601 UTC β€” the difference is the delay), `scheduledTrack` / `track`, `realTime`, `cancelled` and `tripCancelled`. Omit `when` for arrivals from now.", "parameters": { "type": "object", "properties": { "id": { "type": "string", - "description": "The stop's IBNR / id (e.g. '8011160' for Berlin Hbf)." + "description": "Stop id from db_search_locations." }, "when": { "type": "string", - "description": "Optional ISO 8601 reference time (with timezone). Omit for arrivals starting now." + "description": "Optional start of the board as ISO 8601 with timezone or 'Z'. Omit for now." }, - "duration": { + "n": { "type": "number", - "description": "Optional window in minutes to look ahead (default: 10).", - "default": 10 + "description": "Number of arrivals to return (default 12, max 100).", + "default": 12 } }, "required": [ @@ -134,44 +155,51 @@ }, "endpointMapping": { "method": "GET", - "path": "/stops/{id}/arrivals", + "path": "/api/v1/stoptimes", "queryParams": { - "when": "$when", - "duration": "$duration", - "profile": "dbnav" + "stopId": "$id", + "time": "$when", + "n": "$n", + "arriveBy": "true" + } + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "{stop: place.name, stopId: place.stopId, arrivals: stopTimes[].{line: routeShortName, origin: tripFrom.name, headsign: headsign, scheduledArrival: place.scheduledArrival, arrival: place.arrival, scheduledTrack: place.scheduledTrack, track: place.track, realTime: realTime, cancelled: cancelled, tripCancelled: tripCancelled, agency: agencyName}}" } } }, { "name": "db_get_journeys", - "description": "Plan one or more itineraries between two DB stops. Returns `journeys[]` β€” each journey has `legs[]`; each leg includes `origin`/`destination` (stop `id`, `name`), `departure`/`plannedDeparture`, `arrival`/`plannedArrival`, `departurePlatform`/`arrivalPlatform`, `line` (`name`, `product`), and `remarks` (alerts). Use IBNR `id` for `from` and `to`. Pass either `departure` (depart-at) or `arrival` (arrive-by); omit both for 'now'.", + "description": "Plan train journeys between two stations. Returns itineraries with `departure`, `arrival` (ISO 8601 UTC), `durationMinutes`, `transfers` and `legs[]` β€” each leg has `mode` (`WALK` = transfer on foot), `line`, `headsign`, `from` / `to` with scheduled and live times and tracks, `realTime` and `cancelled`. Both ends must be stop ids from db_search_locations (station to station β€” no street routing). `time` is the departure time unless `arrive_by` is true; omit both for now.", "parameters": { "type": "object", "properties": { "from": { "type": "string", - "description": "Origin stop's IBNR / id (e.g. '8000107' for Freiburg Hbf)." + "description": "Origin stop id from db_search_locations, e.g. 'fv_384020'." }, "to": { "type": "string", - "description": "Destination stop's IBNR / id (e.g. '8011160' for Berlin Hbf)." + "description": "Destination stop id from db_search_locations." }, - "departure": { + "time": { "type": "string", - "description": "Depart-at time as ISO 8601 with timezone, e.g. '2026-05-27T08:00:00+02:00'. Omit for 'now'. Do not set together with `arrival`." + "description": "ISO 8601 with timezone or 'Z', e.g. '2026-09-15T08:00:00+02:00'. Departure time, or arrival time when arrive_by is true. Omit for now." }, - "arrival": { - "type": "string", - "description": "Arrive-by time as ISO 8601 with timezone. Use this instead of `departure` for 'arrive before' queries." + "arrive_by": { + "type": "boolean", + "description": "true = `time` is the latest arrival; false/omitted = `time` is the earliest departure." }, "results": { "type": "number", - "description": "Max number of itineraries to return (default: 4).", + "description": "Maximum number of itineraries (default 4).", "default": 4 }, "bike": { "type": "boolean", - "description": "If true, only return connections that allow bicycle carriage. Default: false." + "description": "true = only trains that carry bicycles." } }, "required": [ @@ -181,15 +209,21 @@ }, "endpointMapping": { "method": "GET", - "path": "/journeys", + "path": "/api/v1/plan", "queryParams": { - "from": "$from", - "to": "$to", - "departure": "$departure", - "arrival": "$arrival", - "results": "$results", - "bike": "$bike", - "profile": "dbnav" + "fromPlace": "$from", + "toPlace": "$to", + "time": "$time", + "arriveBy": "$arrive_by", + "numItineraries": "$results", + "requireBikeTransport": "$bike", + "detailedTransfers": "false" + } + }, + "responseMapping": { + "transform": { + "mode": "jmespath", + "expression": "itineraries[].{departure: startTime, arrival: endTime, durationMinutes: duration / `60`, transfers: transfers, legs: legs[].{mode: mode, line: routeShortName, headsign: headsign, from: from.name, fromId: from.stopId, scheduledDeparture: from.scheduledDeparture, departure: from.departure, departureTrack: from.track, to: to.name, toId: to.stopId, scheduledArrival: to.scheduledArrival, arrival: to.arrival, arrivalTrack: to.track, realTime: realTime, cancelled: cancelled}}" } } } diff --git a/packages/backend/src/adapters/de/deutsche-bahn.live.spec.ts b/packages/backend/src/adapters/de/deutsche-bahn.live.spec.ts index e7660f05..eb991d0c 100644 --- a/packages/backend/src/adapters/de/deutsche-bahn.live.spec.ts +++ b/packages/backend/src/adapters/de/deutsche-bahn.live.spec.ts @@ -2,21 +2,24 @@ import * as adapter from './deutsche-bahn.json'; import { RestEngine } from '../../connectors/engines/rest.engine'; import { OAuth2TokenService } from '../../connectors/engines/oauth2-token.service'; import { LoginTokenService } from '../../connectors/engines/login-token.service'; +import { applyResponseTransform } from '../../connectors/response-transform.util'; +import { compile as jmespathCompile } from '@jmespath-community/jmespath'; /** * Two-layer verification for the deutsche-bahn adapter: * - * 1. Static β€” always runs. Locks in the db-rest upstream schema. The connector - * ships pointing at the public db-rest (v6.db.transport.rest) so self-hosters - * use it as-is; AnythingMCP Cloud transparently rewrites the host to an - * internal self-hosted db-rest (see DynamicMcpTools.resolveInternalBaseUrl). - * This guards against a regression back to the int.bahn.de schema (Akamai - * 403) or the bare v6 proxy without the db-rest endpoints. + * 1. Static β€” always runs. Locks in the MOTIS API contract the adapter + * targets (/api/v1/geocode, /stoptimes, /plan) and that every tool ships + * a compiling JMESPath response mapping. Guards against a regression to + * the db-rest / bahn.de path, which Deutsche Bahn blocks from datacenter + * IPs, and against a typo in a mapping silently returning raw 12 KB boards + * (fallbackToRaw hides that in production). * - * 2. Live β€” opt-in. Hits the public db-rest for real and asserts response shape. - * Run with: RUN_DB_LIVE=1 npx jest src/adapters/de/deutsche-bahn.live.spec.ts - * (The public instance is best-effort and may return 503; cloud uses the - * internal instance instead.) + * 2. Live β€” opt-in. Runs the real tools against a MOTIS instance and checks + * the mapped shapes, including that live data is actually flowing. + * Run with: DB_LIVE_MOTIS_URL=http://localhost:8080 npx jest src/adapters/de/deutsche-bahn.live.spec.ts + * (deploy/motis builds one; the cloud's is http://motis:8080 inside the + * stack.) */ const a = adapter as unknown as { @@ -27,6 +30,7 @@ const a = adapter as unknown as { baseUrl: string; authType: string; headers?: Record; + healthcheckPath?: string; }; tools: Array<{ name: string; @@ -35,114 +39,121 @@ const a = adapter as unknown as { method: string; path: string; queryParams?: Record; - bodyMapping?: Record; }; + responseMapping?: { transform?: { mode?: string; expression?: string } }; }>; }; describe('deutsche-bahn adapter β€” static spec conformance', () => { - it('targets the db-rest schema (public default), not int.bahn.de', () => { + it('targets a MOTIS instance the operator provides, not bahn.de or db-rest', () => { expect(a.slug).toBe('deutsche-bahn'); - expect(a.connector.baseUrl).toBe('https://v6.db.transport.rest'); - expect(a.connector.baseUrl).not.toContain('int.bahn.de'); + expect(a.connector.baseUrl).toBe('{{MOTIS_URL}}'); + expect(a.requiredEnvVars).toEqual(['MOTIS_URL']); expect(a.connector.authType).toBe('NONE'); - expect(a.requiredEnvVars).toEqual([]); + expect(a.connector.healthcheckPath).toBe('/'); + expect(JSON.stringify(adapter)).not.toMatch(/transport\.rest|int\.bahn\.de|db-rest|IBNR/); }); - it('does not route through the anti-bot proxy (db-rest needs no unblocker)', () => { + it('does not route through the anti-bot proxy (open data needs no unblocker)', () => { expect(a.tools.some((t) => t.useProxy === true)).toBe(false); }); - // The shipped adapter pins `dbnav` β€” the profile that resolves from a normal - // (residential) self-host IP. AnythingMCP Cloud overrides it to `dbweb` at - // runtime (see resolveDbRestProfile), because over the cloud's Zyte egress - // Deutsche Bahn blocks the `dbnav` mobile endpoints (Method Not Allowed / - // OPS_BLOCKED) while the `dbweb` web-API endpoints work. Self-host must stay - // on `dbnav`, so this contract is locked here. - it('pins the dbnav profile on every tool (self-host default; cloud overrides to dbweb at runtime)', () => { - for (const t of a.tools) { - expect(t.endpointMapping.queryParams?.profile).toBe('dbnav'); - } - }); - - it('exposes the five timetable tools', () => { - expect(a.tools).toHaveLength(5); - const names = a.tools.map((t) => t.name); - expect(names).toEqual([ + it('exposes the five timetable tools against the MOTIS v1 API', () => { + const byName = (n: string) => a.tools.find((t) => t.name === n)!; + expect(a.tools.map((t) => t.name)).toEqual([ 'db_search_locations', 'db_get_stop', 'db_get_departures', 'db_get_arrivals', 'db_get_journeys', ]); + for (const t of a.tools) expect(t.endpointMapping.method).toBe('GET'); + expect(byName('db_search_locations').endpointMapping.path).toBe('/api/v1/geocode'); + expect(byName('db_search_locations').endpointMapping.queryParams?.type).toBe('STOP'); + expect(byName('db_get_stop').endpointMapping.path).toBe('/api/v1/stoptimes'); + expect(byName('db_get_departures').endpointMapping.path).toBe('/api/v1/stoptimes'); + expect(byName('db_get_departures').endpointMapping.queryParams?.arriveBy).toBe('false'); + expect(byName('db_get_arrivals').endpointMapping.queryParams?.arriveBy).toBe('true'); + const j = byName('db_get_journeys'); + expect(j.endpointMapping.path).toBe('/api/v1/plan'); + expect(j.endpointMapping.queryParams?.fromPlace).toBe('$from'); + expect(j.endpointMapping.queryParams?.toPlace).toBe('$to'); + expect(j.endpointMapping.queryParams?.arriveBy).toBe('$arrive_by'); }); - it('uses the db-rest REST endpoints', () => { - const byName = (n: string) => a.tools.find((t) => t.name === n)!; - expect(byName('db_search_locations').endpointMapping.path).toBe('/locations'); - expect(byName('db_get_stop').endpointMapping.path).toBe('/stops/{id}'); - expect(byName('db_get_departures').endpointMapping.path).toBe( - '/stops/{id}/departures', - ); - expect(byName('db_get_arrivals').endpointMapping.path).toBe( - '/stops/{id}/arrivals', - ); - }); - - it('journeys is a GET to /journeys with from/to query params', () => { - const j = a.tools.find((t) => t.name === 'db_get_journeys')!; - expect(j.endpointMapping.method).toBe('GET'); - expect(j.endpointMapping.path).toBe('/journeys'); - expect(j.endpointMapping.queryParams?.from).toBe('$from'); - expect(j.endpointMapping.queryParams?.to).toBe('$to'); - expect(j.endpointMapping.queryParams?.departure).toBe('$departure'); - expect(j.endpointMapping.queryParams?.arrival).toBe('$arrival'); + it('every tool ships a JMESPath mapping that compiles', () => { + for (const t of a.tools) { + const tf = t.responseMapping?.transform; + expect(tf?.mode).toBe('jmespath'); + expect(() => jmespathCompile(tf!.expression!)).not.toThrow(); + } }); }); -const maybe = process.env.RUN_DB_LIVE ? describe : describe.skip; +const MOTIS_URL = process.env.DB_LIVE_MOTIS_URL; +const maybe = MOTIS_URL ? describe : describe.skip; -maybe('deutsche-bahn adapter β€” live smoke test (public db-rest)', () => { +maybe('deutsche-bahn adapter β€” live smoke test against MOTIS', () => { const oauth = {} as unknown as OAuth2TokenService; const login = {} as unknown as LoginTokenService; const engine = new RestEngine(oauth, login); const cfg = { - baseUrl: a.connector.baseUrl, + baseUrl: MOTIS_URL!, authType: 'NONE', headers: a.connector.headers, }; + const tool = (n: string) => a.tools.find((t) => t.name === n)!; + const run = async (n: string, params: Record) => { + const raw = await engine.execute(cfg, tool(n).endpointMapping as any, params); + const out = applyResponseTransform(raw, tool(n).responseMapping); + expect(out.applied).toBe(true); + return out.value as any; + }; + + let freiburg: string; + let berlin: string; - it('search_locations: returns Freiburg(Breisgau) Hbf with id 8000107', async () => { - const res = (await engine.execute( - cfg, - a.tools.find((t) => t.name === 'db_search_locations')!.endpointMapping, - { query: 'Freiburg(Breisgau) Hbf', limit: 3 }, - )) as Array<{ id: string; name: string }>; + it('search_locations: Freiburg Hbf resolves to Freiburg Hauptbahnhof first', async () => { + const res = await run('db_search_locations', { query: 'Freiburg Hbf' }); expect(Array.isArray(res)).toBe(true); - const fr = res.find((r) => r.id === '8000107'); - expect(fr).toBeDefined(); - expect(fr!.name).toContain('Freiburg'); + expect(res[0].name).toMatch(/Freiburg/); + expect(Object.keys(res[0]).sort()).toEqual(['id', 'lat', 'lon', 'modes', 'name']); + freiburg = res[0].id; + berlin = (await run('db_search_locations', { query: 'Berlin Hbf' }))[0].id; }, 30000); - it('get_departures: returns departures[] with line + direction', async () => { - const res = (await engine.execute( - cfg, - a.tools.find((t) => t.name === 'db_get_departures')!.endpointMapping, - { id: '8000107', duration: 30 }, - )) as { departures: Array<{ line: unknown; direction: string }> }; - expect(res.departures).toBeDefined(); + it('get_stop: returns the station by id', async () => { + const res = await run('db_get_stop', { id: freiburg }); + expect(res.id).toBe(freiburg); + expect(res.name).toMatch(/Freiburg/); + }, 30000); + + it('get_departures: a compact board with live data on at least one row', async () => { + const res = await run('db_get_departures', { id: freiburg, n: 20 }); + expect(res.stop).toMatch(/Freiburg/); expect(res.departures.length).toBeGreaterThan(0); - expect(res.departures[0].line).toBeDefined(); + const row = res.departures[0]; + expect(typeof row.line).toBe('string'); + expect(row.scheduledDeparture).toMatch(/^\d{4}-\d{2}-\d{2}T/); + // The GTFS-RT feed is polled every two minutes; a board with no live rows + // means the feed is not being applied, which is the regression to catch. + expect(res.departures.some((d: any) => d.realTime === true)).toBe(true); + expect(JSON.stringify(res).length).toBeLessThan(8000); + }, 30000); + + it('get_arrivals: carries the origin of each train', async () => { + const res = await run('db_get_arrivals', { id: freiburg, n: 5 }); + expect(res.arrivals.length).toBeGreaterThan(0); + expect(typeof res.arrivals[0].origin).toBe('string'); + expect(res.arrivals[0].scheduledArrival).toMatch(/^\d{4}/); }, 30000); - it('get_journeys: Freiburg β†’ Berlin returns at least one journey', async () => { - const res = (await engine.execute( - cfg, - a.tools.find((t) => t.name === 'db_get_journeys')!.endpointMapping, - { from: '8000107', to: '8011160', results: 2 }, - )) as { journeys: unknown[] }; - expect(res.journeys).toBeDefined(); - expect(res.journeys.length).toBeGreaterThan(0); + it('get_journeys: Freiburg β†’ Berlin has a long-distance leg', async () => { + const res = await run('db_get_journeys', { from: freiburg, to: berlin, results: 2 }); + expect(res.length).toBeGreaterThan(0); + const it0 = res[0]; + expect(typeof it0.durationMinutes).toBe('number'); + expect(it0.legs.some((l: any) => /^(ICE|IC|EC|ECE)\b/.test(l.line ?? ''))).toBe(true); }, 60000); }); diff --git a/packages/backend/src/adapters/de/dpd-germany.json b/packages/backend/src/adapters/de/dpd-germany.json index 9cfee6ea..efd427ee 100644 --- a/packages/backend/src/adapters/de/dpd-germany.json +++ b/packages/backend/src/adapters/de/dpd-germany.json @@ -7,6 +7,7 @@ "category": "logistics", "icon": "dpd", "docsUrl": "https://www.dpd.com/de/de/hilfe-service/entwickler-api/", + "selfHostOnly": true, "requiredEnvVars": [], "connector": { "name": "DPD Germany Tracking", @@ -30,7 +31,9 @@ "description": "Locale for event descriptions: 'de_DE', 'en_DE', 'fr_FR', 'it_IT', etc. Defaults to 'en_DE'." } }, - "required": ["parcelNumber"] + "required": [ + "parcelNumber" + ] }, "endpointMapping": { "method": "GET", diff --git a/packages/backend/src/adapters/de/nina-warnung.json b/packages/backend/src/adapters/de/nina-warnung.json index 31e9ad45..99e90f36 100644 --- a/packages/backend/src/adapters/de/nina-warnung.json +++ b/packages/backend/src/adapters/de/nina-warnung.json @@ -7,6 +7,12 @@ "icon": "nina", "docsUrl": "https://nina.api.bund.dev/", "requiredEnvVars": [], + "probe": { + "tool": "nina_get_active_warnings", + "params": { + "source": "mowas" + } + }, "connector": { "name": "NINA Warnungen", "type": "REST", diff --git a/packages/backend/src/adapters/de/vies-vat.json b/packages/backend/src/adapters/de/vies-vat.json index 3e507e5b..54ece533 100644 --- a/packages/backend/src/adapters/de/vies-vat.json +++ b/packages/backend/src/adapters/de/vies-vat.json @@ -8,6 +8,13 @@ "icon": "vies", "docsUrl": "https://ec.europa.eu/taxation_customs/vies/", "requiredEnvVars": [], + "probe": { + "tool": "vies_check_vat", + "params": { + "countryCode": "DE", + "vatNumber": "811907980" + } + }, "connector": { "name": "VIES VAT Validation", "type": "REST", @@ -30,7 +37,10 @@ "description": "VAT number without the country prefix (e.g. '123456789' for DE123456789)." } }, - "required": ["countryCode", "vatNumber"] + "required": [ + "countryCode", + "vatNumber" + ] }, "endpointMapping": { "method": "POST", diff --git a/packages/backend/src/adapters/intl/gtin-lookup.json b/packages/backend/src/adapters/intl/gtin-lookup.json index 4e98b22d..1fcdf041 100644 --- a/packages/backend/src/adapters/intl/gtin-lookup.json +++ b/packages/backend/src/adapters/intl/gtin-lookup.json @@ -8,6 +8,12 @@ "icon": "barcode", "docsUrl": "https://openfoodfacts.github.io/openfoodfacts-server/api/", "requiredEnvVars": [], + "probe": { + "tool": "gtin_lookup_food", + "params": { + "barcode": "3017620422003" + } + }, "connector": { "name": "GTIN / Barcode Lookup", "type": "REST", @@ -31,7 +37,9 @@ "description": "The product barcode (GTIN-13 / EAN-13 / UPC-A digits, keep leading zeros)." } }, - "required": ["barcode"] + "required": [ + "barcode" + ] }, "endpointMapping": { "method": "GET", @@ -57,7 +65,9 @@ "description": "Number of results to return (default 10, max 50)." } }, - "required": ["query"] + "required": [ + "query" + ] }, "endpointMapping": { "method": "GET", @@ -82,7 +92,9 @@ "description": "The product UPC-A (12 digits) or EAN/GTIN-13 (13 digits) barcode." } }, - "required": ["barcode"] + "required": [ + "barcode" + ] }, "endpointMapping": { "method": "GET", @@ -104,7 +116,9 @@ "description": "Keyword search, e.g. 'logitech mouse m100'." } }, - "required": ["query"] + "required": [ + "query" + ] }, "endpointMapping": { "method": "GET", @@ -126,7 +140,9 @@ "description": "The product barcode (GTIN-13 / EAN-13, keep leading zeros)." } }, - "required": ["barcode"] + "required": [ + "barcode" + ] }, "endpointMapping": { "method": "GET", @@ -148,7 +164,9 @@ "description": "The book's ISBN-13 (13 digits, the EAN/GTIN printed on the back cover)." } }, - "required": ["isbn"] + "required": [ + "isbn" + ] }, "endpointMapping": { "method": "GET", diff --git a/packages/backend/src/adapters/intl/idealista.json b/packages/backend/src/adapters/intl/idealista.json index 07d765ac..82e60af8 100644 --- a/packages/backend/src/adapters/intl/idealista.json +++ b/packages/backend/src/adapters/intl/idealista.json @@ -7,6 +7,7 @@ "category": "real-estate", "icon": "idealista", "docsUrl": "https://www.idealista.com", + "selfHostOnly": true, "featured": false, "priority": 75, "requiredEnvVars": [], diff --git a/packages/backend/src/adapters/intl/opentable.json b/packages/backend/src/adapters/intl/opentable.json index 40f48c1e..2a6fe01d 100644 --- a/packages/backend/src/adapters/intl/opentable.json +++ b/packages/backend/src/adapters/intl/opentable.json @@ -7,9 +7,20 @@ "category": "food", "icon": "opentable", "docsUrl": "https://www.opentable.com", + "selfHostOnly": true, "featured": true, "priority": 85, "requiredEnvVars": [], + "probe": { + "tool": "opentable_search_restaurants", + "params": { + "latitude": 51.5074, + "longitude": -0.1278, + "covers": 2, + "leadTime": 14, + "size": 5 + } + }, "connector": { "name": "OpenTable Mobile API", "type": "REST", diff --git a/packages/backend/src/adapters/intl/reddit.json b/packages/backend/src/adapters/intl/reddit.json index 732c7693..1bbbb677 100644 --- a/packages/backend/src/adapters/intl/reddit.json +++ b/packages/backend/src/adapters/intl/reddit.json @@ -7,7 +7,11 @@ "category": "social", "icon": "reddit", "docsUrl": "https://www.reddit.com/dev/api", - "requiredEnvVars": ["REDDIT_CLIENT_ID", "REDDIT_CLIENT_SECRET"], + "selfHostOnly": true, + "requiredEnvVars": [ + "REDDIT_CLIENT_ID", + "REDDIT_CLIENT_SECRET" + ], "connector": { "name": "Reddit API", "type": "REST", @@ -30,11 +34,19 @@ "parameters": { "type": "object", "properties": { - "subreddit": { "type": "string", "description": "Subreddit name without the 'r/' prefix, e.g. 'programming'." } + "subreddit": { + "type": "string", + "description": "Subreddit name without the 'r/' prefix, e.g. 'programming'." + } }, - "required": ["subreddit"] + "required": [ + "subreddit" + ] }, - "endpointMapping": { "method": "GET", "path": "/r/{subreddit}/about" } + "endpointMapping": { + "method": "GET", + "path": "/r/{subreddit}/about" + } }, { "name": "reddit_list_subreddit_posts", @@ -42,14 +54,34 @@ "parameters": { "type": "object", "properties": { - "subreddit": { "type": "string", "description": "Subreddit name." }, - "sort": { "type": "string", "description": "hot, new, top, rising, controversial." }, - "t": { "type": "string", "description": "Time scope for top/controversial: hour, day, week, month, year, all." }, - "limit": { "type": "integer", "description": "Per page (default 25, max 100)." }, - "after": { "type": "string", "description": "Cursor β€” fullname (t3_xxx) of the post to start after." }, - "before": { "type": "string", "description": "Cursor β€” fullname of the post to start before." } + "subreddit": { + "type": "string", + "description": "Subreddit name." + }, + "sort": { + "type": "string", + "description": "hot, new, top, rising, controversial." + }, + "t": { + "type": "string", + "description": "Time scope for top/controversial: hour, day, week, month, year, all." + }, + "limit": { + "type": "integer", + "description": "Per page (default 25, max 100)." + }, + "after": { + "type": "string", + "description": "Cursor β€” fullname (t3_xxx) of the post to start after." + }, + "before": { + "type": "string", + "description": "Cursor β€” fullname of the post to start before." + } }, - "required": ["subreddit"] + "required": [ + "subreddit" + ] }, "endpointMapping": { "method": "GET", @@ -68,16 +100,42 @@ "parameters": { "type": "object", "properties": { - "q": { "type": "string", "description": "Search query." }, - "subreddit": { "type": "string", "description": "Restrict to this subreddit." }, - "sort": { "type": "string", "description": "relevance, hot, top, new, comments." }, - "t": { "type": "string", "description": "Time scope: hour, day, week, month, year, all." }, - "limit": { "type": "integer", "description": "Per page." }, - "after": { "type": "string", "description": "Cursor." }, - "restrict_sr": { "type": "boolean", "description": "If true, restrict_sr=1 (only match in subreddit, used with subreddit param)." }, - "type": { "type": "string", "description": "sr (subreddits), user (users), link (posts only)." } + "q": { + "type": "string", + "description": "Search query." + }, + "subreddit": { + "type": "string", + "description": "Restrict to this subreddit." + }, + "sort": { + "type": "string", + "description": "relevance, hot, top, new, comments." + }, + "t": { + "type": "string", + "description": "Time scope: hour, day, week, month, year, all." + }, + "limit": { + "type": "integer", + "description": "Per page." + }, + "after": { + "type": "string", + "description": "Cursor." + }, + "restrict_sr": { + "type": "boolean", + "description": "If true, restrict_sr=1 (only match in subreddit, used with subreddit param)." + }, + "type": { + "type": "string", + "description": "sr (subreddits), user (users), link (posts only)." + } }, - "required": ["q"] + "required": [ + "q" + ] }, "endpointMapping": { "method": "GET", @@ -99,12 +157,26 @@ "parameters": { "type": "object", "properties": { - "postId": { "type": "string", "description": "Post id36 (without t3_ prefix)." }, - "limit": { "type": "integer", "description": "Max comments." }, - "depth": { "type": "integer", "description": "Max thread depth." }, - "sort": { "type": "string", "description": "best, top, new, controversial, old, qa." } + "postId": { + "type": "string", + "description": "Post id36 (without t3_ prefix)." + }, + "limit": { + "type": "integer", + "description": "Max comments." + }, + "depth": { + "type": "integer", + "description": "Max thread depth." + }, + "sort": { + "type": "string", + "description": "best, top, new, controversial, old, qa." + } }, - "required": ["postId"] + "required": [ + "postId" + ] }, "endpointMapping": { "method": "GET", @@ -122,11 +194,19 @@ "parameters": { "type": "object", "properties": { - "username": { "type": "string", "description": "Reddit username without u/." } + "username": { + "type": "string", + "description": "Reddit username without u/." + } }, - "required": ["username"] + "required": [ + "username" + ] }, - "endpointMapping": { "method": "GET", "path": "/user/{username}/about" } + "endpointMapping": { + "method": "GET", + "path": "/user/{username}/about" + } }, { "name": "reddit_get_user_posts", @@ -134,13 +214,30 @@ "parameters": { "type": "object", "properties": { - "username": { "type": "string", "description": "Username." }, - "sort": { "type": "string", "description": "new, hot, top, controversial." }, - "t": { "type": "string", "description": "Time scope." }, - "limit": { "type": "integer", "description": "Per page." }, - "after": { "type": "string", "description": "Cursor." } + "username": { + "type": "string", + "description": "Username." + }, + "sort": { + "type": "string", + "description": "new, hot, top, controversial." + }, + "t": { + "type": "string", + "description": "Time scope." + }, + "limit": { + "type": "integer", + "description": "Per page." + }, + "after": { + "type": "string", + "description": "Cursor." + } }, - "required": ["username"] + "required": [ + "username" + ] }, "endpointMapping": { "method": "GET", diff --git a/packages/backend/src/adapters/intl/sorare.json b/packages/backend/src/adapters/intl/sorare.json index ab6a9e59..e629744a 100644 --- a/packages/backend/src/adapters/intl/sorare.json +++ b/packages/backend/src/adapters/intl/sorare.json @@ -6,6 +6,7 @@ "category": "gaming", "icon": "sorare", "docsUrl": "https://github.com/sorare/api", + "selfHostOnly": true, "featured": true, "priority": 100, "requiredEnvVars": [ diff --git a/packages/backend/src/adapters/intl/trenitalia.json b/packages/backend/src/adapters/intl/trenitalia.json index 787ed136..5dc86da3 100644 --- a/packages/backend/src/adapters/intl/trenitalia.json +++ b/packages/backend/src/adapters/intl/trenitalia.json @@ -7,9 +7,21 @@ "category": "travel", "icon": "trenitalia", "docsUrl": "https://www.trenitalia.com", + "selfHostOnly": true, "featured": true, "priority": 78, "requiredEnvVars": [], + "probe": { + "tool": "trenitalia_search_trips", + "params": { + "from_name": "Milano Centrale", + "to_name": "Roma Termini", + "departure_date": "__TOMORROW__T08:00:00", + "adults": 1, + "children": 0, + "page": 1 + } + }, "connector": { "name": "Trenitalia Le Frecce Mobile BFF", "type": "REST", diff --git a/packages/backend/src/adapters/intl/untappd.json b/packages/backend/src/adapters/intl/untappd.json index e77f99b9..d6d6dd06 100644 --- a/packages/backend/src/adapters/intl/untappd.json +++ b/packages/backend/src/adapters/intl/untappd.json @@ -7,6 +7,7 @@ "category": "food", "icon": "untappd", "docsUrl": "https://untappd.com/api/docs", + "selfHostOnly": true, "requiredEnvVars": [ "UNTAPPD_CLIENT_ID", "UNTAPPD_CLIENT_SECRET" diff --git a/packages/backend/src/adapters/intl/vinted.json b/packages/backend/src/adapters/intl/vinted.json index fff3147e..93c786ae 100644 --- a/packages/backend/src/adapters/intl/vinted.json +++ b/packages/backend/src/adapters/intl/vinted.json @@ -7,6 +7,7 @@ "category": "e-commerce", "icon": "vinted", "docsUrl": "https://www.vinted.com", + "selfHostOnly": true, "featured": true, "priority": 80, "requiredEnvVars": [], diff --git a/packages/backend/src/adapters/probe.util.ts b/packages/backend/src/adapters/probe.util.ts new file mode 100644 index 00000000..e90ce7a5 --- /dev/null +++ b/packages/backend/src/adapters/probe.util.ts @@ -0,0 +1,62 @@ +import type { AdapterDefinition } from './catalog'; + +/** + * Which call proves a freshly imported connector works. + * + * Until now the first time anyone learned that a pasted token was wrong was + * when the agent said so β€” 63 of the 246 workspaces stuck at "attached, never + * called" had only ever seen upstream 401/403s from their own credentials. + * The probe runs the same engine the agent will, right after import, while + * the user is still on the form with the value in front of them. + * + * Shared with scripts/probe-keyless.mjs, which uses the same `probe` field to + * check keyless adapters from a datacenter IP in CI. Keep the two in step. + */ +export interface ProbeCall { + toolName: string; + params: Record; +} + +type ProbeTool = AdapterDefinition['tools'][number]; + +export function pickProbe(adapter: { + probe?: { tool: string; params?: Record }; + tools: ProbeTool[]; +}): ProbeCall | null { + if (adapter.probe?.tool) { + const tool = adapter.tools.find((t) => t.name === adapter.probe!.tool); + if (!tool) return null; + return { toolName: tool.name, params: materialise(adapter.probe.params ?? {}) }; + } + // No declared probe: the first GET with nothing required is the safest call + // there is. Defaults are passed so optional-but-defaulted params are sent. + const tool = adapter.tools.find((t) => { + const em = t.endpointMapping as { method?: string; path?: unknown }; + const params = t.parameters as { required?: string[] } | undefined; + return ( + String(em?.method ?? '').toUpperCase() === 'GET' && + typeof em?.path === 'string' && + (params?.required ?? []).length === 0 + ); + }); + if (!tool) return null; + const props = ((tool.parameters as { properties?: Record }) + ?.properties ?? {}) as Record; + const params: Record = {}; + for (const [k, p] of Object.entries(props)) { + if (p && p.default !== undefined) params[k] = p.default; + } + return { toolName: tool.name, params }; +} + +/** `__TOMORROW__` inside a string β†’ tomorrow as YYYY-MM-DD. */ +export function materialise( + params: Record, +): Record { + const tomorrow = new Date(Date.now() + 86_400_000).toISOString().slice(0, 10); + const out: Record = {}; + for (const [k, v] of Object.entries(params)) { + out[k] = typeof v === 'string' ? v.replaceAll('__TOMORROW__', tomorrow) : v; + } + return out; +} diff --git a/packages/backend/src/audit/audit.module.ts b/packages/backend/src/audit/audit.module.ts index 6b3ab0e8..0fbb29ae 100644 --- a/packages/backend/src/audit/audit.module.ts +++ b/packages/backend/src/audit/audit.module.ts @@ -2,11 +2,13 @@ import { Global, Module } from '@nestjs/common'; import { AuditService } from './audit.service'; import { AuditController } from './audit.controller'; import { SecurityEventService } from './security-event.service'; +import { ProductEventService } from './product-event.service'; +import { ProductEventController } from './product-event.controller'; @Global() @Module({ - controllers: [AuditController], - providers: [AuditService, SecurityEventService], - exports: [AuditService, SecurityEventService], + controllers: [AuditController, ProductEventController], + providers: [AuditService, SecurityEventService, ProductEventService], + exports: [AuditService, SecurityEventService, ProductEventService], }) export class AuditModule {} diff --git a/packages/backend/src/audit/product-event.controller.ts b/packages/backend/src/audit/product-event.controller.ts new file mode 100644 index 00000000..3d6e6840 --- /dev/null +++ b/packages/backend/src/audit/product-event.controller.ts @@ -0,0 +1,35 @@ +import { Body, Controller, HttpCode, Post, Req, UseGuards } from '@nestjs/common'; +import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { AuthGuard } from '@nestjs/passport'; +import { ProductEventService } from './product-event.service'; + +/** + * POST /api/product-events β€” the UI reports a funnel step. + * + * Always 204, even for an unknown event name: the page fires these and moves + * on, and a rejected beacon would only show up as console noise. Unknown names + * are dropped server-side; the allow-list lives in ProductEventService. + */ +@ApiTags('Product events') +@ApiBearerAuth() +@UseGuards(AuthGuard('jwt')) +@Controller('api/product-events') +export class ProductEventController { + constructor(private readonly events: ProductEventService) {} + + @Post() + @HttpCode(204) + @ApiOperation({ summary: 'Record a product-usage event for the signed-in user' }) + async record( + @Req() req: any, + @Body() body: { event?: unknown; metadata?: Record }, + ): Promise { + if (!this.events.isKnown(body?.event)) return; + await this.events.log({ + event: body.event, + userId: req.user?.sub ?? null, + organizationId: req.user?.organizationId ?? null, + metadata: body.metadata ?? null, + }); + } +} diff --git a/packages/backend/src/audit/product-event.service.ts b/packages/backend/src/audit/product-event.service.ts new file mode 100644 index 00000000..aaaa44a6 --- /dev/null +++ b/packages/backend/src/audit/product-event.service.ts @@ -0,0 +1,86 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaService } from '../common/prisma.service'; + +/** + * Product-usage events the UI reports so the activation funnel can be read + * step by step instead of from its two ends. + * + * The allow-list is the contract: a client cannot invent event names, and a + * name that is not here is dropped rather than stored. Add to it when a page + * gains a step worth measuring, and say in the comment what question the + * event answers. + */ +export const ProductEvents = { + /** Landed on the post-attach page (the one that shows the MCP endpoint). */ + POST_ATTACH_VIEWED: 'post_attach_viewed', + /** Copied the MCP endpoint URL. */ + MCP_URL_COPIED: 'mcp_url_copied', + /** Opened a client's Quick Connect instructions. metadata.client = which. */ + CLIENT_TAB_OPENED: 'client_tab_opened', + /** Copied a ready-made client config or command. metadata.client = which. */ + CLIENT_CONFIG_COPIED: 'client_config_copied', + /** Generated an MCP API key on the page. */ + API_KEY_GENERATED: 'api_key_generated', + /** Left the post-attach page having copied nothing at all. */ + LEFT_WITHOUT_COPY: 'left_page_without_copy', +} as const; + +export type ProductEventName = (typeof ProductEvents)[keyof typeof ProductEvents]; + +const ALLOWED = new Set(Object.values(ProductEvents)); +const MAX_METADATA_BYTES = 1024; + +@Injectable() +export class ProductEventService { + private readonly logger = new Logger(ProductEventService.name); + + constructor(private readonly prisma: PrismaService) {} + + isKnown(event: unknown): event is ProductEventName { + return typeof event === 'string' && ALLOWED.has(event); + } + + /** Best-effort and never throws: a lost event must not break the page. */ + async log(input: { + event: ProductEventName; + userId?: string | null; + organizationId?: string | null; + metadata?: Record | null; + }): Promise { + try { + const metadata = boundMetadata(input.metadata); + await this.prisma.productEvent.create({ + data: { + event: input.event, + userId: input.userId ?? null, + organizationId: input.organizationId ?? null, + metadata: metadata as any, + }, + }); + } catch (err: any) { + this.logger.warn(`product event ${input.event} not recorded: ${err?.message ?? err}`); + } + } +} + +/** + * Metadata keys a page may send. Anything else is dropped: the events carry + * a client name or a server id, nothing that should ever be a secret, and a + * fixed key set is what keeps an untrusted body from choosing property names. + */ +const METADATA_KEYS = ['client', 'serverId', 'connectorId', 'adapterSlug'] as const; + +function boundMetadata( + metadata: Record | null | undefined, +): Record | null { + if (!metadata || typeof metadata !== 'object') return null; + const entries: Array<[string, string | number | boolean]> = []; + for (const key of METADATA_KEYS) { + const v = metadata[key]; + if (typeof v === 'string') entries.push([key, v.slice(0, 200)]); + else if (typeof v === 'number' || typeof v === 'boolean') entries.push([key, v]); + } + if (entries.length === 0) return null; + const out = Object.fromEntries(entries); + return JSON.stringify(out).length > MAX_METADATA_BYTES ? null : out; +} diff --git a/packages/backend/src/common/db-rest.util.spec.ts b/packages/backend/src/common/db-rest.util.spec.ts deleted file mode 100644 index e5869be7..00000000 --- a/packages/backend/src/common/db-rest.util.spec.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { resolveInternalDbRestUrl, resolveDbRestProfile } from './db-rest.util'; - -describe('resolveInternalDbRestUrl', () => { - const PUBLIC = 'https://v6.db.transport.rest'; - const cloud = { DEPLOYMENT_MODE: 'cloud', DB_REST_INTERNAL_URL: 'http://db-rest:3000' }; - - it('swaps the public db-rest host for the internal one in cloud', () => { - expect(resolveInternalDbRestUrl(PUBLIC, cloud as any)).toBe('http://db-rest:3000'); - }); - - it('preserves any path and strips a trailing slash on the internal URL', () => { - expect( - resolveInternalDbRestUrl(`${PUBLIC}/locations`, { - DEPLOYMENT_MODE: 'cloud', - DB_REST_INTERNAL_URL: 'http://db-rest:3000/', - } as any), - ).toBe('http://db-rest:3000/locations'); - }); - - it('leaves the URL untouched when not in cloud (self-host)', () => { - expect( - resolveInternalDbRestUrl(PUBLIC, { DB_REST_INTERNAL_URL: 'http://db-rest:3000' } as any), - ).toBe(PUBLIC); - }); - - it('leaves the URL untouched when the internal URL is not configured', () => { - expect(resolveInternalDbRestUrl(PUBLIC, { DEPLOYMENT_MODE: 'cloud' } as any)).toBe(PUBLIC); - }); - - it('does not touch non-db-rest base URLs', () => { - expect(resolveInternalDbRestUrl('https://api.example.com', cloud as any)).toBe( - 'https://api.example.com', - ); - }); -}); - -describe('resolveDbRestProfile', () => { - const INTERNAL = 'http://db-rest:3000'; - const cloud = { DEPLOYMENT_MODE: 'cloud', DB_REST_INTERNAL_URL: INTERNAL }; - const q = { results: '1', profile: 'dbnav' }; - - it('swaps dbnav β†’ dbweb for internal db-rest requests in cloud', () => { - expect( - resolveDbRestProfile(`${INTERNAL}/locations`, { ...q }, cloud as any), - ).toEqual({ results: '1', profile: 'dbweb' }); - }); - - it('tolerates a trailing slash on the internal URL', () => { - expect( - resolveDbRestProfile(`${INTERNAL}/journeys`, { ...q }, { - DEPLOYMENT_MODE: 'cloud', - DB_REST_INTERNAL_URL: `${INTERNAL}/`, - } as any)?.profile, - ).toBe('dbweb'); - }); - - it('leaves the profile untouched on self-host (not cloud)', () => { - expect( - resolveDbRestProfile(`${INTERNAL}/locations`, { ...q }, { - DB_REST_INTERNAL_URL: INTERNAL, - } as any)?.profile, - ).toBe('dbnav'); - }); - - it('leaves the profile untouched when no internal db-rest is configured', () => { - expect( - resolveDbRestProfile('https://v6.db.transport.rest/locations', { ...q }, { - DEPLOYMENT_MODE: 'cloud', - } as any)?.profile, - ).toBe('dbnav'); - }); - - it('does not touch requests to a non-db-rest host, even in cloud', () => { - expect( - resolveDbRestProfile('https://api.example.com/x', { ...q }, cloud as any)?.profile, - ).toBe('dbnav'); - }); - - it('only rewrites the dbnav profile (leaves other profiles alone)', () => { - expect( - resolveDbRestProfile(`${INTERNAL}/locations`, { profile: 'dbris' }, cloud as any)?.profile, - ).toBe('dbris'); - }); - - it('is a no-op when there are no query params', () => { - expect(resolveDbRestProfile(`${INTERNAL}/locations`, undefined, cloud as any)).toBeUndefined(); - }); -}); diff --git a/packages/backend/src/common/db-rest.util.ts b/packages/backend/src/common/db-rest.util.ts deleted file mode 100644 index ae445844..00000000 --- a/packages/backend/src/common/db-rest.util.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Cloud-only: route the public db-rest base URL to our internal self-hosted - * instance. The shipped Deutsche Bahn connector stores the public base URL - * (`v6.db.transport.rest`) so self-hosters use it as-is; in cloud we swap the - * host to the internal db-rest (`DB_REST_INTERNAL_URL`) at request time. Pure - * host swap β€” same db-rest schema both sides, so paths/params/responses are - * unchanged. Returns the URL untouched on self-host (env unset / not cloud). - * - * Used by both the tool-execution path (DynamicMcpTools) and the connector - * "Test connection" / health-check path so they exercise the SAME endpoint. - */ -const PUBLIC_DB_REST = 'https://v6.db.transport.rest'; - -export function resolveInternalDbRestUrl( - baseUrl: string, - env: NodeJS.ProcessEnv = process.env, -): string { - const internal = env.DB_REST_INTERNAL_URL; - const isCloud = (env.DEPLOYMENT_MODE || '') === 'cloud'; - if (internal && isCloud && baseUrl.startsWith(PUBLIC_DB_REST)) { - return internal.replace(/\/$/, '') + baseUrl.slice(PUBLIC_DB_REST.length); - } - return baseUrl; -} - -/** - * Cloud-only: override the db-rest request profile from `dbnav` to `dbweb`. - * - * The shipped Deutsche Bahn connector pins `profile=dbnav` β€” the db-vendo-client - * profile that works from a normal (residential) self-host IP, so self-hosters - * keep using it verbatim. In AnythingMCP Cloud the internal db-rest egresses - * through the Zyte web-unblocker to defeat Deutsche Bahn's Akamai block of - * datacenter IPs; over that path DB's `dbnav` mobile endpoints reject the request - * (`Method Not Allowed` / `OPS_BLOCKED`) while the `dbweb` (bahn.de web API) - * endpoints work. So in cloud only β€” and only for a `dbnav` request actually - * bound for the internal db-rest β€” swap the profile to `dbweb`. - * - * Returns the query params untouched on self-host (env unset / not cloud), for - * non-db-rest targets, and for any non-`dbnav` profile, so it is safe to call - * unconditionally from the generic REST engine. - */ -export const CLOUD_DB_REST_PROFILE = 'dbweb'; - -export function resolveDbRestProfile( - requestUrl: string, - queryParams: Record | undefined, - env: NodeJS.ProcessEnv = process.env, -): Record | undefined { - const internal = env.DB_REST_INTERNAL_URL; - const isCloud = (env.DEPLOYMENT_MODE || '') === 'cloud'; - if (!queryParams || !isCloud || !internal) return queryParams; - if (queryParams.profile !== 'dbnav') return queryParams; - if (!requestUrl.startsWith(internal.replace(/\/$/, ''))) return queryParams; - return { ...queryParams, profile: CLOUD_DB_REST_PROFILE }; -} diff --git a/packages/backend/src/connectors/catalog-resync.service.ts b/packages/backend/src/connectors/catalog-resync.service.ts index eadad915..0827d69a 100644 --- a/packages/backend/src/connectors/catalog-resync.service.ts +++ b/packages/backend/src/connectors/catalog-resync.service.ts @@ -272,6 +272,15 @@ export class CatalogResyncService { hashContent(et.endpointMapping ?? {}); // In safe mode never touch endpointMapping. if (mode === 'safe' && endpointChanged) continue; + // A response mapping the operator wrote is theirs and stays. One + // the catalog ships is only applied where the tool has none β€” so an + // adapter that gains a mapping (Deutsche Bahn's move to MOTIS cut its + // boards from 12 KB to 2 KB this way) reaches installed connectors, + // while a tool the operator has already shaped is left alone. + const takeCatalogMapping = + mode === 'full' && + et.responseMapping == null && + ct.responseMapping != null; await tx.mcpTool.update({ where: { id: et.id }, data: { @@ -281,11 +290,14 @@ export class CatalogResyncService { mode === 'safe' ? (et.endpointMapping as any) : (ct.endpointMapping as any), + ...(takeCatalogMapping + ? { responseMapping: ct.responseMapping as any } + : {}), // Un-deprecate a tool the catalog brought back; never flip a // user's manual disable. deprecatedAt: null, isEnabled: et.deprecatedAt ? true : et.isEnabled, - // responseMapping / useProxy / roleAccess preserved. + // useProxy / roleAccess preserved. }, }); updatedCount++; diff --git a/packages/backend/src/connectors/connectors.service.ts b/packages/backend/src/connectors/connectors.service.ts index 9d0b51bf..16f72fdf 100644 --- a/packages/backend/src/connectors/connectors.service.ts +++ b/packages/backend/src/connectors/connectors.service.ts @@ -10,7 +10,6 @@ import { DatabaseEngine } from './engines/database.engine'; import { McpClientEngine } from './engines/mcp-client.engine'; import { encrypt, decrypt } from '../common/crypto/encryption.util'; import { getRequiredSecret } from '../common/secrets.util'; -import { resolveInternalDbRestUrl } from '../common/db-rest.util'; import { extractSsrfBlockedHostname } from '../common/ssrf.util'; import { normalizeConnectorBaseUrl } from '../common/url.util'; import { resolveAdapterIcon } from './connector-icon.util'; @@ -232,10 +231,7 @@ export class ConnectorsService { const path = connector.healthcheckPath || '/'; await this.restEngine.execute( { - // Apply the same cloud db-rest host swap as tool execution, so - // "Test connection" exercises the real (internal) endpoint - // instead of the public base URL stored on the connector. - baseUrl: resolveInternalDbRestUrl(connector.baseUrl), + baseUrl: connector.baseUrl, authType: connector.authType, authConfig, headers: connector.headers as Record, @@ -383,10 +379,7 @@ export class ConnectorsService { : undefined; const config = { - // Apply the cloud db-rest host swap so the in-app "Run Test" hits the - // real (internal) endpoint, same as MCP tool execution β€” otherwise it - // calls the public base URL and hangs/times out. - baseUrl: resolveInternalDbRestUrl(connector.baseUrl), + baseUrl: connector.baseUrl, authType: connector.authType, authConfig, headers: connector.headers as Record, diff --git a/packages/backend/src/connectors/engines/rest.engine.ts b/packages/backend/src/connectors/engines/rest.engine.ts index 6ac3cc2d..6574dfc6 100644 --- a/packages/backend/src/connectors/engines/rest.engine.ts +++ b/packages/backend/src/connectors/engines/rest.engine.ts @@ -7,7 +7,6 @@ import axios, { } from 'axios'; import FormData from 'form-data'; import { createUnblockerProxyAgent } from './unblocker-proxy-agent'; -import { resolveDbRestProfile } from '../../common/db-rest.util'; import { buildOAuth1Header } from './oauth1-signer'; import { OAuth2TokenService } from './oauth2-token.service'; import { @@ -15,6 +14,7 @@ import { LoginTokenAuthConfig, } from './login-token.service'; import { assertSafeOutboundUrl } from '../../common/ssrf.util'; +import { XMLParser } from 'fast-xml-parser'; import { pickExposedHeaders } from './response-headers.util'; /** @@ -81,7 +81,7 @@ export class RestEngine { params: Record, ): Promise<{ body: unknown; headers: Record }> { const withMeta = (response: AxiosResponse) => ({ - body: response.data, + body: parseXmlBody(response), headers: pickExposedHeaders( response.headers as Record, endpointMapping.exposeHeaders, @@ -160,13 +160,9 @@ export class RestEngine { mappedQuery[k] = v; } } - // Cloud-only Deutsche Bahn profile override (see resolveDbRestProfile): - // in cloud the internal db-rest egresses via the Zyte unblocker where DB's - // `dbnav` endpoints reject the request, so swap to `dbweb`. Strict no-op - // off-cloud and for every non-db-rest target. axiosConfig.params = { ...(axiosConfig.params as Record | undefined), - ...resolveDbRestProfile(url, mappedQuery), + ...mappedQuery, }; } @@ -875,3 +871,44 @@ const PROXY_ERROR_HINTS: Record = { '/limits/over-user-limit': 'the web-unblocker account is over its limit', }; + +/** + * Turn an XML response body into a plain object so response mapping and the + * MCP client get structured data rather than a string of markup. + * + * Deutsche Bahn's official Timetables API answers only in XML, and it is not + * alone among German public-sector APIs. Attributes are hoisted next to child + * elements without a prefix (`` β†’ `{ s: { id: "1", + * tl: { c: "ICE" } } }`), which is what a JMESPath mapping wants to address. + * Anything that is not declared as XML, or fails to parse, is returned as + * axios delivered it β€” a JSON API is never touched. + */ +const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '', + removeNSPrefix: true, + parseTagValue: false, + parseAttributeValue: false, + trimValues: true, +}); + +export function parseXmlBody(response: { + data: unknown; + headers?: Record; +}): unknown { + const { data } = response; + if (typeof data !== 'string') return data; + const contentType = String(response.headers?.['content-type'] ?? '').toLowerCase(); + const declaredXml = /(^|[/+])xml([;\s]|$)/.test(contentType); + if (!declaredXml) return data; + try { + const parsed = xmlParser.parse(data); + // fast-xml-parser hands back an empty object for non-XML text; keep the + // original so a mislabelled body is still visible to the caller. + return parsed && typeof parsed === 'object' && Object.keys(parsed).length > 0 + ? parsed + : data; + } catch { + return data; + } +} diff --git a/packages/backend/src/connectors/engines/xml-body.spec.ts b/packages/backend/src/connectors/engines/xml-body.spec.ts new file mode 100644 index 00000000..725b6419 --- /dev/null +++ b/packages/backend/src/connectors/engines/xml-body.spec.ts @@ -0,0 +1,48 @@ +import { parseXmlBody } from './rest.engine'; + +describe('parseXmlBody', () => { + it('leaves JSON responses exactly as axios delivered them', () => { + const data = { a: 1 }; + expect( + parseXmlBody({ data, headers: { 'content-type': 'application/json' } }), + ).toBe(data); + }); + + it('leaves a string body alone unless the response is declared XML', () => { + expect( + parseXmlBody({ data: '', headers: { 'content-type': 'text/plain' } }), + ).toBe(''); + expect(parseXmlBody({ data: '' })).toBe(''); + }); + + it('parses application/xml and text/xml, hoisting attributes without a prefix', () => { + // Shape of the Deutsche Bahn Timetables API: everything is an attribute. + const xml = + '' + + '' + + ''; + for (const ct of ['application/xml', 'text/xml; charset=utf-8']) { + const out = parseXmlBody({ data: xml, headers: { 'content-type': ct } }) as any; + expect(out.timetable.station).toBe('Freiburg(Breisgau) Hbf'); + expect(out.timetable.s.tl.c).toBe('ICE'); + expect(out.timetable.s.dp.ppth).toBe('Basel SBB|Bern'); + // Numeric-looking values stay strings: "2609151427" is a timestamp, not + // a number, and "0810" would lose its leading zero. + expect(out.timetable.s.dp.pt).toBe('2609151427'); + } + }); + + it('handles +xml media types and strips namespace prefixes', () => { + const out = parseXmlBody({ + data: '', + headers: { 'content-type': 'application/vnd.example+xml' }, + }) as any; + expect(out.root.item.v).toBe('1'); + }); + + it('returns the original text when the declared XML does not parse to anything', () => { + expect( + parseXmlBody({ data: 'not xml at all', headers: { 'content-type': 'application/xml' } }), + ).toBe('not xml at all'); + }); +}); diff --git a/packages/backend/src/ee/cloud/onboarding-cron.service.spec.ts b/packages/backend/src/ee/cloud/onboarding-cron.service.spec.ts index 36510079..79820555 100644 --- a/packages/backend/src/ee/cloud/onboarding-cron.service.spec.ts +++ b/packages/backend/src/ee/cloud/onboarding-cron.service.spec.ts @@ -18,7 +18,10 @@ describe('OnboardingCronService β€” activation pass', () => { user: { findMany, update }, // The trial-lifecycle pass runs after the activation pass; with no // trials it's a no-op. Stub just enough so it doesn't throw here. - license: { findMany: jest.fn().mockResolvedValue(overrides.trials ?? []) }, + license: { + findMany: jest.fn().mockResolvedValue(overrides.trials ?? []), + updateMany: jest.fn().mockResolvedValue({ count: 0 }), + }, } as any; const email = { sendOnboardingReminderEmail: jest.fn().mockResolvedValue(true), @@ -42,6 +45,7 @@ describe('OnboardingCronService β€” activation pass', () => { email: 'stuck@example.com', name: 'Sam', connectors: [{ id: 'conn123' }], + mcpServers: [], }, ], }); @@ -53,6 +57,7 @@ describe('OnboardingCronService β€” activation pass', () => { 'stuck@example.com', 'Sam', '/connectors/conn123', + 'test-connector', ); expect(update).toHaveBeenCalledWith( expect.objectContaining({ @@ -67,7 +72,7 @@ describe('OnboardingCronService β€” activation pass', () => { it('does not stamp when the email fails to send', async () => { const { service, update } = makeService({ stuckUsers: [ - { id: 'u2', email: 'x@example.com', name: null, connectors: [{ id: 'c2' }] }, + { id: 'u2', email: 'x@example.com', name: null, connectors: [{ id: 'c2' }], mcpServers: [] }, ], sendOk: false, }); @@ -81,7 +86,7 @@ describe('OnboardingCronService β€” activation pass', () => { it('falls back to /connectors when the user has no connector id resolved', async () => { const { service, email } = makeService({ stuckUsers: [ - { id: 'u3', email: 'y@example.com', name: 'Y', connectors: [] }, + { id: 'u3', email: 'y@example.com', name: 'Y', connectors: [], mcpServers: [] }, ], }); @@ -91,6 +96,70 @@ describe('OnboardingCronService β€” activation pass', () => { 'y@example.com', 'Y', '/connectors', + 'test-connector', + ); + }); + + it('sends users who already have an MCP server to its page, with the connect-client copy', async () => { + // 183 of the 246 stuck workspaces had attached a connector and never sent + // a request: the missing step is connecting a client, not testing a tool. + const { service, email } = makeService({ + stuckUsers: [ + { + id: 'u4', + email: 'z@example.com', + name: 'Zed', + connectors: [{ id: 'c4' }], + mcpServers: [{ id: 'srv4' }], + }, + ], + }); + + await service.run(); + + expect(email.sendActivationReminderEmail).toHaveBeenCalledWith( + 'z@example.com', + 'Zed', + '/mcp-server/srv4', + 'connect-client', ); }); }); + +describe('OnboardingCronService β€” trial status transition', () => { + it('marks active trials past expiresAt as expired, after the lifecycle pass', async () => { + const calls: string[] = []; + const prisma = { + user: { + findMany: jest.fn().mockResolvedValue([]), + update: jest.fn(), + }, + license: { + findMany: jest.fn().mockImplementation(async () => { + calls.push('lifecycle'); + return []; + }), + updateMany: jest.fn().mockImplementation(async () => { + calls.push('mark-expired'); + return { count: 3 }; + }), + }, + } as any; + const email = {} as any; + const { OnboardingCronService } = await import('./onboarding-cron.service'); + const svc = new OnboardingCronService(prisma, email); + + const out = await svc.run(); + + expect(out.trialsMarkedExpired).toBe(3); + const where = prisma.license.updateMany.mock.calls[0][0].where; + expect(where.plan).toBe('trial'); + expect(where.status).toBe('active'); + expect(where.expiresAt.lt).toBeInstanceOf(Date); + expect(prisma.license.updateMany.mock.calls[0][0].data).toEqual({ status: 'expired' }); + // The "your trial has ended" email selects on status='active', so the + // flip must come after it or the email would never be sent. + expect(calls).toEqual(['lifecycle', 'mark-expired']); + }); +}); + diff --git a/packages/backend/src/ee/cloud/onboarding-cron.service.ts b/packages/backend/src/ee/cloud/onboarding-cron.service.ts index dbb9c43d..5cc1a839 100644 --- a/packages/backend/src/ee/cloud/onboarding-cron.service.ts +++ b/packages/backend/src/ee/cloud/onboarding-cron.service.ts @@ -43,6 +43,7 @@ export class OnboardingCronService { trialWarn3: number; trialWarn1: number; trialExpired: number; + trialsMarkedExpired: number; skipped: number; }> { const now = Date.now(); @@ -54,6 +55,7 @@ export class OnboardingCronService { trialWarn3: 0, trialWarn1: 0, trialExpired: 0, + trialsMarkedExpired: 0, skipped: 0, }; @@ -158,12 +160,13 @@ export class OnboardingCronService { await this.runActivationPass(now, out); await this.runTrialLifecyclePass(now, out); + out.trialsMarkedExpired = await this.markExpiredTrials(now); this.logger.log( `Onboarding drip: examined=${out.examined} first=${out.firstReminders} ` + `second=${out.secondReminders} activation=${out.activationReminders} ` + `trialWarn3=${out.trialWarn3} trialWarn1=${out.trialWarn1} trialExpired=${out.trialExpired} ` + - `skipped=${out.skipped}`, + `trialsMarkedExpired=${out.trialsMarkedExpired} skipped=${out.skipped}`, ); return out; } @@ -253,6 +256,31 @@ export class OnboardingCronService { } } + /** + * Flip trials past their `expiresAt` to `status = 'expired'`. + * + * Access was never the issue: the licence guard checks `expiresAt` at + * request time, so an expired trial is blocked whether or not its status + * says so. The column was simply never transitioned, and by September 2026 + * 1,021 of 1,094 "active" trials had in fact ended β€” every funnel query, + * dashboard and export that grouped by status was wrong, and the count of + * live trials was overstated by more than an order of magnitude. Runs after + * the lifecycle pass, which selects on `status: 'active'`, so the "your + * trial has ended" email still goes out before the flip. + */ + private async markExpiredTrials(now: number): Promise { + const { count } = await this.prisma.license.updateMany({ + where: { + plan: 'trial', + status: 'active', + expiresAt: { not: null, lt: new Date(now) }, + }, + data: { status: 'expired' }, + }); + if (count > 0) this.logger.log(`Marked ${count} trial(s) as expired`); + return count; + } + /** * Activation pass β€” the cohort that builds a connector but never lands a * successful tool call (the biggest single drop-off). One email only, @@ -289,17 +317,32 @@ export class OnboardingCronService { orderBy: { createdAt: 'desc' }, take: 1, }, + mcpServers: { + select: { id: true }, + orderBy: { createdAt: 'desc' }, + take: 1, + }, }, }); for (const u of stuck) { out.examined++; + // Median attach β†’ first call is 12 minutes; a day of silence after + // attaching means no client was ever connected (183 of the 246 stuck + // workspaces never sent a request). Send those to the page that shows + // their endpoint and the client instructions, not to the tool tester. + const serverId = u.mcpServers[0]?.id; const connectorId = u.connectors[0]?.id; - const path = connectorId ? `/connectors/${connectorId}` : '/connectors'; + const path = serverId + ? `/mcp-server/${serverId}` + : connectorId + ? `/connectors/${connectorId}` + : '/connectors'; const ok = await this.email.sendActivationReminderEmail( u.email, u.name || 'there', path, + serverId ? 'connect-client' : 'test-connector', ); if (ok) { await this.prisma.user.update({ diff --git a/packages/backend/src/mcp-server/dynamic-mcp-tools.ts b/packages/backend/src/mcp-server/dynamic-mcp-tools.ts index 5c3719e0..668050d4 100644 --- a/packages/backend/src/mcp-server/dynamic-mcp-tools.ts +++ b/packages/backend/src/mcp-server/dynamic-mcp-tools.ts @@ -17,7 +17,6 @@ import { CALLER_CONTEXT_PREFIX, buildCallerContextVars, } from '../common/caller-context.util'; -import { resolveInternalDbRestUrl } from '../common/db-rest.util'; import { applyResponseTransform } from '../connectors/response-transform.util'; import { attachResponseMeta, @@ -97,15 +96,6 @@ export class DynamicMcpTools { return proxyUrl; } - /** - * Cloud-only db-rest host swap β€” see resolveInternalDbRestUrl. Kept as a thin - * method so the host swap stays consistent with the connector "Test - * connection" path, which uses the same shared util. - */ - private resolveInternalBaseUrl(baseUrl: string): string { - return resolveInternalDbRestUrl(baseUrl); - } - /** * Effective hourly proxy cap for a workspace: * organizations.proxy_rate_limit (DB, admin-only) ?? PROXY_RATE_LIMIT_DEFAULT @@ -275,7 +265,7 @@ export class DynamicMcpTools { usedProxy = proxyUrl != null; const engineConfig = { - baseUrl: this.resolveInternalBaseUrl(interpolatedConfig.baseUrl), + baseUrl: interpolatedConfig.baseUrl, authType: tool.connectorConfig.authType, authConfig: tool.connectorConfig.authConfig ? JSON.parse(tool.connectorConfig.authConfig) diff --git a/packages/backend/src/settings/email.service.ts b/packages/backend/src/settings/email.service.ts index 50b2edff..3e3a53f5 100644 --- a/packages/backend/src/settings/email.service.ts +++ b/packages/backend/src/settings/email.service.ts @@ -571,6 +571,7 @@ export class EmailService { to: string, name: string, connectorPath: string, + variant: 'connect-client' | 'test-connector' = 'test-connector', ): Promise { const transport = await this.createTransporter(); if (!transport) { @@ -585,8 +586,17 @@ export class EmailService { const connectorUrl = `${cloudUrl}${connectorPath}`; const unsubUrl = `${cloudUrl}/settings/profile`; - const subject = "You're one call away β€” finish setting up your connector"; - const body = `

Hi ${name},

+ const connectClient = variant === 'connect-client'; + const subject = connectClient + ? 'Your MCP server is ready β€” one paste connects Claude, Cursor or ChatGPT' + : "You're one call away β€” finish setting up your connector"; + const body = connectClient + ? `

Hi ${name},

+

Your connector is set up and sitting on an MCP server, but no client has talked to it yet. The last step is a copy and paste.

+

Open the server page, copy the endpoint, and pick your client under Quick Connect β€” Claude, Cursor, ChatGPT and Claude Code each have a two-line recipe there.

+

Connect your client β†’

+

Stuck? Reply to this email β€” we read every one.

` + : `

Hi ${name},

You created a connector in AnythingMCP but it hasn't made a successful call yet. That last step β€” running one tool β€” is where everything clicks.

Open your connector and hit Run test on any tool. If it returns an error, the message now tells you exactly what to fix (a missing API key, a wrong URL, etc.).

Test your connector β†’

@@ -607,7 +617,9 @@ export class EmailService {

`, - text: `Hi ${name},\n\nYou created a connector in AnythingMCP but it hasn't made a successful call yet. Open it and hit "Run test" on any tool β€” error messages now tell you exactly what to fix.\n\nTest your connector: ${connectorUrl}\n\nUnsubscribe: ${unsubUrl}`, + text: connectClient + ? `Hi ${name},\n\nYour connector is set up on an MCP server, but no client has talked to it yet. Open the server page, copy the endpoint and pick your client under Quick Connect.\n\nConnect your client: ${connectorUrl}\n\nUnsubscribe: ${unsubUrl}` + : `Hi ${name},\n\nYou created a connector in AnythingMCP but it hasn't made a successful call yet. Open it and hit "Run test" on any tool β€” error messages now tell you exactly what to fix.\n\nTest your connector: ${connectorUrl}\n\nUnsubscribe: ${unsubUrl}`, }); this.logger.log(`Activation-reminder email sent to ${to}`); return true; diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 4f378e09..f26bfb3b 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@anythingmcp/frontend", - "version": "0.8.1", + "version": "0.9.0", "description": "AnythingMCP β€” Next.js Admin UI", "private": true, "license": "AGPL-3.0-only", diff --git a/packages/frontend/src/app/connectors/store/page.tsx b/packages/frontend/src/app/connectors/store/page.tsx index 681e452e..1e8a4355 100644 --- a/packages/frontend/src/app/connectors/store/page.tsx +++ b/packages/frontend/src/app/connectors/store/page.tsx @@ -233,7 +233,7 @@ function AdapterStoreContent() { try { const adapter = list.find((a) => a.slug === slug); const result = await adapters.import(slug, token, credentials); - setMsg(result.message); + setMsg(describeImport(result.message, result.probe)); setImporting(null); // Show MCP assignment modal setImportedConnector({ id: result.connectorId, name: adapter?.name || slug }); @@ -667,6 +667,21 @@ function AdapterStoreContent() { ); } +/** + * One line for the banner: the import message, plus what the backend's + * test call found. A wrong token used to surface days later, from the agent; + * now it is on screen while the value is still in the form. + */ +function describeImport( + message: string, + probe: { ok: boolean; toolName: string; status?: number | null; message?: string } | null | undefined, +): string { + if (!probe) return message; + if (probe.ok) return `${message} Test call ${probe.toolName} succeeded β€” the connector works.`; + const status = probe.status ? ` (HTTP ${probe.status})` : ''; + return `${message} But the test call ${probe.toolName} failed${status}: ${probe.message ?? 'no details'}. Check the credentials in the connector editor.`; +} + /** Convert ENV_VAR_NAME to a human-readable label */ function formatEnvVarLabel(envVar: string): string { return envVar diff --git a/packages/frontend/src/app/mcp-server/[id]/page.tsx b/packages/frontend/src/app/mcp-server/[id]/page.tsx index 4589979e..3eadf96d 100644 --- a/packages/frontend/src/app/mcp-server/[id]/page.tsx +++ b/packages/frontend/src/app/mcp-server/[id]/page.tsx @@ -1,9 +1,9 @@ 'use client'; -import { useEffect, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { useParams, useRouter } from 'next/navigation'; import { useAuth } from '@/lib/auth-context'; -import { mcpServers, connectors as connectorsApi, mcpKeys } from '@/lib/api'; +import { mcpServers, connectors as connectorsApi, mcpKeys, productEvents } from '@/lib/api'; import { AppShell } from '@/components/app-shell'; import { Button, buttonVariants } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; @@ -37,6 +37,29 @@ export default function McpServerDetailPage() { const [copied, setCopied] = useState(''); const [connectClient, setConnectClient] = useState(null); + // Funnel instrumentation. This is the page that stands between "attached + // a connector" and "sent the first MCP request", and three quarters of the + // workspaces that reach it never send one. Record what people actually do + // here (copy the URL, open a client, generate a key, or leave without + // touching anything) so the next change to it is informed by data. + const copiedAnything = useRef(false); + useEffect(() => { + if (!token || !id) return; + productEvents.track('post_attach_viewed', token, { serverId: id }); + const onLeave = () => { + if (!copiedAnything.current) { + productEvents.track('left_page_without_copy', token, { serverId: id }); + // Fire once, whether pagehide or unmount gets there first. + copiedAnything.current = true; + } + }; + window.addEventListener('pagehide', onLeave); + return () => { + window.removeEventListener('pagehide', onLeave); + onLeave(); + }; + }, [token, id]); + useEffect(() => { if (!token || !id) return; Promise.all([ @@ -109,6 +132,7 @@ export default function McpServerDetailPage() { if (!token || !newKeyName.trim()) return; try { const result = await mcpKeys.generate(newKeyName.trim(), token, id); + productEvents.track('api_key_generated', token, { serverId: id }); setGeneratedKey(result.key); setNewKeyName(''); setKeyMsg('Key generated! Copy it now β€” it will not be shown again.'); @@ -179,6 +203,14 @@ export default function McpServerDetailPage() { if (ok) { setCopied(label); setTimeout(() => setCopied(''), 2000); + copiedAnything.current = true; + if (token) { + if (label === 'endpoint') { + productEvents.track('mcp_url_copied', token, { serverId: id }); + } else { + productEvents.track('client_config_copied', token, { serverId: id, client: label }); + } + } } }; @@ -586,7 +618,10 @@ export default function McpServerDetailPage() { {aiClients.map((client) => (