feat(rest): let a tool see the response headers it asks for (Link pagination, rate limits)

docs/tool-definition.md

@@ -73,6 +73,43 @@ The bridge configuration that transforms MCP tool calls into API requests.
 
 The `$` prefix means "take the value from the tool input parameter with this name."
 
+### Response headers and pagination (`exposeHeaders`)
+
+By default a tool receives the response **body** and nothing else. Some APIs put
+the one thing a model needs to continue in a header instead: GitHub, GitLab,
+Sentry and Shopify paginate with `Link: <...?cursor=xyz>; rel="next"`, and most
+APIs report rate limits in `X-RateLimit-*`. Without those, every list tool is
+exactly one page long.
+
+A REST tool can opt in per header name (case-insensitive):
+
+```json
+{
+  "method": "GET",
+  "path": "/organizations/{{SENTRY_ORG}}/issues/",
+  "queryParams": { "cursor": "$cursor", "query": "$query" },
+  "exposeHeaders": ["link", "x-ratelimit-remaining"]
+}
+```
+
+The selected headers are added to the tool result next to the body, and a
+`Link` header with `rel="next"` is parsed for you:
+
+```json
+{
+  "...the body as before...": "",
+  "_headers": { "link": "<https://sentry.io/api/0/...?cursor=1568:0:0>; rel=\"next\"", "x-ratelimit-remaining": "39" },
+  "_pagination": { "nextUrl": "https://sentry.io/api/0/...?cursor=1568:0:0", "nextCursor": "1568:0:0", "cursorParam": "cursor" }
+}
+```
+
+- `_pagination` is **absent on the last page**; tell the model so in the tool description ("call again with `cursor` = `_pagination.nextCursor` until it is missing").
+- `nextCursor` is recognised for the usual parameter names (`cursor`, `page`, `offset`, `after`, `page_token`, `starting_after`, ...); otherwise only `nextUrl` is set.
+- If the body is not a JSON object (an array, a string) it is wrapped as `data` so the extras have somewhere to live.
+- A response transform (`responseMapping.transform`) runs on the body first; the extras are attached afterwards, so a `select` cannot drop them.
+- The audit log keeps storing the bare body. Headers are cached together with it when `cacheTtl` is set.
+- REST connectors only. Tools that did not set `exposeHeaders` behave exactly as before.
+
 ### By Connector Type
 
 | Connector | method | path | queryParams | bodyMapping | headers |

packages/backend/src/connectors/connectors.service.ts

@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
 import { PrismaService } from '../common/prisma.service';
 import { Connector, ConnectorType, AuthType } from '../generated/prisma/client';
 import { RestEngine } from './engines/rest.engine';
+import { attachResponseMeta } from './engines/response-headers.util';
 import { SoapEngine } from './engines/soap.engine';
 import { GraphqlEngine } from './engines/graphql.engine';
 import { DatabaseEngine } from './engines/database.engine';
@@ -421,8 +422,24 @@ export class ConnectorsService {
     }
 
     switch (connector.type) {
-      case 'REST':
+      case 'REST': {
+        // The in-app "Run Test" must show what a model will see, so a tool
+        // that asked for response headers gets them here as well.
+        const wanted = (endpointMapping as { exposeHeaders?: string[] }).exposeHeaders;
+        if (Array.isArray(wanted) && wanted.length > 0) {
+          const out = await this.restEngine.executeWithMeta(
+            config,
+            endpointMapping,
+            mergedParams,
+          );
+          return attachResponseMeta(
+            out.body,
+            { headers: out.headers },
+            endpointMapping.queryParams,
+          );
+        }
         return this.restEngine.execute(config, endpointMapping, mergedParams);
+      }
       case 'SOAP':
         return this.soapEngine.execute(config, endpointMapping, mergedParams);
       case 'GRAPHQL':

packages/backend/src/connectors/engines/response-headers.util.spec.ts

@@ -0,0 +1,79 @@
+import {
+  describePagination,
+  parseLinkHeader,
+  pickExposedHeaders,
+} from './response-headers.util';
+
+describe('pickExposedHeaders', () => {
+  it('returns only the asked-for headers, lower-cased, regardless of the wire casing', () => {
+    const out = pickExposedHeaders(
+      { 'X-RateLimit-Remaining': '39', Link: '<u>; rel="next"', 'set-cookie': 'nope' },
+      ['link', 'x-ratelimit-remaining'],
+    );
+    expect(out).toEqual({ link: '<u>; rel="next"', 'x-ratelimit-remaining': '39' });
+  });
+
+  it('joins multi-valued headers and returns nothing when no tool asked', () => {
+    expect(pickExposedHeaders({ link: ['a', 'b'] }, ['LINK'])).toEqual({ link: 'a, b' });
+    expect(pickExposedHeaders({ link: 'x' }, undefined)).toEqual({});
+    expect(pickExposedHeaders(undefined, ['link'])).toEqual({});
+  });
+});
+
+describe('parseLinkHeader', () => {
+  it('parses the GitHub / Sentry shape', () => {
+    const rels = parseLinkHeader(
+      '<https://api.example.com/issues/?cursor=100:1:0>; rel="previous"; results="false", ' +
+        '<https://api.example.com/issues/?cursor=100:0:1>; rel="next"; results="true"',
+    );
+    expect(rels.next).toBe('https://api.example.com/issues/?cursor=100:0:1');
+    expect(rels.previous).toBe('https://api.example.com/issues/?cursor=100:1:0');
+  });
+
+  it('accepts unquoted rel and a rel listing several tokens', () => {
+    expect(parseLinkHeader('<https://x/a?page=3>; rel=next last')).toEqual({
+      next: 'https://x/a?page=3',
+      last: 'https://x/a?page=3',
+    });
+  });
+});
+
+describe('describePagination', () => {
+  it('lifts the cursor out of the next link', () => {
+    const p = describePagination({
+      link: '<https://sentry.io/api/0/organizations/o/issues/?cursor=1568%3A0%3A0&query=is%3Aunresolved>; rel="next"',
+    });
+    expect(p).toEqual({
+      nextUrl:
+        'https://sentry.io/api/0/organizations/o/issues/?cursor=1568%3A0%3A0&query=is%3Aunresolved',
+      nextCursor: '1568:0:0',
+      cursorParam: 'cursor',
+    });
+  });
+
+  it('recognises page numbers and keeps the previous page when announced', () => {
+    const p = describePagination({
+      link: '<https://x/repos?page=1>; rel="prev", <https://x/repos?page=3>; rel="next"',
+    });
+    expect(p?.nextCursor).toBe('3');
+    expect(p?.cursorParam).toBe('page');
+    expect(p?.prevUrl).toBe('https://x/repos?page=1');
+  });
+
+  it('is absent on the last page, and absent without a Link header', () => {
+    expect(describePagination({ link: '<https://x/repos?page=1>; rel="first"' })).toBeUndefined();
+    expect(describePagination({})).toBeUndefined();
+  });
+
+  it('prefers the parameter the tool maps when the link carries several (GitHub: after + page)', () => {
+    const link =
+      '<https://api.github.com/repositories/1/issues?per_page=2&after=Y3Vyc29y&page=2>; rel="next"';
+    expect(describePagination({ link }, ['page'])).toMatchObject({ nextCursor: '2', cursorParam: 'page' });
+    expect(describePagination({ link })).toMatchObject({ nextCursor: 'Y3Vyc29y', cursorParam: 'after' });
+  });
+
+  it('still returns nextUrl when the URL has no recognisable cursor', () => {
+    const p = describePagination({ link: '<https://x/feed/abc123>; rel="next"' });
+    expect(p).toEqual({ nextUrl: 'https://x/feed/abc123' });
+  });
+});

packages/backend/src/connectors/engines/response-headers.util.ts

@@ -0,0 +1,144 @@
+/**
+ * The few response headers a tool may ask to see.
+ *
+ * A REST tool normally gets the body and nothing else, which is right for
+ * almost every call and wrong for exactly one kind: list endpoints that
+ * paginate through a `Link` header (GitHub, GitLab, Sentry, Shopify, ...).
+ * The model can pass a `cursor` in, but never learns the next one, so every
+ * such tool is one page long. Rate-limit headers are the other honest use.
+ *
+ * A tool opts in with `endpointMapping.exposeHeaders: ["link", ...]`. Nothing
+ * here runs for a tool that did not ask.
+ */
+
+/** Header names an adapter may ask for, matched case-insensitively. */
+export function pickExposedHeaders(
+  headers: Record<string, unknown> | undefined,
+  names: string[] | undefined,
+): Record<string, string> {
+  const picked: Record<string, string> = {};
+  if (!headers || !names?.length) return picked;
+  const wanted = new Set(names.map((n) => n.toLowerCase()));
+  for (const [key, value] of Object.entries(headers)) {
+    const name = key.toLowerCase();
+    if (!wanted.has(name) || value === undefined || value === null) continue;
+    picked[name] = Array.isArray(value)
+      ? value.map(String).join(', ')
+      : String(value);
+  }
+  return picked;
+}
+
+/** RFC 8288 `Link` header → { rel: url }. Tolerant of the usual sloppiness. */
+export function parseLinkHeader(value: string): Record<string, string> {
+  const rels: Record<string, string> = {};
+  for (const part of value.split(',')) {
+    const m = part.match(/<\s*([^>]*)\s*>\s*;([^]*)/);
+    if (!m) continue;
+    const url = m[1].trim();
+    const rel = m[2].match(/\brel\s*=\s*"?([^";]+)"?/i)?.[1]?.trim();
+    if (!rel) continue;
+    // A single rel attribute may list several tokens: rel="next last".
+    for (const token of rel.split(/\s+/)) {
+      if (token && !(token in rels)) rels[token] = url;
+    }
+  }
+  return rels;
+}
+
+/** Query parameters that, in practice, carry the "where to continue" value. */
+const CURSOR_PARAMS = [
+  'cursor',
+  'page_token',
+  'pageToken',
+  'starting_after',
+  'after',
+  'offset',
+  'page',
+  'page_info',
+  'continuation',
+];
+
+export interface Pagination {
+  /** The full URL of the next page, exactly as the API sent it. */
+  nextUrl: string;
+  /** The value to pass back as the tool's cursor parameter, when recognisable. */
+  nextCursor?: string;
+  /** Which query parameter that value belongs to (`cursor`, `page`, ...). */
+  cursorParam?: string;
+  /** Present when the API also announced a previous page. */
+  prevUrl?: string;
+}
+
+/**
+ * What a model needs to fetch the next page, lifted out of `Link`. Returns
+ * undefined when there is no `next` relation — that is the "last page"
+ * signal, and it should read as absence, not as an empty object.
+ */
+export function describePagination(
+  headers: Record<string, string>,
+  preferredParams: string[] = [],
+): Pagination | undefined {
+  const link = headers['link'];
+  if (!link) return undefined;
+  const rels = parseLinkHeader(link);
+  if (!rels.next) return undefined;
+
+  const page: Pagination = { nextUrl: rels.next };
+  if (rels.prev) page.prevUrl = rels.prev;
+  try {
+    const params = new URL(rels.next).searchParams;
+    // A next link can carry more than one candidate (GitHub sends both
+    // `after=` and `page=`). The parameter the tool actually maps wins, so
+    // the model can feed the value straight back; the generic list is the
+    // fallback for tools that map none of them.
+    for (const name of [...preferredParams, ...CURSOR_PARAMS]) {
+      const v = params.get(name);
+      if (v !== null && v !== '') {
+        page.nextCursor = v;
+        page.cursorParam = name;
+        break;
+      }
+    }
+  } catch {
+    // Relative or malformed URL: nextUrl is still useful on its own.
+  }
+  return page;
+}
+
+/** What the engine hands back besides the body, when a tool asked for it. */
+export interface ResponseMeta {
+  headers: Record<string, string>;
+}
+
+/**
+ * Puts the exposed headers, and the pagination read out of `Link`, next to
+ * the body the model already gets. An object body is extended in place;
+ * anything else (an array, a string) is wrapped as `data` so the extras have
+ * somewhere to live. `_pagination` is absent on the last page on purpose:
+ * absence is the signal.
+ */
+export function attachResponseMeta(
+  value: unknown,
+  meta: ResponseMeta,
+  queryParams?: Record<string, unknown>,
+): unknown {
+  const extras: { _headers: Record<string, string>; _pagination?: Pagination } = {
+    _headers: meta.headers,
+  };
+  const pagination = describePagination(meta.headers, mappedQueryParams(queryParams));
+  if (pagination) extras._pagination = pagination;
+
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    return { ...(value as Record<string, unknown>), ...extras };
+  }
+  return { data: value, ...extras };
+}
+
+/** Query parameter names a tool feeds from its own inputs (`page: "$page"`). */
+function mappedQueryParams(queryParams?: Record<string, unknown>): string[] {
+  if (!queryParams) return [];
+  return Object.entries(queryParams)
+    .filter(([, v]) => typeof v === 'string' && (v as string).startsWith('$'))
+    .map(([k]) => k);
+}

packages/backend/src/connectors/engines/rest.engine.spec.ts

@@ -59,6 +59,39 @@ describe('RestEngine', () => {
     );
   });
 
+  it('hands back only the response headers the mapping asked for, lower-cased', async () => {
+    mockedAxios.mockResolvedValue({
+      data: [{ id: 1 }],
+      headers: {
+        Link: '<https://api.example.com/items?cursor=n>; rel="next"',
+        'X-RateLimit-Remaining': '9',
+        'Set-Cookie': 'secret=1',
+      },
+    });
+
+    const out = await engine.executeWithMeta(
+      { baseUrl: 'https://api.example.com', authType: 'NONE' },
+      { method: 'GET', path: '/items', exposeHeaders: ['link', 'x-ratelimit-remaining'] },
+      {},
+    );
+
+    expect(out.body).toEqual([{ id: 1 }]);
+    expect(out.headers).toEqual({
+      link: '<https://api.example.com/items?cursor=n>; rel="next"',
+      'x-ratelimit-remaining': '9',
+    });
+  });
+
+  it('returns no headers at all when the mapping did not opt in', async () => {
+    mockedAxios.mockResolvedValue({ data: {}, headers: { Link: '<u>; rel="next"' } });
+    const out = await engine.executeWithMeta(
+      { baseUrl: 'https://api.example.com', authType: 'NONE' },
+      { method: 'GET', path: '/items' },
+      {},
+    );
+    expect(out.headers).toEqual({});
+  });
+
   it('expands __rawquery into flat query params with dynamic keys (weclapp filter)', async () => {
     mockedAxios.mockResolvedValue({ data: {} });