From 6e08c679555b3d7e742e805d0df55c3f106e87cd Mon Sep 17 00:00:00 2001 From: Thor Haaland Date: Sat, 3 Oct 2026 23:24:58 +0000 Subject: [PATCH] feat(egress): optional per-host HTTPS passthrough route allowlist (OPT-1171) The signed network policy is host-only: a host with httpsPassthrough accepts any method and path, so a Read Only code run can send any request to any Console route (for example POST /api/auth/requestPasswordReset). A host may now list httpsPassthroughRoutes: exact {method, path} pairs. When listed, the gateway's passthrough validation (openHttpsPassthrough, with the envelope's method) allows only a listed method on a listed, already-normalized path with no query, and refuses anything else as HOST_NOT_ALLOWED before connecting. Paths must be canonical (no query, fragment, percent-encoding, backslash or dot segments). Routes are part of the snapshot, so the signed digest covers them, and the deployment policy caps them: listed on both sides, every signed route must be in the deployment's; listed on one side, that side applies. The field is optional and absent from today's policies, which serialize, digest and intersect exactly as before. --- service/src/egress-gateway.test.ts | 2 +- .../src/external-fetch-boundary.fixture.ts | 45 +++++ service/src/external-fetch-policy.test.ts | 187 ++++++++++++++++++ service/src/external-fetch-policy.ts | 136 +++++++++++++ service/src/external-fetch.ts | 2 +- 5 files changed, 370 insertions(+), 2 deletions(-) diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 0b72da30..34d03805 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -2090,7 +2090,7 @@ describe('egress gateway routes', () => { }); } else { setHttpsPassthroughOpenForTest(async args => { - const target = validateHttpsPassthroughUrl('https://allowed.test/pkg.tgz', args.policy); + const target = validateHttpsPassthroughUrl('https://allowed.test/pkg.tgz', args.policy, args.method); await args.beforeRequest?.(target); return { response, diff --git a/service/src/external-fetch-boundary.fixture.ts b/service/src/external-fetch-boundary.fixture.ts index 25c40625..8ce4ebc2 100644 --- a/service/src/external-fetch-boundary.fixture.ts +++ b/service/src/external-fetch-boundary.fixture.ts @@ -299,6 +299,51 @@ async function main(): Promise { ); assert.equal(requests.length, passthroughRequestCount + 1); + // A host that lists routes: only the listed method and path reach the origin; any other + // request is refused before a connection is made. + const routed = parseExternalFetchPolicy({ + version: 1, + limits: policy().limits, + hosts: { + [HOST]: { + httpsPassthrough: true, + httpsPassthroughRoutes: [{ method: 'POST', path: '/passthrough' }], + }, + }, + }); + const routedRequestCount = requests.length; + const allowed = await openHttpsPassthrough({ + url: `https://${HOST}/passthrough`, + policy: routed, + resolver: dns.resolver, + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: Buffer.from('{}'), + }); + for await (const _chunk of allowed.response) { + // drain + } + allowed.close(); + assert.equal(allowed.response.statusCode, 201); + for (const [url, method] of [ + [`https://${HOST}/passthrough`, 'GET'], + [`https://${HOST}/success`, 'POST'], + [`https://${HOST}/passthrough?x=1`, 'POST'], + ] as const) { + await expectCode( + () => openHttpsPassthrough({ + url, + policy: routed, + resolver: dns.resolver, + method, + headers: {}, + body: Buffer.alloc(0), + }), + 'HOST_NOT_ALLOWED', + ); + } + assert.equal(requests.length, routedRequestCount + 1); + const disconnectOpened = await openExternalFetch({ url: `https://${HOST}/success`, policy: policy(), diff --git a/service/src/external-fetch-policy.test.ts b/service/src/external-fetch-policy.test.ts index 99d7df9e..fe817958 100644 --- a/service/src/external-fetch-policy.test.ts +++ b/service/src/external-fetch-policy.test.ts @@ -1,8 +1,11 @@ +import crypto from 'node:crypto'; import { describe, expect, test } from 'bun:test'; import { HARD_EXTERNAL_FETCH_LIMITS, HARD_MAX_EXTERNAL_FETCH_HOSTS, + effectiveExternalFetchPolicy, externalFetchPolicyDigest, + intersectExternalFetchPolicies, parseExternalFetchPolicy, serializeExternalFetchPolicy, validateExternalFetchUrl, @@ -81,6 +84,7 @@ describe('external fetch policy parser', () => { validateHttpsPassthroughUrl( `https://${consoleHost}/api/optale/mcp`, policy, + 'POST', ).host, ).toBe(consoleHost); }); @@ -465,3 +469,186 @@ describe('external fetch address validation', () => { ); }); }); + +const CONSOLE_HOST = 'console.optale.com'; +const PASSTHROUGH_LIMITS = { ...HARD_EXTERNAL_FETCH_LIMITS, maxFetchesPerGrant: 8 }; + +function consolePolicy(routes?: unknown): unknown { + return { + version: 1, + limits: { ...HARD_EXTERNAL_FETCH_LIMITS }, + hosts: { + [CONSOLE_HOST]: { + httpsPassthrough: true, + ...(routes === undefined ? {} : { httpsPassthroughRoutes: routes }), + }, + }, + }; +} + +const READ_ONLY_ROUTES = [ + { method: 'POST', path: '/api/optale/mcp' }, + { method: 'GET', path: '/api/optale/composio/catalog/OPTALE_CORE/actions' }, +]; + +describe('HTTPS passthrough routes', () => { + test('parses routes into one canonical order that the digest covers', () => { + const listed = parseExternalFetchPolicy(consolePolicy(READ_ONLY_ROUTES)); + const reversed = parseExternalFetchPolicy(consolePolicy([...READ_ONLY_ROUTES].reverse())); + const unrouted = parseExternalFetchPolicy(consolePolicy()); + + expect(serializeExternalFetchPolicy(listed).hosts[CONSOLE_HOST]?.httpsPassthroughRoutes).toEqual([ + { method: 'GET', path: '/api/optale/composio/catalog/OPTALE_CORE/actions' }, + { method: 'POST', path: '/api/optale/mcp' }, + ]); + expect(externalFetchPolicyDigest(reversed)).toBe(externalFetchPolicyDigest(listed)); + expect(externalFetchPolicyDigest(unrouted)).not.toBe(externalFetchPolicyDigest(listed)); + }); + + test.each([ + ['routes without HTTPS passthrough', { [CONSOLE_HOST]: { contentTypes: ['application/pdf'], httpsPassthroughRoutes: READ_ONLY_ROUTES } }], + ['an empty route list', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [] } }], + ['an unknown method', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'TRACE', path: '/x' }] } }], + ['a lowercase method', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'post', path: '/x' }] } }], + ['a query', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'GET', path: '/x?y=1' }] } }], + ['percent-encoding', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'GET', path: '/api/optale/%6dcp' }] } }], + ['dot segments', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'GET', path: '/api/x/../mcp' }] } }], + ['a relative path', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ method: 'GET', path: 'api/optale/mcp' }] } }], + ['a duplicate route', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [READ_ONLY_ROUTES[0], READ_ONLY_ROUTES[0]] } }], + ['an extra route key', { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughRoutes: [{ ...READ_ONLY_ROUTES[0], query: true }] } }], + ])('rejects %s', (_label, hosts) => { + expect(() => + parseExternalFetchPolicy({ version: 1, limits: { ...HARD_EXTERNAL_FETCH_LIMITS }, hosts }), + ).toThrow(); + }); + + test('lets a routed host answer only its listed method and path, without a query', () => { + const policy = parseExternalFetchPolicy(consolePolicy(READ_ONLY_ROUTES)); + const mcp = `https://${CONSOLE_HOST}/api/optale/mcp`; + + expect(validateHttpsPassthroughUrl(mcp, policy, 'POST').url.pathname).toBe('/api/optale/mcp'); + expect(validateHttpsPassthroughUrl(mcp, policy, 'post').host).toBe(CONSOLE_HOST); + expect( + validateHttpsPassthroughUrl(`https://${CONSOLE_HOST}/api/optale/x/../mcp`, policy, 'POST').url + .pathname, + ).toBe('/api/optale/mcp'); + for (const [url, method] of [ + [mcp, 'GET'], + [mcp, 'DELETE'], + [`${mcp}?debug=1`, 'POST'], + [`${mcp}/`, 'POST'], + [`https://${CONSOLE_HOST}/API/optale/mcp`, 'POST'], + [`https://${CONSOLE_HOST}/api/optale/%6dcp`, 'POST'], + [`https://${CONSOLE_HOST}/api/auth/requestPasswordReset`, 'POST'], + [`https://${CONSOLE_HOST}/api/optale/composio/catalog/OPTALE_CORE/actions`, 'POST'], + ] as const) { + expectCode(() => validateHttpsPassthroughUrl(url, policy, method), 'HOST_NOT_ALLOWED'); + } + }); + + test('caps signed routes at the deployment routes, and applies deployment routes to an unrouted signature', () => { + const deployment = parseExternalFetchPolicy(consolePolicy(READ_ONLY_ROUTES)); + const subset = parseExternalFetchPolicy(consolePolicy([READ_ONLY_ROUTES[0]])); + const outside = parseExternalFetchPolicy( + consolePolicy([{ method: 'POST', path: '/api/auth/requestPasswordReset' }]), + ); + const unrouted = parseExternalFetchPolicy(consolePolicy()); + + expect( + intersectExternalFetchPolicies(subset, deployment).hosts.get(CONSOLE_HOST) + ?.httpsPassthroughRoutes, + ).toEqual([{ method: 'POST', path: '/api/optale/mcp' }]); + expectCode(() => intersectExternalFetchPolicies(outside, deployment), 'HOST_NOT_ALLOWED'); + const capped = intersectExternalFetchPolicies(unrouted, deployment); + expectCode( + () => + validateHttpsPassthroughUrl( + `https://${CONSOLE_HOST}/api/auth/requestPasswordReset`, + capped, + 'POST', + ), + 'HOST_NOT_ALLOWED', + ); + }); +}); + +/** + * The Console signs `sha256(JSON.stringify())` (packages/api + * canonicalizeCodeApiNetworkPolicy). Snapshots it signs today carry no routes; this engine must + * keep verifying them and keep their effective policy unchanged. + */ +function consoleDigest(snapshot: unknown): string { + const sort = (value: unknown): unknown => + Array.isArray(value) + ? value.map(sort) + : value !== null && typeof value === 'object' + ? Object.fromEntries( + Object.keys(value as Record) + .sort() + .map(key => [key, sort((value as Record)[key])]), + ) + : value; + return crypto.createHash('sha256').update(JSON.stringify(sort(snapshot)), 'utf8').digest('base64url'); +} + +const PRODUCTION_SHAPED_DEPLOYMENT = parseExternalFetchPolicy({ + version: 1, + limits: { + maxRedirects: 3, + maxResponseBytes: 26_214_400, + maxAggregateBytesPerGrant: 52_428_800, + maxFetchesPerGrant: 8, + connectTimeoutMs: 3_000, + headersTimeoutMs: 5_000, + totalTimeoutMs: 15_000, + }, + hosts: { + [FROZEN_HOST]: { contentTypes: ['application/pdf'] }, + ...Object.fromEntries( + ['console-staging.optale.com', CONSOLE_HOST, 'figent.optale.com', 'console-lab-callback.optale.com'].map( + host => [host, { httpsPassthrough: true, httpsPassthroughTotalTimeoutMs: 300_000, limits: { maxFetchesPerGrant: 8, maxResponseBytes: 2_097_152 } }], + ), + ), + 'pypi.org': { packageTransport: true }, + 'files.pythonhosted.org': { packageTransport: true }, + 'registry.npmjs.org': { packageTransport: true }, + }, +}); + +describe('compatibility with the policies the Console signs today', () => { + test('a Read Only snapshot (Console passthrough only) verifies and still reaches every Console route', () => { + const snapshot = { + version: 1, + limits: PRODUCTION_SHAPED_DEPLOYMENT.limits, + hosts: { + [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughTotalTimeoutMs: 300_000, limits: PASSTHROUGH_LIMITS }, + }, + }; + const effective = effectiveExternalFetchPolicy( + snapshot as never, + consoleDigest(snapshot), + PRODUCTION_SHAPED_DEPLOYMENT, + ); + + expect(effective.hosts.get(CONSOLE_HOST)?.httpsPassthroughRoutes).toBeUndefined(); + expect( + validateHttpsPassthroughUrl(`https://${CONSOLE_HOST}/api/auth/requestPasswordReset`, effective, 'POST').host, + ).toBe(CONSOLE_HOST); + }); + + test('an Auto snapshot with package hosts verifies with the digest the Console computes', () => { + const snapshot = { + version: 1, + limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits }, + hosts: { + 'files.pythonhosted.org': { packageTransport: true, limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits } }, + 'pypi.org': { packageTransport: true, limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits } }, + [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughTotalTimeoutMs: 300_000, limits: PASSTHROUGH_LIMITS }, + }, + }; + + expect(externalFetchPolicyDigest(parseExternalFetchPolicy(snapshot))).toBe(consoleDigest(snapshot)); + const effective = effectiveExternalFetchPolicy(snapshot as never, consoleDigest(snapshot), PRODUCTION_SHAPED_DEPLOYMENT); + expect([...effective.hosts.keys()].sort()).toEqual([CONSOLE_HOST, 'files.pythonhosted.org', 'pypi.org']); + }); +}); diff --git a/service/src/external-fetch-policy.ts b/service/src/external-fetch-policy.ts index 66a2b4fb..c55e82df 100644 --- a/service/src/external-fetch-policy.ts +++ b/service/src/external-fetch-policy.ts @@ -13,10 +13,31 @@ export interface ExternalFetchLimits { totalTimeoutMs: number; } +export const HTTPS_PASSTHROUGH_METHODS = [ + 'DELETE', + 'GET', + 'HEAD', + 'PATCH', + 'POST', + 'PUT', +] as const; +export type HttpsPassthroughMethod = (typeof HTTPS_PASSTHROUGH_METHODS)[number]; +export const HARD_MAX_HTTPS_PASSTHROUGH_ROUTES = 64; + +/** + * One HTTPS passthrough request a host accepts: an exact method and an exact, already + * normalized path, with no query string. A host without routes accepts any passthrough request. + */ +export interface HttpsPassthroughRoute { + method: HttpsPassthroughMethod; + path: string; +} + export interface ExternalFetchHostPolicy { contentTypes: Set; httpsPassthrough: boolean; httpsPassthroughTotalTimeoutMs?: number; + httpsPassthroughRoutes?: HttpsPassthroughRoute[]; packageTransport: boolean; limits: ExternalFetchLimits; } @@ -25,6 +46,7 @@ export interface ExternalFetchPolicySnapshotHost { contentTypes?: string[]; httpsPassthrough?: true; httpsPassthroughTotalTimeoutMs?: number; + httpsPassthroughRoutes?: HttpsPassthroughRoute[]; packageTransport?: true; limits: ExternalFetchLimits; } @@ -267,6 +289,61 @@ function validatePolicyHostname(host: string): void { } } +function compareRoutes(a: HttpsPassthroughRoute, b: HttpsPassthroughRoute): number { + if (a.path !== b.path) return a.path < b.path ? -1 : 1; + if (a.method !== b.method) return a.method < b.method ? -1 : 1; + return 0; +} + +/** + * A route's path must already be in the exact form the gateway compares against: what the URL + * parser yields as `pathname` (dot segments resolved), without query, fragment, percent-encoding + * or backslashes, so no two spellings can name the same request. + */ +function parseHttpsPassthroughRoutes( + value: unknown, + host: string, +): HttpsPassthroughRoute[] { + const label = `External fetch host ${host} httpsPassthroughRoutes`; + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > HARD_MAX_HTTPS_PASSTHROUGH_ROUTES + ) { + throw new Error( + `${label} must list 1 to ${HARD_MAX_HTTPS_PASSTHROUGH_ROUTES} routes`, + ); + } + const routes: HttpsPassthroughRoute[] = []; + const seen = new Set(); + for (const entry of value) { + const raw = objectValue(entry, label); + assertOnlyKeys(raw, ['method', 'path'], label); + const method = raw.method; + const path = raw.path; + if ( + typeof method !== 'string' || + !(HTTPS_PASSTHROUGH_METHODS as readonly string[]).includes(method) + ) { + throw new Error(`${label} has an invalid method`); + } + if ( + typeof path !== 'string' || + !path.startsWith('/') || + path.length > 2_048 || + /[?#%\\\s\u0000-\u001f\u007f]/.test(path) || + new URL(path, 'https://route.invalid').pathname !== path + ) { + throw new Error(`${label} has an invalid path`); + } + const key = `${method} ${path}`; + if (seen.has(key)) throw new Error(`${label} lists a route twice`); + seen.add(key); + routes.push({ method: method as HttpsPassthroughMethod, path }); + } + return routes.sort(compareRoutes); +} + export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { const raw = objectValue(value, 'External fetch policy'); assertOnlyKeys( @@ -299,6 +376,7 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { 'contentTypes', 'httpsPassthrough', 'httpsPassthroughTotalTimeoutMs', + 'httpsPassthroughRoutes', 'packageTransport', 'limits', ], @@ -339,6 +417,15 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, ) : undefined; + if (rawHost.httpsPassthroughRoutes !== undefined && !httpsPassthrough) { + throw new Error( + `External fetch host ${host} cannot set httpsPassthroughRoutes without HTTPS passthrough`, + ); + } + const httpsPassthroughRoutes = + rawHost.httpsPassthroughRoutes === undefined + ? undefined + : parseHttpsPassthroughRoutes(rawHost.httpsPassthroughRoutes, host); if ( rawHost.contentTypes !== undefined && !Array.isArray(rawHost.contentTypes) @@ -377,6 +464,7 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { ...(httpsPassthroughTotalTimeoutMs === undefined ? {} : { httpsPassthroughTotalTimeoutMs }), + ...(httpsPassthroughRoutes === undefined ? {} : { httpsPassthroughRoutes }), limits: rawHost.limits === undefined ? { ...limits } @@ -478,14 +566,30 @@ export function validateExternalFetchUrl( return validated; } +/** + * A passthrough request is allowed when its host enables passthrough and, if the host lists + * routes, when its method and normalized path match one exactly and it carries no query. + */ export function validateHttpsPassthroughUrl( raw: string, policy: ExternalFetchPolicy, + method: string, ): ValidatedExternalFetchUrl { const validated = validatePolicyUrl(raw, policy); if (!validated.policy.httpsPassthrough) { throw new ExternalFetchError('HOST_NOT_ALLOWED'); } + const routes = validated.policy.httpsPassthroughRoutes; + if (routes) { + const requested = method.toUpperCase(); + const pathname = validated.url.pathname; + if ( + validated.queryPresent || + !routes.some(route => route.method === requested && route.path === pathname) + ) { + throw new ExternalFetchError('HOST_NOT_ALLOWED'); + } + } return validated; } @@ -550,6 +654,13 @@ export function serializeExternalFetchPolicy( httpsPassthroughTotalTimeoutMs: entry.httpsPassthroughTotalTimeoutMs, }), + ...(entry.httpsPassthroughRoutes === undefined + ? {} + : { + httpsPassthroughRoutes: entry.httpsPassthroughRoutes.map( + route => ({ ...route }), + ), + }), ...(entry.packageTransport ? { packageTransport: true as const } : {}), @@ -566,6 +677,24 @@ export function externalFetchPolicyDigest(policy: ExternalFetchPolicy): string { .digest('base64url'); } +/** + * Routes a signed host may use under the deployment's upper bound: absent on one side means + * that side does not restrict; listed on both, every signed route must be in the deployment's. + */ +function intersectHttpsPassthroughRoutes( + requested: HttpsPassthroughRoute[] | undefined, + upper: HttpsPassthroughRoute[] | undefined, +): HttpsPassthroughRoute[] | undefined { + if (!upper) return requested?.map(route => ({ ...route })); + if (!requested) return upper.map(route => ({ ...route })); + const allowed = new Set(upper.map(route => `${route.method} ${route.path}`)); + for (const route of requested) { + if (!allowed.has(`${route.method} ${route.path}`)) + throw new ExternalFetchError('HOST_NOT_ALLOWED'); + } + return requested.map(route => ({ ...route })); +} + export function intersectExternalFetchPolicies( signed: ExternalFetchPolicy, deployment: ExternalFetchPolicy, @@ -596,6 +725,12 @@ export function intersectExternalFetchPolicies( Math.min(requested.limits[key], upper.limits[key], limits[key]), ]), ) as unknown as ExternalFetchLimits; + const httpsPassthroughRoutes = requested.httpsPassthrough + ? intersectHttpsPassthroughRoutes( + requested.httpsPassthroughRoutes, + upper.httpsPassthroughRoutes, + ) + : undefined; hosts.set(host, { contentTypes: new Set(requested.contentTypes), httpsPassthrough: requested.httpsPassthrough, @@ -609,6 +744,7 @@ export function intersectExternalFetchPolicies( ), } : {}), + ...(httpsPassthroughRoutes ? { httpsPassthroughRoutes } : {}), packageTransport: requested.packageTransport, limits: hostLimits, }); diff --git a/service/src/external-fetch.ts b/service/src/external-fetch.ts index 938befa3..d5738d5f 100644 --- a/service/src/external-fetch.ts +++ b/service/src/external-fetch.ts @@ -424,7 +424,7 @@ async function requestPinnedPassthrough( export async function openHttpsPassthrough( args: OpenHttpsPassthroughArgs, ): Promise { - const target = validateHttpsPassthroughUrl(args.url, args.policy); + const target = validateHttpsPassthroughUrl(args.url, args.policy, args.method); const addresses = await resolveExternalFetchAddresses( target.host, args.resolver,