diff --git a/content/docs/automation/connectors.mdx b/content/docs/automation/connectors.mdx index 794faad7dd7..7bebaa796de 100644 --- a/content/docs/automation/connectors.mdx +++ b/content/docs/automation/connectors.mdx @@ -184,9 +184,20 @@ match. ```bash pnpm --filter @objectstack/connector-openapi normalize-openapi \ --in https://example.com/openapi.yaml --out ./specs/crm-openapi.json \ - --include-pattern '^GET /v1/customers' + --include-pattern '^GET /v1/customers' --trim ``` +With `--trim` the document it writes holds **only** the operations the +patterns match: every other operation, every path item left with none, and +`webhooks` are dropped before inlining, so nothing reachable only from a +dropped operation is written, and the printed sha256 is the hash of those +trimmed bytes. Package that output — a large vendor document shrinks to the +operations the app calls (Stripe's 612 operations, 63 MB at the default +depth, become about 1.9 MB for six). Without `--trim` the whole document is +written and the patterns only compute `include`, so `include` can be widened +later by editing config alone; with it, widening means re-running the script. +`--trim` without an `--include-pattern` is refused. + Inlining is bounded: a recursive schema, or one more than `--max-depth` (default 2) named schemas deep, becomes a typed stub whose description names the reference it replaced. A Swagger 2.0 operation whose empty `consumes` list diff --git a/content/docs/deployment/marketplace-handbook.mdx b/content/docs/deployment/marketplace-handbook.mdx index 7dbce7f5d3f..61de1bc156d 100644 --- a/content/docs/deployment/marketplace-handbook.mdx +++ b/content/docs/deployment/marketplace-handbook.mdx @@ -113,8 +113,10 @@ administrator, not for a contributor: every one of them ([Setting a credential](/docs/automation/connectors#setting-a-credential)). The README is still where the administrator learns what each value is and where to get it. An app that ships declarative - connectors gains an offline spec-normalization tool with - [#26709](https://github.com/objectstack-ai/objectstack/issues/26709). + OpenAPI connectors produces its packaged document with the offline + `normalize-openapi` script: `--trim` keeps only the operations the app calls, + and the sha256 it prints is the one to record here + ([`provider: 'openapi'`](/docs/automation/connectors#provider-openapi--one-action-per-operation)). ## Build and verify before publishing diff --git a/packages/connectors/connector-openapi/scripts/normalize-openapi.mts b/packages/connectors/connector-openapi/scripts/normalize-openapi.mts index 613cae347a0..a2190cd6ea1 100644 --- a/packages/connectors/connector-openapi/scripts/normalize-openapi.mts +++ b/packages/connectors/connector-openapi/scripts/normalize-openapi.mts @@ -7,8 +7,8 @@ * * pnpm --filter @objectstack/connector-openapi normalize-openapi \ * --in --out \ - * [--include-pattern ]... [--max-depth ] [--server-url ] \ - * [--empty-consumes ] + * [--include-pattern ]... [--trim] [--max-depth ] \ + * [--server-url ] [--empty-consumes ] * * The runtime never converts: `openapi-provider.ts` refuses a document that is * not 3.x, and the generator skips a `$ref` parameter, request body or response @@ -25,9 +25,25 @@ * `securitySchemes` stay: they are referenced by name, not by `$ref`); * 5. make `servers[0].url` absolute (server variables take their defaults; a * relative URL resolves against the source URL, or `--server-url`); - * 6. optionally compute the `providerConfig.include` list from allow-patterns; + * 6. optionally compute the `providerConfig.include` list from allow-patterns, + * and with `--trim` keep ONLY the operations they match (below); * 7. write the JSON and print its sha256 -- the hash of the bytes written. * + * ## `--trim`: package the operations the app calls, not the vendor's catalogue + * + * Without `--trim` the whole document is written and the allow-patterns only + * compute `include`: the app can widen `include` later by editing config. A + * large vendor document is then shipped whole -- measured on Stripe's + * `spec3.json`, 612 operations and 63,317,655 bytes at the default depth for an + * app that calls six. `--trim` (which requires at least one + * `--include-pattern`) drops every path operation no pattern matches, every + * path item left with none, and `webhooks`, BEFORE step 4 -- so only the kept + * operations are dereferenced, and nothing reachable only from a dropped one + * survives into the output. The `include` list is the same one an untrimmed + * run computes (the same patterns, the same `operationActionKey`), and the + * printed sha256 is still the hash of the bytes written: the trimmed bytes. + * The `securitySchemes` stay, as in step 4: they are referenced by name. + * * ## Where a `$ref` is cut, and why it has to be * * "No `$ref` left" and "every schema fully inlined" cannot both hold for a real @@ -89,6 +105,8 @@ export interface NormalizeOptions { input: string; /** Allow-patterns (JavaScript regular expressions) for the generated `include` list. */ includePatterns?: readonly string[]; + /** Keep only the operations {@link includePatterns} match (requires at least one pattern). */ + trim?: boolean; /** Named-schema hops inlined before a stub (default {@link DEFAULT_MAX_DEPTH}). */ maxDepth?: number; /** Absolute http(s) URL that replaces `servers[0].url`. */ @@ -113,8 +131,10 @@ export interface NormalizeResult { sourceVersion: string; /** The output document's `openapi` version. */ openapi: string; - /** Every operation's action key, in document order, de-duplicated. */ + /** Every written operation's action key, in document order, de-duplicated. */ operationKeys: string[]; + /** Operations in the source document -- more than `operationKeys` only when trimmed. */ + sourceOperations: number; /** The `providerConfig.include` list, when allow-patterns were given. */ include?: string[]; /** Schema references replaced by a stub, by reason. */ @@ -145,6 +165,12 @@ export async function normalizeOpenApi(options: NormalizeOptions): Promise op.key))]; let include: string[] | undefined; - if (options.includePatterns && options.includePatterns.length > 0) { - const patterns = options.includePatterns.map((source) => { - try { - return new RegExp(source); - } catch (err) { - throw new NormalizeError(`--include-pattern '${source}' is not a valid regular expression: ${(err as Error).message}`); - } - }); - include = [ - ...new Set( - operations - .filter((op) => patterns.some((p) => p.test(op.key) || p.test(`${op.method.toUpperCase()} ${op.path}`))) - .map((op) => op.key), - ), - ]; - if (include.length === 0) { - throw new NormalizeError( - `--include-pattern matched none of the ${operationKeys.length} operation(s); a pattern is tested against ` + - `each action key (an operationId, or the method_path key) and against 'METHOD /path'.`, - ); - } + if (patterns.length > 0) { + include = [...new Set(operations.filter((op) => matches(patterns, op)).map((op) => op.key))]; + if (include.length === 0) throw noMatch(operations.length); } // 7. The bytes, and their hash. @@ -263,6 +280,7 @@ export async function normalizeOpenApi(options: NormalizeOptions): Promise { + try { + return new RegExp(source); + } catch (err) { + throw new NormalizeError(`--include-pattern '${source}' is not a valid regular expression: ${(err as Error).message}`); + } + }); +} + +/** An allow-pattern is tested against the action key and against 'METHOD /path'. */ +const matches = (patterns: readonly RegExp[], op: { key: string; method: string; path: string }): boolean => + patterns.some((p) => p.test(op.key) || p.test(`${op.method.toUpperCase()} ${op.path}`)); + +const noMatch = (total: number): NormalizeError => + new NormalizeError( + `--include-pattern matched none of the ${total} operation(s); a pattern is tested against ` + + `each action key (an operationId, or the method_path key) and against 'METHOD /path'.`, + ); + +/** + * `--trim`: the `paths` object with only the operations a pattern matches. A + * path item keeps its non-operation keys (`parameters`, `summary`, `servers`, + * extensions); one left with no operation is dropped. A `$ref` path item is + * resolved first (its operations live in the target), without walking it. + */ +function trimPaths( + paths: Json | undefined, + patterns: readonly RegExp[], + deref: Dereferencer, +): { paths: JsonObject; kept: number; sourceOperations: number } { + const out: JsonObject = {}; + let kept = 0; + let sourceOperations = 0; + for (const [path, raw] of Object.entries(isObject(paths) ? paths : {})) { + if (path.startsWith('x-')) { + put(out, path, raw); + continue; + } + const item = deref.target(raw); + if (!isObject(item)) continue; + const trimmed: JsonObject = {}; + let keptHere = 0; + for (const [key, value] of Object.entries(item)) { + const method = HTTP_METHODS.find((m) => m === key); + if (method === undefined) { + put(trimmed, key, value); + continue; + } + if (!isObject(value)) continue; + sourceOperations++; + const operationId = typeof value.operationId === 'string' ? value.operationId : undefined; + if (matches(patterns, { key: operationActionKey({ operationId, method, path }), method, path })) { + put(trimmed, key, value); + keptHere++; + } + } + if (keptHere > 0) { + put(out, path, trimmed); + kept += keptHere; + } + } + return { paths: out, kept, sourceOperations }; +} + /** Count `$ref` keys anywhere in a value. */ function countRefs(node: Json): number { if (Array.isArray(node)) return node.reduce((sum, item) => sum + countRefs(item), 0); @@ -391,6 +475,27 @@ class Dereferencer { return out; } + /** + * A `$ref` object's target, followed to the end of its chain with the + * referring object's sibling keys laid over it, and nothing inside walked: + * what a `$ref` path item stands for, before `--trim` selects its operations. + */ + target(node: Json): Json { + const seen = new Set(); + let current = node; + let siblings: JsonObject = {}; + while (isObject(current) && typeof current.$ref === 'string') { + if (seen.has(current.$ref)) { + throw new NormalizeError(`the document's $ref '${current.$ref}' refers to itself outside a schema; it has no inline form.`); + } + seen.add(current.$ref); + const { $ref, ...rest } = current; + siblings = { ...rest, ...siblings }; + current = this.resolve($ref); + } + return isObject(current) ? { ...current, ...siblings } : current; + } + /** Literal data: a `$ref` to it is resolved, and nothing inside it is walked. */ private literal(node: Json): Json { const seen = new Set(); @@ -537,7 +642,7 @@ function listOperations(doc: JsonObject): { key: string; method: string; path: s } const USAGE = - 'usage: normalize-openapi --in --out [--include-pattern ]... ' + + 'usage: normalize-openapi --in --out [--include-pattern ]... [--trim] ' + `[--max-depth (default ${DEFAULT_MAX_DEPTH})] [--server-url ] [--empty-consumes ]`; /** @@ -547,8 +652,13 @@ const USAGE = */ export async function main(argv: readonly string[], io = { out: process.stdout, err: process.stderr }): Promise { const flags: Record = {}; + let trim = false; for (let i = 0; i < argv.length; i++) { const arg = argv[i] as string; + if (arg === '--trim') { + trim = true; + continue; + } const value = argv[i + 1]; if (!/^--(in|out|include-pattern|max-depth|server-url|empty-consumes)$/.test(arg) || value === undefined) { io.err.write(`normalize-openapi: unexpected argument '${arg}'.\n${USAGE}\n`); @@ -573,6 +683,7 @@ export async function main(argv: readonly string[], io = { out: process.stdout, input, cwd, includePatterns: flags['include-pattern'], + trim, maxDepth: depthFlag === undefined ? undefined : Number(depthFlag), serverUrl: single('server-url'), emptyConsumes: single('empty-consumes'), @@ -590,6 +701,7 @@ export async function main(argv: readonly string[], io = { out: process.stdout, sourceVersion: result.sourceVersion, openapi: result.openapi, operations: result.operationKeys.length, + ...(trim ? { trimmedFrom: result.sourceOperations } : {}), maxDepth: depthFlag === undefined ? DEFAULT_MAX_DEPTH : Number(depthFlag), cuts: result.cuts, refsLeft: result.refsLeft, diff --git a/packages/connectors/connector-openapi/scripts/normalize-openapi.test.ts b/packages/connectors/connector-openapi/scripts/normalize-openapi.test.ts index 3bfe0f8d691..d5eb6f7025c 100644 --- a/packages/connectors/connector-openapi/scripts/normalize-openapi.test.ts +++ b/packages/connectors/connector-openapi/scripts/normalize-openapi.test.ts @@ -181,6 +181,105 @@ describe('normalize-openapi: OpenAPI 3.0 YAML across two files (Stripe-shaped)', }); }); +describe('normalize-openapi: --trim keeps only the matched operations', () => { + const PATTERNS = ['^GET /v1/customers', 'balance']; + + it('writes exactly the matched operations, with the include list an untrimmed run computes', async () => { + const untrimmed = await normalizeOpenApi({ input: OPENAPI3_YAML, includePatterns: PATTERNS }); + const trimmed = await normalizeOpenApi({ input: OPENAPI3_YAML, includePatterns: PATTERNS, trim: true }); + expect(trimmed.operationKeys).toEqual(['GetCustomers', 'GetCustomersCustomer', 'get_v1_balance']); + expect(trimmed.include).toEqual(untrimmed.include); + expect(trimmed.sourceOperations).toBe(4); + const paths = trimmed.document.paths as Record>; + expect(Object.keys(paths)).toEqual(['/v1/customers', '/v1/customers/{customer}', '/v1/balance']); + // The unmatched POST is gone from a path item whose GET is kept. + expect(Object.keys(paths['/v1/customers'] ?? {})).toEqual(['get']); + expect(trimmed.text.length).toBeLessThan(untrimmed.text.length); + + const { def } = await load(trimmed); + expect(def.actions?.map((a) => a.key)).toEqual(trimmed.include); + expect(refPaths(def)).toEqual([]); + expect(refPaths(trimmed.document)).toEqual([]); + expect(trimmed.refsLeft).toBe(0); + }); + + it('carries nothing reachable only from a dropped operation, and drops path items left empty', async () => { + const trimmed = await normalizeOpenApi({ input: OPENAPI3_YAML, includePatterns: ['balance'], trim: true }); + expect(Object.keys(trimmed.document.paths as object)).toEqual(['/v1/balance']); + // customer, customer_create, the shared error response and the shared parameter were reachable + // only from the customer operations. + for (const gone of ['A customer.', 'Error response.', 'someone@example.com', '"limit"', 'subscription']) { + expect(trimmed.text).not.toContain(gone); + } + // Only the kept operation is walked: none of the customer schemas' cuts are counted. + expect(trimmed.cuts).toEqual({ recursive: 0, maxDepth: 0, extension: 0 }); + expect(trimmed.document.components).toEqual({ securitySchemes: { bearerAuth: { type: 'http', scheme: 'bearer' } } }); + const { def } = await load(trimmed); + expect(def.actions?.map((a) => a.key)).toEqual(['get_v1_balance']); + }); + + it('trims a Swagger 2.0 document after conversion, keyed by operationId', async () => { + const trimmed = await normalizeOpenApi({ + input: SWAGGER2, + emptyConsumes: 'application/json', + includePatterns: ['^Envelopes_GetEnvelope$'], + trim: true, + }); + expect(trimmed.operationKeys).toEqual(['Envelopes_GetEnvelope']); + expect(trimmed.sourceOperations).toBe(3); + const { def } = await load(trimmed); + expect(def.actions?.map((a) => a.key)).toEqual(['Envelopes_GetEnvelope']); + expect(refPaths(def)).toEqual([]); + }); + + it('resolves a $ref path item before selecting its operations', async () => { + const dir = await mkdtemp(join(tmpdir(), 'normalize-openapi-')); + try { + const file = join(dir, 'ref-path-item.json'); + await writeFile( + file, + JSON.stringify({ + openapi: '3.1.0', + info: { title: 't', version: '1' }, + servers: [{ url: 'https://api.example.com' }], + paths: { + '/a': { $ref: '#/paths/~1b', summary: 'A, shaped like B' }, + '/b': { + get: { operationId: 'getB', responses: { '200': { description: 'OK' } } }, + delete: { operationId: 'deleteB', responses: { '200': { description: 'OK' } } }, + }, + }, + webhooks: { ping: { post: { operationId: 'ping', responses: { '200': { description: 'OK' } } } } }, + }), + ); + const trimmed = await normalizeOpenApi({ input: file, includePatterns: ['^GET /a$'], trim: true }); + expect(trimmed.document.paths).toEqual({ + '/a': { summary: 'A, shaped like B', get: { operationId: 'getB', responses: { '200': { description: 'OK' } } } }, + }); + expect(trimmed.document).not.toHaveProperty('webhooks'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('refuses --trim without a pattern, and a pattern that matches nothing', async () => { + expect((await refusal(normalizeOpenApi({ input: OPENAPI3_YAML, trim: true }))).message).toContain( + 'give at least one --include-pattern', + ); + expect( + (await refusal(normalizeOpenApi({ input: OPENAPI3_YAML, includePatterns: ['^nope$'], trim: true }))).message, + ).toContain('matched none of the 4 operation(s)'); + }); + + it('without --trim the whole document is written, as before (control)', async () => { + const result = await normalizeOpenApi({ input: OPENAPI3_YAML, includePatterns: ['balance'] }); + expect(result.operationKeys).toEqual(['GetCustomers', 'PostCustomers', 'GetCustomersCustomer', 'get_v1_balance']); + expect(result.sourceOperations).toBe(4); + expect(result.include).toEqual(['get_v1_balance']); + expect(result.text).toContain('A customer.'); + }); +}); + describe('normalize-openapi: servers, sources and refusals', () => { const doc = (servers: unknown) => JSON.stringify({ openapi: '3.1.0', info: { title: 't', version: '1' }, servers, paths: { '/x': { get: { responses: {} } } } }); @@ -257,6 +356,36 @@ describe('normalize-openapi: the CLI', () => { expect(summary.include).toEqual(['GetCustomers', 'GetCustomersCustomer']); }); + it('with --trim, writes the trimmed bytes and prints their sha256', async () => { + const dir = await mkdtemp(join(tmpdir(), 'normalize-openapi-')); + dirs.push(dir); + const out = join(dir, 'billing.trimmed.json'); + const { chunks, io } = capture(); + const code = await main(['--in', OPENAPI3_YAML, '--out', out, '--trim', '--include-pattern', '^Get'], io); + expect(code).toBe(0); + const summary = JSON.parse(chunks.out) as { + sha256: string; + bytes: number; + include: string[]; + operations: number; + trimmedFrom: number; + }; + const written = await readFile(out); + expect(summary.bytes).toBe(written.length); + expect(summary.sha256).toBe(createHash('sha256').update(written).digest('hex')); + expect(summary.include).toEqual(['GetCustomers', 'GetCustomersCustomer']); + expect(summary.operations).toBe(2); + expect(summary.trimmedFrom).toBe(4); + const paths = (JSON.parse(written.toString('utf8')) as { paths: Record }).paths; + expect(Object.keys(paths)).toEqual(['/v1/customers', '/v1/customers/{customer}']); + expect(Object.keys(paths['/v1/customers'] ?? {})).toEqual(['get']); + + const refused = capture(); + expect(await main(['--in', OPENAPI3_YAML, '--out', join(dir, 'y.json'), '--trim'], refused.io)).toBe(1); + expect(refused.chunks.err).toContain('--include-pattern'); + await expect(readFile(join(dir, 'y.json'))).rejects.toThrow(); + }); + it('exits 1 on a refusal and 2 on a usage error, writing nothing', async () => { const dir = await mkdtemp(join(tmpdir(), 'normalize-openapi-')); dirs.push(dir);