Deduplicate the request layer and make dynamicPageRegex additive

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 2026-09-02
+
+### Changes
+
+- Config: `dynamicPageRegex` now extends the built-in dynamic-segment heuristics (numeric, UUID,
+  ULID, hex) instead of replacing them. Previously, setting a custom pattern silently disabled
+  every built-in match on any segment the custom pattern didn't also cover.
+- API Requests: Page-URL generalization and the API endpoint list now share one implementation.
+  The endpoint list used to walk paths with its own copy of the dynamic-segment logic, which could
+  drift from the URL matcher used everywhere else.
+
 ## 2026-09-01
 
 ### Changes

boat/api-tester/src/ai/curler-tools.ts

@@ -66,7 +66,7 @@ export function createCurlerTools(apiClient: ApiClient, requestState: RequestSto
           queryParams: input.queryParams,
         });
 
-        requestState.addRequest(result);
+        requestState.addMadeRequest(result);
 
         if (result.error) {
           tag('error').log(`${input.method} ${input.path} > Network error: ${result.error}`);

boat/prima/src/prima.ts

@@ -1112,7 +1112,7 @@ export class Prima {
       aria: result.ariaSnapshot,
       html: await result.combinedHtml(),
       screenshot: result.screenshot,
-      requests: this.bot.requestStore().getRequests(),
+      requests: this.bot.requestStore().getMadeRequests(),
     });
   }
 

boat/prima/tests/prima.test.ts

@@ -95,7 +95,7 @@ function fakePrima(options: Record<string, unknown> = {}) {
     }),
     getCurrentState: () => fakeState(),
     getConfig: () => ({}),
-    requestStore: () => ({ getRequests: () => [] }),
+    requestStore: () => ({ getMadeRequests: () => [] }),
     getProvider: () => ({ chat: async () => '' }),
   };
   (prima as any).artifactsDir = artifactsRoot;

src/ai/provider.ts

@@ -51,12 +51,7 @@ export async function flushTelemetry(): Promise<void> {
 const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 
 function extractCachedTokens(usage: any): number {
-  if (!usage) return 0;
-  const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
-  if (typeof direct === 'number') return direct;
-  const raw = usage.raw;
-  const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
-  return typeof fromRaw === 'number' ? fromRaw : 0;
+  return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
 
 function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController): Promise<never> {
@@ -243,8 +238,8 @@ export class Provider {
   private recordUsage(agentName: string, modelName: string, usage: any): void {
     if (!usage) return;
     Stats.recordTokens(agentName, modelName, {
-      input: usage.inputTokens ?? usage.promptTokens ?? 0,
-      output: usage.outputTokens ?? usage.completionTokens ?? 0,
+      input: usage.inputTokens ?? 0,
+      output: usage.outputTokens ?? 0,
       total: usage.totalTokens ?? 0,
       cached: extractCachedTokens(usage),
     });
@@ -428,7 +423,7 @@ export class Provider {
     let invalidRequestFeedbackAdded = false;
     const executedStepMessages: ModelMessage[] = [];
     try {
-      const response = await this.withModelRequestSlot(() =>
+      let response = await this.withModelRequestSlot(() =>
         withRetry(async () => {
           const stepMessages: ModelMessage[] = [];
           const onStepEnd = (step: any) => {
@@ -458,7 +453,7 @@ export class Provider {
 
       clearActivity();
 
-      withExecutedSteps(response, executedStepMessages);
+      response = withExecutedSteps(response, executedStepMessages);
 
       // Log tool usage summary
       if (response.toolCalls && response.toolCalls.length > 0) {

src/api/request-store.ts

@@ -1,6 +1,6 @@
 import { existsSync, readdirSync } from 'node:fs';
 import path from 'node:path';
-import { isDynamicSegment } from '../utils/url-matcher.ts';
+import { generalizeUrl, isDynamicSegment } from '../utils/url-matcher.ts';
 import { RequestResult } from './request-result.ts';
 
 const AUTH_HEADERS = ['authorization', 'x-api-key', 'x-csrf-token'];
@@ -46,10 +46,6 @@ export class RequestStore {
     result.save(this.outputDir);
   }
 
-  addRequest(result: RequestResult): void {
-    this.addMadeRequest(result);
-  }
-
   getCapturedRequests(): RequestResult[] {
     return this.capturedRequests;
   }
@@ -58,27 +54,10 @@ export class RequestStore {
     return this.madeRequests;
   }
 
-  getRequests(): RequestResult[] {
-    return this.madeRequests;
-  }
-
   getLastRequest(): RequestResult | undefined {
     return this.madeRequests[this.madeRequests.length - 1];
   }
 
-  getRequestsByEndpoint(pathPrefix: string): RequestResult[] {
-    return this.madeRequests.filter((r) => r.path.startsWith(pathPrefix));
-  }
-
-  getRequestsByMethod(method: string): RequestResult[] {
-    const upper = method.toUpperCase();
-    return this.madeRequests.filter((r) => r.method === upper);
-  }
-
-  getRequestsByStatus(status: number): RequestResult[] {
-    return this.madeRequests.filter((r) => r.status === status);
-  }
-
   toEndpointList(scopePath?: string): string {
     let requests = this.capturedRequests;
     if (scopePath) requests = this.getWriteRequestsForScope(scopePath);
@@ -87,7 +66,7 @@ export class RequestStore {
     const lines: string[] = [];
 
     for (const req of requests) {
-      const key = `${req.method} ${normalizePathPattern(req.path)}`;
+      const key = `${req.method} ${generalizeUrl(req.path, () => '{id}')}`;
       if (seen.has(key)) continue;
       seen.add(key);
       lines.push(key);
@@ -113,14 +92,18 @@ export class RequestStore {
 
   findCapturedRequest(method: string, searchPath: string): RequestResult | undefined {
     const upper = method.toUpperCase();
-    const search = normalizePathPattern(searchPath).split('/').filter(Boolean);
+    const search = generalizeUrl(searchPath, () => '{id}')
+      .split('/')
+      .filter(Boolean);
 
     let best: RequestResult | undefined;
     let bestScore = -1;
 
     for (const req of this.capturedRequests) {
       if (req.method !== upper) continue;
-      const segments = normalizePathPattern(req.path).split('/').filter(Boolean);
+      const segments = generalizeUrl(req.path, () => '{id}')
+        .split('/')
+        .filter(Boolean);
       if (segments.length < search.length) continue;
       if (!search.every((segment, i) => segment === segments[i])) continue;
 
@@ -159,8 +142,7 @@ export class RequestStore {
   }
 
   getWriteRequestsForScope(scopePath: string): RequestResult[] {
-    const writeMethods = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
-    const writes = this.capturedRequests.filter((r) => writeMethods.has(r.method));
+    const writes = this.capturedRequests.filter((r) => r.isWrite);
     const scopeSegments = scopePath.split('/').filter(Boolean);
     if (scopeSegments.length === 0) return writes;
 

src/utils/url-matcher.ts

@@ -4,7 +4,7 @@ import { ConfigParser } from '../config.js';
 export function isDynamicSegment(segment: string): boolean {
   try {
     const configRegex = ConfigParser.getInstance().getConfig().dynamicPageRegex;
-    if (configRegex) return new RegExp(configRegex, 'i').test(segment);
+    if (configRegex && new RegExp(configRegex, 'i').test(segment)) return true;
   } catch {
     /* config not loaded yet */
   }
@@ -52,10 +52,10 @@ export function generalizeSegment(segment: string): string {
   return '[^/]+';
 }
 
-export function generalizeUrl(url: string): string {
+export function generalizeUrl(url: string, replaceSegment: (segment: string) => string = generalizeSegment): string {
   return url
     .split('/')
-    .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? generalizeSegment(seg) : seg))
+    .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? replaceSegment(seg) : seg))
     .join('/');
 }
 

tests/integration/prima-do.test.ts

@@ -90,7 +90,7 @@ describe('Prima.do with aimock', () => {
       getExplorer: () => ({ action: () => action, capture: async () => null }),
       stateManager: () => ({ getCurrentState: () => boardState, getVisitCount: () => 1 }),
       getCurrentState: () => boardState,
-      requestStore: () => ({ getRequests: () => [] }),
+      requestStore: () => ({ getMadeRequests: () => [] }),
       getProvider: () => provider,
       experienceTracker: () => ({ renderExperienceTocFor: () => '' }),
       agentResearcher: () => ({}),

tests/unit/url-matcher.test.ts

@@ -70,12 +70,15 @@ describe('url-matcher', () => {
       expect(isDynamicSegment('8471')).toBe(true);
     });
 
-    it('respects user-provided dynamicPageRegex override', () => {
+    it('user dynamicPageRegex extends the built-in patterns', () => {
       const instance = ConfigParser.getInstance();
       (instance as any).config = { ...(instance as any).config, dynamicPageRegex: '^custom-\\d+$' };
 
       expect(isDynamicSegment('custom-42')).toBe(true);
       expect(isDynamicSegment('custom-X')).toBe(false);
+      expect(isDynamicSegment('123')).toBe(true);
+      expect(isDynamicSegment('550e8400-e29b-41d4-a716-446655440000')).toBe(true);
+      expect(isDynamicSegment('login')).toBe(false);
     });
   });