From 79be921680fb3f193c6eff0d37b0afeb300463f8 Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Mon, 29 Jun 2026 09:15:00 -0400 Subject: [PATCH 1/6] fix(auditor): tighten context generation prompts and feed complete vendor list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Generated context for SOC 2 reports contains defects that require manual cleanup before the report is usable. The Critical Vendors list omits vendors that exist in the Vendors tab, Sub-service Organisations includes identity/SaaS tools instead of only hosting providers, and narrative fields state headcount and name personnel all of which get lifted verbatim into the customer report. ## Root cause The generation prompts in `generate-auditor-content.ts` lack explicit exclusions for personnel/headcount in narrative fields, cap Critical Vendors at 3 6 entries without feeding the complete Vendors tab, and permit identity providers in the Subservice Organisations list. The Q&A context (which includes employee count and executive names) is concatenated into all prompts without filtering. ## Fix Added strict exclusion rules to all four narrative field prompts: no employee counts, no named individuals or roles, no marketing language. Rewrote Critical Vendors prompt to pull from the complete Vendors tab (not a capped subset) and return all entries in `Name - Type - (function)` format. Restricted Subservice Organisations to only IaaS/PaaS hosting providers from the Vendors tab; explicitly banned identity/SSO tools and general SaaS. Outputs now use schema-enforced structured fields to ensure all values are always present and reliably parseable. ## Explicitly NOT touched - Auth, RBAC, schema, or billing logic - Organisation scoping (organizationId is preserved) - The database schema or vendor model - Per-field Regenerate button UI (separate feature) ## Verification ✅ Narrative fields contain no employee/headcount numbers ✅ Narrative fields contain no named individuals, roles, or titles ✅ Narrative tone is declarative, present-tense, with no hedging ("may", "appears") or attribution ("according to", "the site says") ✅ Critical Vendors list includes every vendor in the Vendors tab, correctly formatted, none invented or dropped ✅ Sub-service Organisations contains only hosting/IaaS/PaaS providers; identity/SSO tools never appear; returns empty list when no qualifying vendors exist ✅ Structured output schema enforced; all fields always present --- .../generate-auditor-content-prompts.test.ts | 99 ++++++++ .../generate-auditor-content-prompts.ts | 238 ++++++++++++++++++ .../tasks/auditor/generate-auditor-content.ts | 205 +++------------ 3 files changed, 366 insertions(+), 176 deletions(-) create mode 100644 apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts create mode 100644 apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts new file mode 100644 index 0000000000..90eb1d903f --- /dev/null +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; +import { + buildSectionUserPrompt, + buildVendorsBlock, + NARRATIVE_SECTIONS, + sectionPrompts, + type VendorTabEntry, +} from './generate-auditor-content-prompts'; + +// Fake fixtures only — no customer data. A mix of hosting (cloud/infra), +// general SaaS, identity, and finance vendors, like a real Vendors tab. +const VENDORS: VendorTabEntry[] = [ + { name: 'AWS', description: 'Cloud hosting', category: 'cloud', website: 'https://aws.example' }, + { name: 'Vercel', description: 'App hosting', category: 'infrastructure', website: null }, + { name: 'Slack', description: 'Team chat', category: 'software_as_a_service', website: null }, + { name: 'Okta', description: 'Identity provider', category: 'software_as_a_service', website: null }, + { name: 'GitHub', description: 'Source control', category: 'software_as_a_service', website: null }, + { name: 'Stripe', description: 'Payments', category: 'finance', website: null }, +]; + +describe('buildVendorsBlock', () => { + it('lists EVERY vendor from the Vendors tab (CS-589: list came back too small)', () => { + const block = buildVendorsBlock(VENDORS); + + for (const vendor of VENDORS) { + expect(block).toContain(vendor.name); + } + // One line per vendor — nothing dropped. + expect(block.split('\n')).toHaveLength(VENDORS.length); + }); + + it('does not invent entries for an empty Vendors tab', () => { + expect(buildVendorsBlock([])).toContain('No vendors'); + }); +}); + +describe('buildSectionUserPrompt', () => { + it('feeds the full Vendors tab into the prompt (CS-589: the tab was never passed)', () => { + const prompt = buildSectionUserPrompt({ + section: 'critical-vendors', + organization: { name: 'Acme Corp', website: 'https://acme.test' }, + websiteContent: 'Acme builds widgets.', + contextHubText: 'Q: Where do you host your applications and data?\nA: AWS', + vendorsBlock: buildVendorsBlock(VENDORS), + }); + + expect(prompt).toContain('VENDORS TAB'); + for (const vendor of VENDORS) { + expect(prompt).toContain(vendor.name); + } + }); +}); + +describe('critical-vendors prompt', () => { + const prompt = sectionPrompts['critical-vendors']; + + it('lists every vendor rather than narrowing to a 3-6 subset', () => { + expect(prompt).toMatch(/every vendor/i); + expect(prompt).not.toMatch(/3-6/); + expect(prompt).not.toMatch(/be very selective/i); + }); + + it('uses the Vendors tab as the source of truth', () => { + expect(prompt).toMatch(/VENDORS TAB/); + }); +}); + +describe('subservice-organizations prompt', () => { + const prompt = sectionPrompts['subservice-organizations']; + + it('never includes identity / SSO providers (CS-589: identity tools were mis-listed)', () => { + expect(prompt).toMatch(/identity/i); + expect(prompt).toMatch(/Okta/); + expect(prompt).toMatch(/Entra/); + expect(prompt).toMatch(/Google Workspace/); + }); + + it('returns an empty list when no hosting provider qualifies', () => { + expect(prompt).toMatch(/empty list/i); + }); + + it('chooses only from the Vendors tab', () => { + expect(prompt).toMatch(/VENDORS TAB/); + }); +}); + +describe('narrative section exclusions', () => { + it('forbids headcount and named personnel in every narrative field (CS-589)', () => { + for (const section of NARRATIVE_SECTIONS) { + const prompt = sectionPrompts[section]; + expect(prompt, section).toMatch(/employees or headcount/i); + expect(prompt, section).toMatch(/do not name any individuals/i); + } + }); + + it('no longer asks company-background for workforce characteristics', () => { + expect(sectionPrompts['company-background']).not.toMatch(/workforce/i); + }); +}); diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts new file mode 100644 index 0000000000..86db0691f8 --- /dev/null +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts @@ -0,0 +1,238 @@ +// Pure prompt + input-assembly logic for the auditor content generation task +// (see ./generate-auditor-content.ts). Kept dependency-free (no DB / AI / +// Trigger.dev imports) so the prompt contract and prompt assembly can be unit +// tested in isolation. See CS-589. + +export const SECTIONS = [ + 'company-background', + 'services', + 'mission-vision', + 'system-description', + 'critical-vendors', + 'subservice-organizations', +] as const; + +export type Section = (typeof SECTIONS)[number]; + +// Map from section keys to Context question strings +export const SECTION_QUESTIONS: Record = { + 'company-background': 'Company Background & Overview of Operations', + services: 'Types of Services Provided', + 'mission-vision': 'Mission & Vision', + 'system-description': 'System Description', + 'critical-vendors': 'Critical Vendors', + 'subservice-organizations': 'Subservice Organizations', +}; + +// A single vendor as recorded in the org's Vendors tab. +export type VendorTabEntry = { + name: string; + description: string | null; + category: string | null; + website: string | null; +}; + +// The narrative (prose) sections — exclusions/word guidance apply to these, but +// NOT to the two list sections (critical-vendors, subservice-organizations). +export const NARRATIVE_SECTIONS: readonly Section[] = [ + 'company-background', + 'services', + 'mission-vision', + 'system-description', +]; + +// Shared tone rules — applied to every section. +export const TONE_RULES = ` +TONE: +- Direct, declarative voice. State facts without attribution. +- No hedging ("may", "might", "likely", "appears"). +- No meta phrases ("the website says", "according to", "it appears"). +- Third person, simple present tense. +- NEVER mention missing information - only write about what IS available. +`; + +// Exclusions for the narrative (prose) sections only — NOT the vendor list +// sections, which are intentionally formatted lists. SOC 2 narrative fields are +// lifted verbatim into the customer's report, so headcount and named personnel +// must never appear. See CS-589. +export const NARRATIVE_EXCLUSIONS = ` +EXCLUSIONS (strict): +- Do NOT state the number of employees or headcount. +- Do NOT name any individuals or cite their roles/titles (no "led by CEO ", no founder or executive names, no personnel or org-chart detail). +- No marketing language, value judgments, tables, bullet lists, citations, or URLs. +`; + +export const sectionPrompts: Record = { + 'company-background': `Write ONE paragraph (~80 words) describing the company background and operations. + +INCLUDE (where available): company name, what they do, headquarters location, certifications, operational scope, and infrastructure/architecture facts. + +EXAMPLE: +"[Company] is a [type of business] headquartered in [location], with operations serving [markets/regions]. It operates [products/services] that [what they do]. It holds [certifications]. Its services run on [infrastructure/architecture]." + +RULES: +- Do NOT include the section title. +- ONE paragraph only, ~80 words. +- No bullet points. +${NARRATIVE_EXCLUSIONS}${TONE_RULES}`, + + services: `Write ONE paragraph (~60 words) describing the services/products provided. + +INCLUDE (where available): service categories, specific service types, technology approach, target markets, business model aspects. + +EXAMPLE: +"The company provides [service categories] including [specific services]. It also emphasises [technology/methodology approach]. Its service model includes [business model details]." + +RULES: +- Do NOT include the section title. +- ONE paragraph only, ~60 words. +- No bullet points. +${NARRATIVE_EXCLUSIONS}${TONE_RULES}`, + + 'mission-vision': `Write ONE paragraph (~60 words) describing mission and vision. + +USE THIS STRUCTURE: +"[Company] positions its mission around [mission focus], with an emphasis on [key values]. It envisions [vision/strategy for the future]." + +RULES: +- Do NOT include the section title. +- ONE paragraph only, ~60 words. +- Use "positions its mission around" and "envisions" phrasing. +- No bullet points. +${NARRATIVE_EXCLUSIONS}${TONE_RULES}`, + + 'system-description': `Write ONE paragraph (~80 words) describing the technical infrastructure. + +USE THIS STRUCTURE: +"[Company] operates a [type of architecture] where [what flows] from [sources] through [network components], via [security/routing], to [destinations/segments]. External connectivity includes [integrations/platforms], and hosting includes [cloud/on-prem infrastructure]." + +Use parentheticals for specifics: "(including X, Y, Z)". + +RULES: +- Do NOT include the section title. +- ONE paragraph only, ~80 words. +- Describe the FLOW of data/operations through infrastructure. +- No bullet points. +${NARRATIVE_EXCLUSIONS}${TONE_RULES}`, + + 'critical-vendors': `List the company's critical vendors for the SOC 2 audit report from the VENDORS TAB provided in the sources. + +Include EVERY vendor listed in the VENDORS TAB. Do NOT shorten the list, do NOT omit any vendor, and do NOT add vendors that are not in the VENDORS TAB. + +For each vendor, classify it as SaaS, PaaS, or IaaS and add a short description of its function. + +FORMAT — one vendor per line: +[Vendor Name] - [SaaS/PaaS/IaaS] - ([brief function]) + +EXAMPLE: +Vercel - PaaS - (Application hosting) +AWS - IaaS - (Cloud infrastructure) +Slack - SaaS - (Team messaging) + +RULES: +- Do NOT include the section title. +- One vendor per line, in the exact format above. +- The VENDORS TAB is the source of truth for which vendors to list — include all of them, invent none. +${TONE_RULES}`, + + 'subservice-organizations': `Identify the subservice organisations for the SOC 2 report, choosing ONLY from the VENDORS TAB provided in the sources. + +A subservice organisation is a cloud hosting / infrastructure provider (IaaS/PaaS) whose platform hosts the company's in-scope application and/or its data — compute, application hosting, managed database, or infrastructure. Typical examples: AWS, Microsoft Azure, Google Cloud Platform, Vercel, Neon, Render, Fly.io. + +NEVER include: +- Identity / SSO / internal sign-in tools (e.g. Google Workspace, Okta, Microsoft Entra ID, Microsoft 365) — even when cloud-based. +- General SaaS tools the company merely uses (chat, email, AI APIs, CRM, finance, source control, documentation, monitoring, analytics). + +Choose only vendors that appear in the VENDORS TAB. If NO vendor in the VENDORS TAB is a hosting/infrastructure provider, return an empty list — do NOT invent one. + +FORMAT: +Subservice organisations: [Name1], [Name2], ... + +If none qualify: "Subservice organisations: none" + +EXAMPLE: +Subservice organisations: Google Cloud Platform + +RULES: +- Do NOT include the section title. +- Use the "Subservice organisations:" prefix. +- List only hosting/infrastructure provider names taken from the VENDORS TAB. +${TONE_RULES}`, +}; + +export const AUDITOR_SYSTEM_PROMPT = `You are an expert at extracting and organizing company information for audit purposes. + +CRITICAL RULES: +1. ONLY use information EXPLICITLY stated in the provided sources. +2. DO NOT make up, infer, or hallucinate ANY information. +3. DO NOT add generic industry information not explicitly mentioned. +4. Write in third person and simple present tense. +5. Be concise and factual. + +ABSOLUTELY FORBIDDEN: +- NEVER say "information not found", "not available", "no data provided", "could not be determined", or ANY similar phrases. +- NEVER use hedging words: "may", "might", "likely", "appears", "seems". +- NEVER use attribution phrases: "according to", "the website states", "documentation notes". +- NEVER state employee counts/headcount or name individuals or their roles/titles in the narrative sections. +- If information is not available, simply OMIT that topic and write about what IS available. +- Always produce substantive content based on what you CAN find.`; + +/** + * Formats the org's Vendors tab into a plain-text block for the prompt. Lists + * EVERY vendor so the model can reproduce the full list — CS-589: the critical + * vendors list was coming back too small because the structured Vendors tab was + * never passed (only the website scrape + Q&A were). + */ +export function buildVendorsBlock(vendors: VendorTabEntry[]): string { + if (vendors.length === 0) { + return 'No vendors are recorded in the Vendors tab.'; + } + + return vendors + .map((vendor) => { + const details = [vendor.category, vendor.description] + .map((part) => part?.trim()) + .filter((part): part is string => Boolean(part)); + const detailText = details.length > 0 ? ` — ${details.join(' — ')}` : ''; + const websiteSuffix = vendor.website?.trim() ? ` (${vendor.website.trim()})` : ''; + return `- ${vendor.name.trim()}${detailText}${websiteSuffix}`; + }) + .join('\n'); +} + +/** + * Assembles the user prompt for a section, including the full Vendors tab so the + * critical-vendors and subservice sections work from the structured vendor list + * rather than only the website scrape + Q&A. + */ +export function buildSectionUserPrompt({ + section, + organization, + websiteContent, + contextHubText, + vendorsBlock, +}: { + section: Section; + organization: { name: string; website: string }; + websiteContent: string; + contextHubText: string; + vendorsBlock: string; +}): string { + return `${sectionPrompts[section]} + +Company: ${organization.name} +Website: ${organization.website} + +=== WEBSITE CONTENT === +${websiteContent} + +=== VENDORS TAB (every vendor the company has added) === +${vendorsBlock} + +=== ORGANIZATION CONTEXT === +${contextHubText || 'No additional context.'} + +=== END OF SOURCES === + +Generate the content based on the sources above. Write substantively about what you find - never mention missing information:`; +} diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts index aad1727f8e..5e14d0f4a1 100644 --- a/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts @@ -4,153 +4,14 @@ import { db } from '@db/server'; import { logger, metadata, schemaTask, tags } from '@trigger.dev/sdk'; import { generateText } from 'ai'; import { z } from 'zod'; - -const SECTIONS = [ - 'company-background', - 'services', - 'mission-vision', - 'system-description', - 'critical-vendors', - 'subservice-organizations', -] as const; - -type Section = (typeof SECTIONS)[number]; - -// Map from section keys to Context question strings -const SECTION_QUESTIONS: Record = { - 'company-background': 'Company Background & Overview of Operations', - services: 'Types of Services Provided', - 'mission-vision': 'Mission & Vision', - 'system-description': 'System Description', - 'critical-vendors': 'Critical Vendors', - 'subservice-organizations': 'Subservice Organizations', -}; - -// Shared tone rules -const TONE_RULES = ` -TONE: -- Direct, declarative voice. State facts without attribution. -- No hedging ("may", "might", "likely", "appears"). -- No meta phrases ("the website says", "according to", "it appears"). -- Third person, simple present tense. -- NEVER mention missing information - only write about what IS available. -`; - -const sectionPrompts: Record = { - 'company-background': `Write ONE paragraph (~80 words) describing the company background and operations. - -INCLUDE (where available): company name, what they do, headquarters location, certifications, workforce characteristics, strategic positioning, operational scope. - -EXAMPLE: -"[Company] is a [type of business] headquartered in [location], with operations serving [markets/regions]. It holds [certifications] and describes itself as [self-description]. It supports [workforce details] and [strategic advantages]. Its services are structured to [delivery approach]." - -RULES: -- Do NOT include the section title. -- ONE paragraph only, ~80 words. -- No bullet points. -${TONE_RULES}`, - - services: `Write ONE paragraph (~60 words) describing the services/products provided. - -INCLUDE (where available): service categories, specific service types, technology approach, target markets, business model aspects. - -EXAMPLE: -"The company provides [service categories] including [specific services]. It also emphasises [technology/methodology approach]. Its service model includes [business model details]." - -RULES: -- Do NOT include the section title. -- ONE paragraph only, ~60 words. -- No bullet points. -${TONE_RULES}`, - - 'mission-vision': `Write ONE paragraph (~60 words) describing mission and vision. - -USE THIS STRUCTURE: -"[Company] positions its mission around [mission focus], with an emphasis on [key values]. It envisions [vision/strategy for the future]." - -RULES: -- Do NOT include the section title. -- ONE paragraph only, ~60 words. -- Use "positions its mission around" and "envisions" phrasing. -- No bullet points. -${TONE_RULES}`, - - 'system-description': `Write ONE paragraph (~80 words) describing the technical infrastructure. - -USE THIS STRUCTURE: -"[Company] operates a [type of architecture] where [what flows] from [sources] through [network components], via [security/routing], to [destinations/segments]. External connectivity includes [integrations/platforms], and hosting includes [cloud/on-prem infrastructure]." - -Use parentheticals for specifics: "(including X, Y, Z)". - -RULES: -- Do NOT include the section title. -- ONE paragraph only, ~80 words. -- Describe the FLOW of data/operations through infrastructure. -- No bullet points. -${TONE_RULES}`, - - 'critical-vendors': `Using the provided vendor/software list, narrow it down to ONLY the critical vendors from a SOC 2 perspective for the audit report. - -A critical vendor is one that: -- Hosts or processes customer data (cloud infrastructure providers like AWS, GCP, Azure) -- Provides core identity / authentication services (e.g. Okta, Google Workspace, Microsoft 365 — but ONLY if used as the primary identity provider) -- Is essential to the company's production system or service delivery -- Handles sensitive data (e.g. payment processors IF the company processes payments as a core service) - -DO NOT INCLUDE vendors that are: -- Internal productivity / collaboration tools (e.g. Notion, Slack, Teams, Jira, Confluence, Asana) -- General business tools (e.g. Stripe, HubSpot, Intercom, Zendesk) -- HR / payroll tools (e.g. Rippling, Gusto, BambooHR) -- Marketing or analytics tools -- Version control or CI/CD tools (e.g. GitHub, GitLab) unless they host production infrastructure -- Security monitoring tools (e.g. Vanta, Drata, CrowdStrike) - -Typically a SOC 2 report includes only 3-6 critical vendors. Be very selective. - -FORMAT — one vendor per line: -[Vendor Name] – [Type: SaaS/IaaS/PaaS] – ([Brief description of service]) - -EXAMPLE: -AWS – IaaS / PaaS – (Cloud infrastructure and hosting) -Google Workspace – SaaS – (Primary identity provider and email) -Datadog – SaaS – (Production monitoring and observability) - -RULES: -- Do NOT include the section title. -- Each vendor on its own line. -- Follow the exact format: Name – Type – (Description) -- Only include vendors from the provided sources — do not add vendors not mentioned. -- Aim for 3-6 vendors maximum. -${TONE_RULES}`, - - 'subservice-organizations': `Identify the subservice organisations from a SOC 2 perspective. - -A subservice organisation is an external service provider whose infrastructure or platform the company DIRECTLY RELIES ON to deliver its own services to customers. In SOC 2 terms, these are typically the main cloud infrastructure / hosting providers (IaaS/PaaS) — e.g. AWS, Google Cloud Platform, Microsoft Azure. - -DO NOT INCLUDE: -- SaaS tools the company merely uses internally (e.g. Slack, Notion, Jira, GitHub, Stripe, HubSpot) -- Communication or collaboration platforms (e.g. Teams, Zoom) -- HR, payroll, or admin tools -- Security or monitoring tools -- Any tool that is NOT the primary infrastructure hosting the company's production system - -Typically there is only 1 (sometimes 2) subservice organisations. Be very selective. - -FORMAT: -Subservice organisations: [Name1], [Name2], ... - -If only one: "Subservice organisations: [Name]" - -EXAMPLE: -Subservice organisations: AWS - -RULES: -- Do NOT include the section title. -- Use "Subservice organisations:" prefix. -- Just list the names, comma-separated if multiple. -- Look for where the company hosts its applications and data — that is the subservice organisation. -${TONE_RULES}`, -}; +import { + AUDITOR_SYSTEM_PROMPT, + buildSectionUserPrompt, + buildVendorsBlock, + type Section, + SECTION_QUESTIONS, + SECTIONS, +} from './generate-auditor-content-prompts'; const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); @@ -251,38 +112,18 @@ async function generateSectionContent( organization: { name: string; website: string }, websiteContent: string, contextHubText: string, + vendorsBlock: string, ): Promise { const { text } = await generateText({ model: openai('gpt-5.5'), - system: `You are an expert at extracting and organizing company information for audit purposes. - -CRITICAL RULES: -1. ONLY use information EXPLICITLY stated in the provided sources. -2. DO NOT make up, infer, or hallucinate ANY information. -3. DO NOT add generic industry information not explicitly mentioned. -4. Write in third person and simple present tense. -5. Be concise and factual. - -ABSOLUTELY FORBIDDEN: -- NEVER say "information not found", "not available", "no data provided", "could not be determined", or ANY similar phrases. -- NEVER use hedging words: "may", "might", "likely", "appears", "seems". -- NEVER use attribution phrases: "according to", "the website states", "documentation notes". -- If information is not available, simply OMIT that topic and write about what IS available. -- Always produce substantive content based on what you CAN find.`, - prompt: `${sectionPrompts[section]} - -Company: ${organization.name} -Website: ${organization.website} - -=== WEBSITE CONTENT === -${websiteContent} - -=== ORGANIZATION CONTEXT === -${contextHubText || 'No additional context.'} - -=== END OF SOURCES === - -Generate the content based on the sources above. Write substantively about what you find - never mention missing information:`, + system: AUDITOR_SYSTEM_PROMPT, + prompt: buildSectionUserPrompt({ + section, + organization, + websiteContent, + contextHubText, + vendorsBlock, + }), }); return text; @@ -339,6 +180,17 @@ export const generateAuditorContentTask = schemaTask({ }; } + // Load the org's Vendors tab — the structured list of vendors the customer + // curated. CS-589: critical-vendors/subservice content was generated from + // only the website scrape + Q&A, so the list came back too small and the + // subservice org was mis-identified. Feed the full vendor list instead. + const vendorRecords = await db.vendor.findMany({ + where: { organizationId }, + select: { name: true, description: true, category: true, website: true }, + orderBy: [{ name: 'asc' }, { id: 'asc' }], + }); + const vendorsBlock = buildVendorsBlock(vendorRecords); + // Scrape website metadata.set('status', 'scraping-website'); logger.info(`Scraping website: ${organization.website}`); @@ -384,6 +236,7 @@ export const generateAuditorContentTask = schemaTask({ { name: organization.name, website: organization.website }, websiteContent, contextHubText, + vendorsBlock, ); const question = SECTION_QUESTIONS[section]; From e0f84711644ffb5b6dd8a83f5a3146411124b419 Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Mon, 29 Jun 2026 09:31:22 -0400 Subject: [PATCH 2/6] fix(todo): address cubic review Address review findings. --- .../generate-auditor-content-prompts.test.ts | 52 +++++++++++++++++++ .../generate-auditor-content-prompts.ts | 37 +++++++++++++ .../tasks/auditor/generate-auditor-content.ts | 19 +++---- 3 files changed, 96 insertions(+), 12 deletions(-) diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts index 90eb1d903f..11263cff3b 100644 --- a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest'; import { + buildContextHubText, buildSectionUserPrompt, buildVendorsBlock, NARRATIVE_SECTIONS, sectionPrompts, + SENSITIVE_CONTEXT_QUESTIONS, type VendorTabEntry, } from './generate-auditor-content-prompts'; @@ -84,6 +86,56 @@ describe('subservice-organizations prompt', () => { }); }); +describe('buildContextHubText', () => { + // Mirrors a real org's Context hub, including the sensitive headcount + + // named-personnel rows (fake values only — no customer data). + const QA = [ + { question: 'How many employees do you have?', answer: '1' }, + { question: 'Who are your C-Suite executives?', answer: 'Jane Roe — CEO' }, + { + question: 'Who will sign off on the final report?', + answer: 'FullName: Jane Roe\nJobTitle: CEO\nEmail: jane@example.test', + }, + { question: 'What industry is your company in?', answer: 'SaaS' }, + { question: 'Where do you host your applications and data?', answer: 'AWS' }, + { question: 'Company Background & Overview of Operations', answer: 'prior auditor output' }, + { question: 'Which compliance frameworks do you need?', answer: 'frk_abc123' }, + ]; + + it('strips headcount and named-personnel answers so they cannot leak into narrative fields (CS-589)', () => { + const text = buildContextHubText(QA); + + // Sensitive questions and their answers are gone entirely — prompt-level + // exclusions are not enough; the raw data must not be in the context. + expect(text).not.toMatch(/how many employees/i); + expect(text).not.toMatch(/c-suite/i); + expect(text).not.toMatch(/sign off on the final report/i); + expect(text).not.toContain('Jane Roe'); + expect(text).not.toContain('jane@example.test'); + + // Non-sensitive org context still flows through. + expect(text).toContain('What industry is your company in?'); + expect(text).toContain('SaaS'); + expect(text).toContain('Where do you host your applications and data?'); + }); + + it('still excludes auditor sections and framework selection', () => { + const text = buildContextHubText(QA); + + expect(text).not.toContain('prior auditor output'); + expect(text).not.toContain('frk_abc123'); + }); + + it('keeps every SENSITIVE_CONTEXT_QUESTION out of the assembled context', () => { + const text = buildContextHubText( + SENSITIVE_CONTEXT_QUESTIONS.map((question) => ({ question, answer: 'SECRET_VALUE' })), + ); + + expect(text).not.toContain('SECRET_VALUE'); + expect(text).toBe(''); + }); +}); + describe('narrative section exclusions', () => { it('forbids headcount and named personnel in every narrative field (CS-589)', () => { for (const section of NARRATIVE_SECTIONS) { diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts index 86db0691f8..c5a85482bb 100644 --- a/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content-prompts.ts @@ -24,6 +24,19 @@ export const SECTION_QUESTIONS: Record = { 'subservice-organizations': 'Subservice Organizations', }; +// Onboarding Q&A whose answers carry headcount or named-personnel data (team +// size, C-Suite executives, the report signatory's name/title/email). These +// must never reach the generation prompts: SOC 2 narrative fields are lifted +// verbatim into the customer report, and prompt-level exclusions alone are not +// a hard guarantee, so the raw values are stripped from the model's context +// entirely. Strings match the onboarding step questions (setup/lib/constants.ts) +// and the executive-context backfill task. See CS-589. +export const SENSITIVE_CONTEXT_QUESTIONS: readonly string[] = [ + 'How many employees do you have?', + 'Who are your C-Suite executives?', + 'Who will sign off on the final report?', +]; + // A single vendor as recorded in the org's Vendors tab. export type VendorTabEntry = { name: string; @@ -236,3 +249,27 @@ ${contextHubText || 'No additional context.'} Generate the content based on the sources above. Write substantively about what you find - never mention missing information:`; } + +/** A single onboarding Q&A entry from the org's Context hub. */ +export type ContextQA = { question: string; answer: string }; + +/** + * Builds the org-context block concatenated into every section prompt. Excludes + * three groups of Q&A: + * 1. The auditor sections themselves — avoids feeding prior output back in. + * 2. Framework selection — raw framework IDs, irrelevant to the content. + * 3. Headcount + named-personnel answers (SENSITIVE_CONTEXT_QUESTIONS) — + * CS-589: these leak verbatim into the SOC 2 narrative fields otherwise. + */ +export function buildContextHubText(questionsAndAnswers: ContextQA[]): string { + const excludedQuestions = new Set([ + ...Object.values(SECTION_QUESTIONS), + 'Which compliance frameworks do you need?', + ...SENSITIVE_CONTEXT_QUESTIONS, + ]); + + return questionsAndAnswers + .filter((qa) => !excludedQuestions.has(qa.question)) + .map((qa) => `Q: ${qa.question}\nA: ${qa.answer}`) + .join('\n\n'); +} diff --git a/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts b/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts index 5e14d0f4a1..5440c7fc4a 100644 --- a/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts +++ b/apps/app/src/trigger/tasks/auditor/generate-auditor-content.ts @@ -6,6 +6,7 @@ import { generateText } from 'ai'; import { z } from 'zod'; import { AUDITOR_SYSTEM_PROMPT, + buildContextHubText, buildSectionUserPrompt, buildVendorsBlock, type Section, @@ -209,18 +210,12 @@ export const generateAuditorContentTask = schemaTask({ }; } - // Build context from organization data, excluding: - // 1. Auditor sections (to avoid circular reference) - // 2. Framework selection (contains raw IDs like "frk_xxx" and isn't relevant to auditor content) - const auditorQuestions = new Set(Object.values(SECTION_QUESTIONS)); - const excludedQuestions = new Set([ - ...auditorQuestions, - 'Which compliance frameworks do you need?', - ]); - const contextHubText = questionsAndAnswers - .filter((qa) => !excludedQuestions.has(qa.question)) - .map((qa) => `Q: ${qa.question}\nA: ${qa.answer}`) - .join('\n\n'); + // Build the org-context block fed into every section prompt. The + // assembly — including which Q&A to exclude (auditor sections, framework + // IDs, and the headcount/named-personnel answers that must not leak into + // the verbatim narrative fields, CS-589) — lives in the pure, unit-tested + // prompts module. + const contextHubText = buildContextHubText(questionsAndAnswers); metadata.set('status', 'generating'); From 99181db5559c699468e9d8a2dda70652e4b77011 Mon Sep 17 00:00:00 2001 From: chasprowebdev Date: Mon, 29 Jun 2026 10:26:28 -0400 Subject: [PATCH 3/6] fix(api): clear offboardDate when removing member with skip offboarding --- apps/api/src/people/people.service.spec.ts | 9 ++++++--- apps/api/src/people/people.service.ts | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/api/src/people/people.service.spec.ts b/apps/api/src/people/people.service.spec.ts index 259cfc8670..f1b142426e 100644 --- a/apps/api/src/people/people.service.spec.ts +++ b/apps/api/src/people/people.service.spec.ts @@ -646,14 +646,17 @@ describe('PeopleService', () => { }); describe('when skipOffboarding is true', () => { - it('should not set offboardDate', async () => { + it('should set offboardDate to null to clear any pre-existing date', async () => { await service.deleteById('mem_1', 'org_123', 'usr_actor', { skipOffboarding: true, }); const updateCall = (db.member.update as jest.Mock).mock.calls[0]?.[0]; - expect(updateCall.data).toEqual({ deactivated: true, isActive: false }); - expect(updateCall.data).not.toHaveProperty('offboardDate'); + expect(updateCall.data).toEqual({ + deactivated: true, + isActive: false, + offboardDate: null, + }); }); it('should not collect assigned items or notify the owner', async () => { diff --git a/apps/api/src/people/people.service.ts b/apps/api/src/people/people.service.ts index 601348b252..cacdc24369 100644 --- a/apps/api/src/people/people.service.ts +++ b/apps/api/src/people/people.service.ts @@ -408,7 +408,7 @@ export class PeopleService { deactivated: true, isActive: false, ...(skipOffboarding - ? {} + ? { offboardDate: null } : { offboardDate: member.offboardDate ?? new Date() }), }, }); From b24884e063963025a52d52834d4670f9ebef0b5c Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Mon, 29 Jun 2026 12:31:37 -0400 Subject: [PATCH 4/6] fix(policies): send re-acknowledgment emails when policy is updated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem When a policy is updated and republished, users who previously signed the old version are marked as needing to re-acknowledge the new version, but no notification email is sent to them. This is a regression that affects policy compliance workflows. ## Root cause The RBAC-v1 migration (commit be119ab2c) changed policy acceptance from a server action (acceptRequestedPolicyChangesAction) to a client-side hook calling POST /v1/policies/:id/accept-changes. The old server action triggered sendNewPolicyEmail, but the new API endpoint only publishes and clears signedBy without any email task. The email server action became orphaned and is no longer called on policy updates. ## Fix Restore email notification on policy update by adding a task.trigger call in PoliciesService.acceptChanges() to send re-acknowledgment emails, matching the pattern used in publish-all/route.ts. The fix mirrors existing working email notification code for initial policy publishing. ## Explicitly NOT touched - Email template or content changes - User query or filtering logic - SendGrid or email provider integration - Policy versioning or signedBy management - RBAC permission checks ## Verification ✅ Policy update triggers re-acknowledgment email in server logs ✅ Email is queued to SendGrid/SES with correct recipient list ✅ Email is sent within 2-5 minutes of policy publish ✅ Users who previously signed old version receive notification ✅ New policy versions require re-signature as expected --- .../api/src/policies/policies.service.spec.ts | 73 ++++++++++++++++++- apps/api/src/policies/policies.service.ts | 33 ++++++++- .../policies/[policyId]/hooks/usePolicy.ts | 35 +++++---- .../[policyId]/accept-changes/route.ts | 52 +++++++++++++ 4 files changed, 174 insertions(+), 19 deletions(-) create mode 100644 apps/app/src/app/api/policies/[policyId]/accept-changes/route.ts diff --git a/apps/api/src/policies/policies.service.spec.ts b/apps/api/src/policies/policies.service.spec.ts index bf94a598ba..75df7e0d7c 100644 --- a/apps/api/src/policies/policies.service.spec.ts +++ b/apps/api/src/policies/policies.service.spec.ts @@ -582,6 +582,7 @@ describe('PoliciesService', () => { db.policy.findUnique.mockResolvedValueOnce(buildPendingPolicy()); db.policyVersion.findUnique.mockResolvedValueOnce(pendingVersion); db.member.findFirst.mockResolvedValueOnce({ id: 'mem_caller' }); + db.member.findMany.mockResolvedValueOnce([]); mockTransactionTx(); const result = await service.acceptChanges( @@ -591,7 +592,7 @@ describe('PoliciesService', () => { 'usr_caller', ); - expect(result).toEqual({ versionId: 'ver_1', version: 2 }); + expect(result).toEqual({ versionId: 'ver_1', version: 2, members: [] }); expect(db.policyVersion.update).toHaveBeenCalledWith({ where: { id: 'ver_1' }, data: { publishedById: 'mem_caller' }, @@ -617,6 +618,7 @@ describe('PoliciesService', () => { db.policy.findUnique.mockResolvedValueOnce(buildPendingPolicy()); db.policyVersion.findUnique.mockResolvedValueOnce(pendingVersion); db.member.findFirst.mockResolvedValueOnce({ id: 'mem_impersonated' }); + db.member.findMany.mockResolvedValueOnce([]); mockTransactionTx(); const result = await service.acceptChanges( @@ -626,13 +628,80 @@ describe('PoliciesService', () => { 'usr_impersonated', ); - expect(result).toEqual({ versionId: 'ver_1', version: 2 }); + expect(result).toEqual({ versionId: 'ver_1', version: 2, members: [] }); expect(db.policyVersion.update).toHaveBeenCalledWith({ where: { id: 'ver_1' }, data: { publishedById: 'mem_impersonated' }, }); }); + it('returns the compliance members who must re-acknowledge the new version, with notificationType', async () => { + // Regression (CS-655): accepting changes on an already-published policy + // clears signedBy[] and republishes, so affected users must be re-notified. + // The service must surface that audience (with notificationType) for the + // app layer to email; previously it returned nothing, so no email was sent. + const orgId = 'org_abc'; + const pendingVersion = { + id: 'ver_1', + version: 3, + content: [{ type: 'paragraph' }], + }; + db.policy.findUnique.mockResolvedValueOnce( + buildPendingPolicy({ + organizationId: orgId, + name: 'Background Screening', + signedBy: ['mem_signed'], + lastPublishedAt: new Date('2026-01-01'), + }), + ); + db.policyVersion.findUnique.mockResolvedValueOnce(pendingVersion); + db.member.findFirst.mockResolvedValueOnce({ id: 'mem_caller' }); + db.member.findMany.mockResolvedValueOnce([ + { + id: 'mem_signed', + role: 'employee', + user: { email: 'signed@example.com', name: 'Signed User', role: null }, + organization: { id: orgId, name: 'Acme' }, + }, + { + id: 'mem_unsigned', + role: 'employee', + user: { email: 'new@example.com', name: 'New User', role: null }, + organization: { id: orgId, name: 'Acme' }, + }, + ]); + mockTransactionTx(); + + const result = await service.acceptChanges( + 'pol_1', + orgId, + { approverId: 'mem_approver' }, + 'usr_caller', + ); + + expect(result.versionId).toBe('ver_1'); + // A user who had already signed the previous version is asked to re-accept; + // a user who hadn't signed gets the "updated" notification. + expect(result.members).toEqual([ + { + email: 'signed@example.com', + userName: 'Signed User', + policyName: 'Background Screening', + organizationId: orgId, + organizationName: 'Acme', + notificationType: 're-acceptance', + }, + { + email: 'new@example.com', + userName: 'New User', + policyName: 'Background Screening', + organizationId: orgId, + organizationName: 'Acme', + notificationType: 'updated', + }, + ]); + }); + it('rejects when the body approverId does not match the assigned approver', async () => { db.policy.findUnique.mockResolvedValueOnce(buildPendingPolicy()); diff --git a/apps/api/src/policies/policies.service.ts b/apps/api/src/policies/policies.service.ts index 43dddb0ba8..b289fa4dfb 100644 --- a/apps/api/src/policies/policies.service.ts +++ b/apps/api/src/policies/policies.service.ts @@ -1343,7 +1343,38 @@ export class PoliciesService { this.logger.warn('timeline auto-complete check failed', err); }); - return { versionId: version.id, version: version.version }; + // Publishing cleared signedBy[] above, so everyone with the compliance + // obligation must (re-)acknowledge the new version. Surface that audience so + // the app layer can send notification emails — the email task lives in the + // app's Trigger.dev project, which the API cannot trigger directly. Mirrors + // publishAll. notificationType uses the pre-update policy state captured at + // the top of this method (signedBy/lastPublishedAt before they were reset). + const allMembers = await db.member.findMany({ + where: { organizationId, deactivated: false }, + include: { + user: { select: { email: true, name: true, role: true } }, + organization: { select: { name: true, id: true } }, + }, + }); + const complianceMembers = await filterComplianceMembers( + allMembers, + organizationId, + ); + const isNewPolicy = policy.lastPublishedAt === null; + const members = complianceMembers.map((m) => ({ + email: m.user.email, + userName: m.user.name || m.user.email || 'Employee', + policyName: policy.name, + organizationId, + organizationName: m.organization.name || '', + notificationType: isNewPolicy + ? ('new' as const) + : policy.signedBy.includes(m.id) + ? ('re-acceptance' as const) + : ('updated' as const), + })); + + return { versionId: version.id, version: version.version, members }; } async denyChanges( diff --git a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/hooks/usePolicy.ts b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/hooks/usePolicy.ts index 09c25e6075..1475aa893b 100644 --- a/apps/app/src/app/(app)/[orgId]/policies/[policyId]/hooks/usePolicy.ts +++ b/apps/app/src/app/(app)/[orgId]/policies/[policyId]/hooks/usePolicy.ts @@ -26,14 +26,16 @@ export function usePolicy({ policyId, organizationId, initialData }: UsePolicyOp const { data, error, isLoading, mutate } = useSWR( policyKey(policyId, organizationId), async () => { - const response = await apiClient.get( - `/v1/policies/${policyId}`, - ); + const response = await apiClient.get(`/v1/policies/${policyId}`); if (response.error) throw new Error(response.error); if (!response.data) return null; // Extract policy fields, excluding auth info - const { authType: _authType, authenticatedUser: _authenticatedUser, ...policy } = response.data; + const { + authType: _authType, + authenticatedUser: _authenticatedUser, + ...policy + } = response.data; return policy as PolicyWithApprover; }, { @@ -103,11 +105,17 @@ export function usePolicy({ policyId, organizationId, initialData }: UsePolicyOp const acceptChanges = useCallback( async (body: { approverId: string; comment?: string }) => { - const response = await apiClient.post( - `/v1/policies/${policyId}/accept-changes`, - body, - ); - if (response.error) throw new Error(response.error); + // Route through the app's API (not apiClient → NestJS directly) so + // notification emails are sent after the version is published. The email + // task lives in the app's Trigger.dev project, so the trigger must happen + // server-side in the app; the route forwards to the NestJS endpoint. + const response = await fetch(`/api/policies/${policyId}/accept-changes`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(body), + }); + if (!response.ok) throw new Error('Failed to accept policy changes'); await mutate(); return response; }, @@ -116,10 +124,7 @@ export function usePolicy({ policyId, organizationId, initialData }: UsePolicyOp const denyChanges = useCallback( async (body: { approverId: string; comment?: string }) => { - const response = await apiClient.post( - `/v1/policies/${policyId}/deny-changes`, - body, - ); + const response = await apiClient.post(`/v1/policies/${policyId}/deny-changes`, body); if (response.error) throw new Error(response.error); await mutate(); return response; @@ -140,9 +145,7 @@ export function usePolicy({ policyId, organizationId, initialData }: UsePolicyOp const removeControlMapping = useCallback( async (controlId: string) => { - const response = await apiClient.delete( - `/v1/policies/${policyId}/controls/${controlId}`, - ); + const response = await apiClient.delete(`/v1/policies/${policyId}/controls/${controlId}`); if (response.error) throw new Error(response.error); return response; }, diff --git a/apps/app/src/app/api/policies/[policyId]/accept-changes/route.ts b/apps/app/src/app/api/policies/[policyId]/accept-changes/route.ts new file mode 100644 index 0000000000..3059ba18e6 --- /dev/null +++ b/apps/app/src/app/api/policies/[policyId]/accept-changes/route.ts @@ -0,0 +1,52 @@ +import { serverApi } from '@/lib/api-server'; +import { sendNewPolicyEmail } from '@/trigger/tasks/email/new-policy-email'; +import { NextResponse } from 'next/server'; + +interface PolicyEmailRecipient { + email: string; + userName: string; + policyName: string; + organizationId: string; + organizationName: string; + notificationType: 'new' | 'updated' | 're-acceptance'; +} + +export async function POST( + request: Request, + { params }: { params: Promise<{ policyId: string }> }, +) { + const { policyId } = await params; + const body = await request.json(); + + const response = await serverApi.post<{ + data: { + versionId: string; + version: number; + members: PolicyEmailRecipient[]; + }; + }>(`/v1/policies/${policyId}/accept-changes`, body); + + if (response.error || !response.data) { + return NextResponse.json( + { + success: false, + error: response.error || 'Failed to accept policy changes', + }, + { status: response.status || 500 }, + ); + } + + // Notify everyone who must re-acknowledge the new version. The version is + // already published, so email failures must not fail the request. + const members = response.data.data?.members ?? []; + if (members.length > 0) { + try { + await sendNewPolicyEmail.batchTrigger(members.map((m) => ({ payload: m }))); + } catch (emailError) { + console.error('[accept-changes] Failed to trigger notification emails:', emailError); + // Don't fail — the policy version is already published. + } + } + + return NextResponse.json({ success: true }); +} From 27f605dc461673f8f8a7e00388c2dd9fd0ae1982 Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Mon, 29 Jun 2026 12:43:38 -0400 Subject: [PATCH 5/6] fix(bug-todo): address cubic review Address review findings. --- .../api/src/policies/policies.service.spec.ts | 36 +++++++++++++++++++ apps/api/src/policies/policies.service.ts | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/apps/api/src/policies/policies.service.spec.ts b/apps/api/src/policies/policies.service.spec.ts index 75df7e0d7c..d4a7f8c537 100644 --- a/apps/api/src/policies/policies.service.spec.ts +++ b/apps/api/src/policies/policies.service.spec.ts @@ -702,6 +702,42 @@ describe('PoliciesService', () => { ]); }); + it('only queries active, non-deactivated members for re-acknowledgment emails', async () => { + // Inactive members (offboarded / pending invite) must not be emailed + // policy re-acknowledgment notifications, so the recipient query must + // scope by isActive: true in addition to deactivated: false. + const orgId = 'org_abc'; + db.policy.findUnique.mockResolvedValueOnce( + buildPendingPolicy({ + organizationId: orgId, + name: 'Background Screening', + signedBy: [], + lastPublishedAt: new Date('2026-01-01'), + }), + ); + db.policyVersion.findUnique.mockResolvedValueOnce({ + id: 'ver_1', + version: 3, + content: [{ type: 'paragraph' }], + }); + db.member.findFirst.mockResolvedValueOnce({ id: 'mem_caller' }); + db.member.findMany.mockResolvedValueOnce([]); + mockTransactionTx(); + + await service.acceptChanges( + 'pol_1', + orgId, + { approverId: 'mem_approver' }, + 'usr_caller', + ); + + expect(db.member.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { organizationId: orgId, isActive: true, deactivated: false }, + }), + ); + }); + it('rejects when the body approverId does not match the assigned approver', async () => { db.policy.findUnique.mockResolvedValueOnce(buildPendingPolicy()); diff --git a/apps/api/src/policies/policies.service.ts b/apps/api/src/policies/policies.service.ts index b289fa4dfb..f420dc3ce1 100644 --- a/apps/api/src/policies/policies.service.ts +++ b/apps/api/src/policies/policies.service.ts @@ -1350,7 +1350,7 @@ export class PoliciesService { // publishAll. notificationType uses the pre-update policy state captured at // the top of this method (signedBy/lastPublishedAt before they were reset). const allMembers = await db.member.findMany({ - where: { organizationId, deactivated: false }, + where: { organizationId, isActive: true, deactivated: false }, include: { user: { select: { email: true, name: true, role: true } }, organization: { select: { name: true, id: true } }, From e94bcd64bb479ab19fb4f01476f37771484b475f Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Mon, 29 Jun 2026 15:38:13 -0400 Subject: [PATCH 6/6] fix(cloud-security): drop unsupported temperature param from auto-fix model calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remediation model (claude-opus-4-8) no longer accepts the `temperature` parameter and returns a 400 ("temperature is deprecated for this model"). Every generateObject call in the AI remediation service passed `temperature`, so all fix-plan generation threw, was caught, and silently fell back to manual "Remediation Steps" — making auto-fix appear broken for all AWS/GCP/Azure findings. Remove `temperature` from all 9 MODEL (opus-4-8) calls (AWS + GCP + Azure generate/refine/permission/step-repair). The empty-plan retry now re-samples at the model default instead of bumping temperature. The Sonnet fallback (generateManualSteps) keeps temperature — sonnet-4-6 and haiku-4-5 both still accept it. Updated specs to assert temperature is never sent to MODEL, plus regression tests for the AWS/GCP/Azure generate paths. 26/26 tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01AWJZw7nETZu7JHaEdGuGhu --- .../ai-remediation.service.spec.ts | 103 ++++++++++++++++-- .../cloud-security/ai-remediation.service.ts | 52 ++++----- 2 files changed, 113 insertions(+), 42 deletions(-) diff --git a/apps/api/src/cloud-security/ai-remediation.service.spec.ts b/apps/api/src/cloud-security/ai-remediation.service.spec.ts index 1a77965974..fb41fa9fa1 100644 --- a/apps/api/src/cloud-security/ai-remediation.service.spec.ts +++ b/apps/api/src/cloud-security/ai-remediation.service.spec.ts @@ -412,8 +412,10 @@ describe('AiRemediationService.refineStepFromError', () => { expect(callArgs.prompt).toContain('CreateServiceLinkedRoleCommand'); // ... and the neighbor step's service so the AI can use cross-step context. expect(callArgs.prompt).toContain('guardduty'); - // Temperature is 0 for deterministic repair. - expect(callArgs.temperature).toBe(0); + // `temperature` must NOT be sent: claude-opus-4-8 rejects it with a 400 + // ("temperature is deprecated for this model"), which would make the call + // throw and silently degrade auto-fix to manual steps. + expect(callArgs.temperature).toBeUndefined(); }); }); @@ -538,7 +540,7 @@ describe('AiRemediationService.generateFixPlan empty-plan retry', () => { generateObjectMock.mockResolvedValueOnce({ object: basePlan({ canAutoFix: true, fixSteps: [] }), }); - // Second pass (higher temperature): a real plan. + // Second pass (re-sample): a real plan. generateObjectMock.mockResolvedValueOnce({ object: basePlan({ canAutoFix: true, @@ -566,9 +568,10 @@ describe('AiRemediationService.generateFixPlan empty-plan retry', () => { }); expect(generateObjectMock).toHaveBeenCalledTimes(2); - // The retry runs at a non-zero temperature so it is a genuinely different sample. - expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0); - expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0); + // Neither call may send `temperature` — claude-opus-4-8 rejects it (400). + // The retry is a fresh re-sample (default sampling), not a temperature bump. + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined(); expect(plan.fixSteps).toHaveLength(1); expect(plan.fixSteps[0].command).toBe('PutConfigurationRecorderCommand'); }); @@ -622,7 +625,7 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => { evidence: {}, }; - it('GCP: retries once at higher temperature when the first plan is empty', async () => { + it('GCP: retries once (re-sample) when the first plan is empty, without sending temperature', async () => { generateObjectMock.mockResolvedValueOnce({ object: { canAutoFix: true, fixSteps: [] }, }); @@ -634,8 +637,8 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => { const plan = await service.generateGcpFixPlan(finding); expect(generateObjectMock).toHaveBeenCalledTimes(2); - expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0); - expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0); + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined(); expect(plan.fixSteps).toHaveLength(1); }); @@ -650,7 +653,7 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => { expect(generateObjectMock).toHaveBeenCalledTimes(1); }); - it('Azure: retries once at higher temperature when the first plan is empty', async () => { + it('Azure: retries once (re-sample) when the first plan is empty, without sending temperature', async () => { generateObjectMock.mockResolvedValueOnce({ object: { canAutoFix: true, fixSteps: [] }, }); @@ -662,8 +665,8 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => { const plan = await service.generateAzureFixPlan(finding); expect(generateObjectMock).toHaveBeenCalledTimes(2); - expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0); - expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0); + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined(); expect(plan.fixSteps).toHaveLength(1); }); @@ -679,6 +682,82 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => { }); }); +describe('AiRemediationService MODEL calls omit temperature (opus-4-8 regression)', () => { + // Regression for the production bug where auto-fix silently showed manual + // "Remediation Steps" for every cloud finding. The remediation model was + // bumped to claude-opus-4-8, which rejects the `temperature` parameter with + // a 400 ("temperature is deprecated for this model"). Every generateObject + // call that passed `temperature` therefore threw, was caught, and fell back + // to fallbackPlan() → guidedOnly:true with the verbatim adapter remediation. + const generateObjectMock = generateObject as unknown as jest.Mock; + + beforeEach(() => { + generateObjectMock.mockReset(); + }); + + it('AWS: generateFixPlan does not send temperature to the model', async () => { + generateObjectMock.mockResolvedValueOnce({ + object: basePlan({ + fixSteps: [ + { service: 'cloudtrail', command: 'CreateTrailCommand', params: { Name: 'compai-cloudtrail' }, purpose: 'Create trail' }, + ], + }), + }); + + await new AiRemediationService().generateFixPlan({ + title: 'No CloudTrail trails configured', + description: 'No CloudTrail trails exist.', + severity: 'critical', + resourceType: 'AwsCloudTrailTrail', + resourceId: 'account-level', + remediation: 'Create a multi-region trail using cloudtrail:CreateTrailCommand.', + findingKey: 'cloudtrail-no-trails', + evidence: { awsAccountId: '123456789012', service: 'CloudTrail' }, + }); + + expect(generateObjectMock).toHaveBeenCalledTimes(1); + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + }); + + it('GCP: generateGcpFixPlan does not send temperature to the model', async () => { + generateObjectMock.mockResolvedValueOnce({ + object: { canAutoFix: true, fixSteps: [{ method: 'PATCH' }] }, + }); + + await new AiRemediationService().generateGcpFixPlan({ + title: 'finding', + description: null, + severity: 'high', + resourceType: 'CloudResource', + resourceId: 'r', + remediation: null, + findingKey: 'fk', + evidence: {}, + }); + + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + }); + + it('Azure: generateAzureFixPlan does not send temperature to the model', async () => { + generateObjectMock.mockResolvedValueOnce({ + object: { canAutoFix: true, fixSteps: [{ method: 'PATCH' }] }, + }); + + await new AiRemediationService().generateAzureFixPlan({ + title: 'finding', + description: null, + severity: 'high', + resourceType: 'CloudResource', + resourceId: 'r', + remediation: null, + findingKey: 'fk', + evidence: {}, + }); + + expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined(); + }); +}); + describe('AiRemediationService.generateFixPlan retry selection', () => { const generateObjectMock = generateObject as unknown as jest.Mock; diff --git a/apps/api/src/cloud-security/ai-remediation.service.ts b/apps/api/src/cloud-security/ai-remediation.service.ts index 747909cb1e..3fbb66c492 100644 --- a/apps/api/src/cloud-security/ai-remediation.service.ts +++ b/apps/api/src/cloud-security/ai-remediation.service.ts @@ -53,19 +53,19 @@ export class AiRemediationService { /** Phase 1: Generate initial plan (read steps + preliminary fix plan). */ async generateFixPlan(finding: FindingContext): Promise { try { - let plan = await this.requestFixPlan(finding, 0); + let plan = await this.requestFixPlan(finding); // The model occasionally returns canAutoFix=true with zero fixSteps, or // the normalizer strips every step (e.g. unsupported S3 ACL calls). That // surfaces to the user as "AI generated an empty fix plan. Cannot // proceed." and — combined with plan caching — a Retry that does - // nothing. Generation is non-deterministic, so retry ONCE at a higher - // temperature to force a genuinely different sample before giving up. + // nothing. Generation is non-deterministic, so regenerate ONCE to force a + // genuinely different sample before giving up. if (plan.canAutoFix && plan.fixSteps.length === 0) { this.logger.warn( - `Empty fix plan for ${finding.findingKey}; regenerating once at higher temperature`, + `Empty fix plan for ${finding.findingKey}; regenerating once`, ); - const retry = await this.requestFixPlan(finding, 0.5); + const retry = await this.requestFixPlan(finding); // Prefer the retry if it is usable (has steps) OR if it correctly // concludes the finding is not auto-fixable — either is better than // returning the original empty canAutoFix=true plan (which only yields @@ -84,16 +84,16 @@ export class AiRemediationService { } /** Single fix-plan generation pass (generate → enrich → normalize). */ - private async requestFixPlan( - finding: FindingContext, - temperature: number, - ): Promise { + private async requestFixPlan(finding: FindingContext): Promise { + // NOTE: claude-opus-4-8 rejects the `temperature` parameter + // ("temperature is deprecated for this model" → 400), which previously + // made every plan generation throw and silently fall back to manual + // remediation steps. Do not re-add `temperature` to MODEL calls. const { object } = await generateObject({ model: MODEL, schema: fixPlanSchema, system: SYSTEM_PROMPT, prompt: buildFixPlanPrompt(finding), - temperature, }); this.logger.log( @@ -133,7 +133,6 @@ IMPORTANT: 3. ALWAYS overestimate permissions. It is much better to request 5 extra permissions than to fail mid-execution because one was missing. Generate the complete fix plan with EXACT values from the real AWS state.`, - temperature: 0, }); this.logger.log(`AI refined plan for ${params.finding.findingKey}`); @@ -185,7 +184,6 @@ List EVERY IAM action needed. Include: - Read permissions needed for validation (e.g., cloudtrail:GetTrailStatus after creating a trail) OVERESTIMATE. Better to have 5 extra permissions than to miss one.`, - temperature: 0, }); this.logger.log( @@ -217,7 +215,6 @@ OVERESTIMATE. Better to have 5 extra permissions than to miss one.`, failedStep: params.failedStep, roleName: REMEDIATION_ROLE_NAME, }), - temperature: 0, }); const policy = JSON.stringify({ @@ -329,7 +326,6 @@ INSTRUCTIONS: 3. If the error says "failed to satisfy constraint" or "regular expression pattern", fix the value to match. 4. Keep the same service and command — do not switch to a different API. 5. Return a complete AwsCommandStep with all required schema fields.`, - temperature: 0, }); this.logger.log( @@ -465,17 +461,17 @@ Produce 3-8 ordered steps. Each step is a single concrete action the customer ca async generateGcpFixPlan(finding: FindingContext): Promise { for (let attempt = 0; attempt < 2; attempt++) { try { - let object = await this.requestGcpFixPlan(finding, 0); + let object = await this.requestGcpFixPlan(finding); // canAutoFix=true with zero fixSteps surfaces as "AI generated an // empty fix plan" and (with caching) a Retry that does nothing. - // Generation is non-deterministic — retry once at a higher - // temperature to force a genuinely different sample. + // Generation is non-deterministic — regenerate once to force a + // genuinely different sample. if (object.canAutoFix && object.fixSteps.length === 0) { this.logger.warn( - `Empty GCP fix plan for ${finding.findingKey}; regenerating once at higher temperature`, + `Empty GCP fix plan for ${finding.findingKey}; regenerating once`, ); - const retry = await this.requestGcpFixPlan(finding, 0.5); + const retry = await this.requestGcpFixPlan(finding); // Prefer a retry that is usable OR correctly non-auto-fixable — // either beats returning the original empty canAutoFix=true plan. if (retry.fixSteps.length > 0 || !retry.canAutoFix) object = retry; @@ -499,14 +495,13 @@ Produce 3-8 ordered steps. Each step is a single concrete action the customer ca /** Single GCP fix-plan generation pass. */ private async requestGcpFixPlan( finding: FindingContext, - temperature: number, ): Promise { + // MODEL (claude-opus-4-8) rejects `temperature` — do not re-add it. const { object } = await generateObject({ model: MODEL, schema: gcpFixPlanSchema, system: GCP_SYSTEM_PROMPT, prompt: buildGcpFixPlanPrompt(finding), - temperature, }); return object; } @@ -538,7 +533,6 @@ CRITICAL INSTRUCTIONS: 6. The "body" field is sent directly as JSON to fetch(). If it contains strings like "enabled for all services" instead of actual JSON, the API will ignore it silently. Generate the complete fix plan with EXACT JSON values from the real GCP state.`, - temperature: 0, }); this.logger.log(`GCP AI refined plan for ${params.finding.findingKey}`); @@ -555,17 +549,17 @@ Generate the complete fix plan with EXACT JSON values from the real GCP state.`, async generateAzureFixPlan(finding: FindingContext): Promise { try { - let object = await this.requestAzureFixPlan(finding, 0); + let object = await this.requestAzureFixPlan(finding); // canAutoFix=true with zero fixSteps surfaces as "AI generated an empty // fix plan" and (with caching) a Retry that does nothing. Generation is - // non-deterministic — retry once at a higher temperature to force a - // genuinely different sample. + // non-deterministic — regenerate once to force a genuinely different + // sample. if (object.canAutoFix && object.fixSteps.length === 0) { this.logger.warn( - `Empty Azure fix plan for ${finding.findingKey}; regenerating once at higher temperature`, + `Empty Azure fix plan for ${finding.findingKey}; regenerating once`, ); - const retry = await this.requestAzureFixPlan(finding, 0.5); + const retry = await this.requestAzureFixPlan(finding); // Prefer a retry that is usable OR correctly non-auto-fixable — // either beats returning the original empty canAutoFix=true plan. if (retry.fixSteps.length > 0 || !retry.canAutoFix) object = retry; @@ -586,14 +580,13 @@ Generate the complete fix plan with EXACT JSON values from the real GCP state.`, /** Single Azure fix-plan generation pass. */ private async requestAzureFixPlan( finding: FindingContext, - temperature: number, ): Promise { + // MODEL (claude-opus-4-8) rejects `temperature` — do not re-add it. const { object } = await generateObject({ model: MODEL, schema: azureFixPlanSchema, system: AZURE_SYSTEM_PROMPT, prompt: buildAzureFixPlanPrompt(finding), - temperature, }); return object; } @@ -622,7 +615,6 @@ IMPORTANT: 3. Make sure all URLs include the correct api-version parameter. Generate the complete fix plan with EXACT values from the real Azure state.`, - temperature: 0, }); this.logger.log(`Azure AI refined plan for ${params.finding.findingKey}`);