Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 91 additions & 12 deletions apps/api/src/cloud-security/ai-remediation.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -412,8 +412,10 @@ describe('AiRemediationService.refineStepFromError', () => {
expect(callArgs.prompt).toContain('CreateServiceLinkedRoleCommand');
// ... and the neighbor step's service so the AI can use cross-step context.
expect(callArgs.prompt).toContain('guardduty');
// Temperature is 0 for deterministic repair.
expect(callArgs.temperature).toBe(0);
// `temperature` must NOT be sent: claude-opus-4-8 rejects it with a 400
// ("temperature is deprecated for this model"), which would make the call
// throw and silently degrade auto-fix to manual steps.
expect(callArgs.temperature).toBeUndefined();
});
});

Expand Down Expand Up @@ -538,7 +540,7 @@ describe('AiRemediationService.generateFixPlan empty-plan retry', () => {
generateObjectMock.mockResolvedValueOnce({
object: basePlan({ canAutoFix: true, fixSteps: [] }),
});
// Second pass (higher temperature): a real plan.
// Second pass (re-sample): a real plan.
generateObjectMock.mockResolvedValueOnce({
object: basePlan({
canAutoFix: true,
Expand Down Expand Up @@ -566,9 +568,10 @@ describe('AiRemediationService.generateFixPlan empty-plan retry', () => {
});

expect(generateObjectMock).toHaveBeenCalledTimes(2);
// The retry runs at a non-zero temperature so it is a genuinely different sample.
expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0);
expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0);
// Neither call may send `temperature` — claude-opus-4-8 rejects it (400).
// The retry is a fresh re-sample (default sampling), not a temperature bump.
expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined();
expect(plan.fixSteps).toHaveLength(1);
expect(plan.fixSteps[0].command).toBe('PutConfigurationRecorderCommand');
});
Expand Down Expand Up @@ -622,7 +625,7 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => {
evidence: {},
};

it('GCP: retries once at higher temperature when the first plan is empty', async () => {
it('GCP: retries once (re-sample) when the first plan is empty, without sending temperature', async () => {
generateObjectMock.mockResolvedValueOnce({
object: { canAutoFix: true, fixSteps: [] },
});
Expand All @@ -634,8 +637,8 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => {
const plan = await service.generateGcpFixPlan(finding);

expect(generateObjectMock).toHaveBeenCalledTimes(2);
expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0);
expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0);
expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined();
expect(plan.fixSteps).toHaveLength(1);
});

Expand All @@ -650,7 +653,7 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => {
expect(generateObjectMock).toHaveBeenCalledTimes(1);
});

it('Azure: retries once at higher temperature when the first plan is empty', async () => {
it('Azure: retries once (re-sample) when the first plan is empty, without sending temperature', async () => {
generateObjectMock.mockResolvedValueOnce({
object: { canAutoFix: true, fixSteps: [] },
});
Expand All @@ -662,8 +665,8 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => {
const plan = await service.generateAzureFixPlan(finding);

expect(generateObjectMock).toHaveBeenCalledTimes(2);
expect(generateObjectMock.mock.calls[0][0].temperature).toBe(0);
expect(generateObjectMock.mock.calls[1][0].temperature).toBeGreaterThan(0);
expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
expect(generateObjectMock.mock.calls[1][0].temperature).toBeUndefined();
expect(plan.fixSteps).toHaveLength(1);
});

Expand All @@ -679,6 +682,82 @@ describe('AiRemediationService GCP/Azure empty-plan retry', () => {
});
});

describe('AiRemediationService MODEL calls omit temperature (opus-4-8 regression)', () => {
// Regression for the production bug where auto-fix silently showed manual
// "Remediation Steps" for every cloud finding. The remediation model was
// bumped to claude-opus-4-8, which rejects the `temperature` parameter with
// a 400 ("temperature is deprecated for this model"). Every generateObject
// call that passed `temperature` therefore threw, was caught, and fell back
// to fallbackPlan() → guidedOnly:true with the verbatim adapter remediation.
const generateObjectMock = generateObject as unknown as jest.Mock;

beforeEach(() => {
generateObjectMock.mockReset();
});

it('AWS: generateFixPlan does not send temperature to the model', async () => {
generateObjectMock.mockResolvedValueOnce({
object: basePlan({
fixSteps: [
{ service: 'cloudtrail', command: 'CreateTrailCommand', params: { Name: 'compai-cloudtrail' }, purpose: 'Create trail' },
],
}),
});

await new AiRemediationService().generateFixPlan({
title: 'No CloudTrail trails configured',
description: 'No CloudTrail trails exist.',
severity: 'critical',
resourceType: 'AwsCloudTrailTrail',
resourceId: 'account-level',
remediation: 'Create a multi-region trail using cloudtrail:CreateTrailCommand.',
findingKey: 'cloudtrail-no-trails',
evidence: { awsAccountId: '123456789012', service: 'CloudTrail' },
});

expect(generateObjectMock).toHaveBeenCalledTimes(1);
expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
});

it('GCP: generateGcpFixPlan does not send temperature to the model', async () => {
generateObjectMock.mockResolvedValueOnce({
object: { canAutoFix: true, fixSteps: [{ method: 'PATCH' }] },
});

await new AiRemediationService().generateGcpFixPlan({
title: 'finding',
description: null,
severity: 'high',
resourceType: 'CloudResource',
resourceId: 'r',
remediation: null,
findingKey: 'fk',
evidence: {},
});

expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
});

it('Azure: generateAzureFixPlan does not send temperature to the model', async () => {
generateObjectMock.mockResolvedValueOnce({
object: { canAutoFix: true, fixSteps: [{ method: 'PATCH' }] },
});

await new AiRemediationService().generateAzureFixPlan({
title: 'finding',
description: null,
severity: 'high',
resourceType: 'CloudResource',
resourceId: 'r',
remediation: null,
findingKey: 'fk',
evidence: {},
});

expect(generateObjectMock.mock.calls[0][0].temperature).toBeUndefined();
});
});

describe('AiRemediationService.generateFixPlan retry selection', () => {
const generateObjectMock = generateObject as unknown as jest.Mock;

Expand Down
52 changes: 22 additions & 30 deletions apps/api/src/cloud-security/ai-remediation.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,19 @@ export class AiRemediationService {
/** Phase 1: Generate initial plan (read steps + preliminary fix plan). */
async generateFixPlan(finding: FindingContext): Promise<FixPlan> {
try {
let plan = await this.requestFixPlan(finding, 0);
let plan = await this.requestFixPlan(finding);

// The model occasionally returns canAutoFix=true with zero fixSteps, or
// the normalizer strips every step (e.g. unsupported S3 ACL calls). That
// surfaces to the user as "AI generated an empty fix plan. Cannot
// proceed." and — combined with plan caching — a Retry that does
// nothing. Generation is non-deterministic, so retry ONCE at a higher
// temperature to force a genuinely different sample before giving up.
// nothing. Generation is non-deterministic, so regenerate ONCE to force a
// genuinely different sample before giving up.
if (plan.canAutoFix && plan.fixSteps.length === 0) {
this.logger.warn(
`Empty fix plan for ${finding.findingKey}; regenerating once at higher temperature`,
`Empty fix plan for ${finding.findingKey}; regenerating once`,
);
const retry = await this.requestFixPlan(finding, 0.5);
const retry = await this.requestFixPlan(finding);
// Prefer the retry if it is usable (has steps) OR if it correctly
// concludes the finding is not auto-fixable — either is better than
// returning the original empty canAutoFix=true plan (which only yields
Expand All @@ -84,16 +84,16 @@ export class AiRemediationService {
}

/** Single fix-plan generation pass (generate → enrich → normalize). */
private async requestFixPlan(
finding: FindingContext,
temperature: number,
): Promise<FixPlan> {
private async requestFixPlan(finding: FindingContext): Promise<FixPlan> {
// NOTE: claude-opus-4-8 rejects the `temperature` parameter
// ("temperature is deprecated for this model" → 400), which previously
// made every plan generation throw and silently fall back to manual
// remediation steps. Do not re-add `temperature` to MODEL calls.
const { object } = await generateObject({
model: MODEL,
schema: fixPlanSchema,
system: SYSTEM_PROMPT,
prompt: buildFixPlanPrompt(finding),
temperature,
});

this.logger.log(
Expand Down Expand Up @@ -133,7 +133,6 @@ IMPORTANT:
3. ALWAYS overestimate permissions. It is much better to request 5 extra permissions than to fail mid-execution because one was missing.

Generate the complete fix plan with EXACT values from the real AWS state.`,
temperature: 0,
});

this.logger.log(`AI refined plan for ${params.finding.findingKey}`);
Expand Down Expand Up @@ -185,7 +184,6 @@ List EVERY IAM action needed. Include:
- Read permissions needed for validation (e.g., cloudtrail:GetTrailStatus after creating a trail)

OVERESTIMATE. Better to have 5 extra permissions than to miss one.`,
temperature: 0,
});

this.logger.log(
Expand Down Expand Up @@ -217,7 +215,6 @@ OVERESTIMATE. Better to have 5 extra permissions than to miss one.`,
failedStep: params.failedStep,
roleName: REMEDIATION_ROLE_NAME,
}),
temperature: 0,
});

const policy = JSON.stringify({
Expand Down Expand Up @@ -329,7 +326,6 @@ INSTRUCTIONS:
3. If the error says "failed to satisfy constraint" or "regular expression pattern", fix the value to match.
4. Keep the same service and command — do not switch to a different API.
5. Return a complete AwsCommandStep with all required schema fields.`,
temperature: 0,
});

this.logger.log(
Expand Down Expand Up @@ -465,17 +461,17 @@ Produce 3-8 ordered steps. Each step is a single concrete action the customer ca
async generateGcpFixPlan(finding: FindingContext): Promise<GcpFixPlan> {
for (let attempt = 0; attempt < 2; attempt++) {
try {
let object = await this.requestGcpFixPlan(finding, 0);
let object = await this.requestGcpFixPlan(finding);

// canAutoFix=true with zero fixSteps surfaces as "AI generated an
// empty fix plan" and (with caching) a Retry that does nothing.
// Generation is non-deterministic — retry once at a higher
// temperature to force a genuinely different sample.
// Generation is non-deterministic — regenerate once to force a
// genuinely different sample.
if (object.canAutoFix && object.fixSteps.length === 0) {
this.logger.warn(
`Empty GCP fix plan for ${finding.findingKey}; regenerating once at higher temperature`,
`Empty GCP fix plan for ${finding.findingKey}; regenerating once`,
);
const retry = await this.requestGcpFixPlan(finding, 0.5);
const retry = await this.requestGcpFixPlan(finding);
// Prefer a retry that is usable OR correctly non-auto-fixable —
// either beats returning the original empty canAutoFix=true plan.
if (retry.fixSteps.length > 0 || !retry.canAutoFix) object = retry;
Expand All @@ -499,14 +495,13 @@ Produce 3-8 ordered steps. Each step is a single concrete action the customer ca
/** Single GCP fix-plan generation pass. */
private async requestGcpFixPlan(
finding: FindingContext,
temperature: number,
): Promise<GcpFixPlan> {
// MODEL (claude-opus-4-8) rejects `temperature` — do not re-add it.
const { object } = await generateObject({
model: MODEL,
schema: gcpFixPlanSchema,
system: GCP_SYSTEM_PROMPT,
prompt: buildGcpFixPlanPrompt(finding),
temperature,
});
return object;
}
Expand Down Expand Up @@ -538,7 +533,6 @@ CRITICAL INSTRUCTIONS:
6. The "body" field is sent directly as JSON to fetch(). If it contains strings like "enabled for all services" instead of actual JSON, the API will ignore it silently.

Generate the complete fix plan with EXACT JSON values from the real GCP state.`,
temperature: 0,
});

this.logger.log(`GCP AI refined plan for ${params.finding.findingKey}`);
Expand All @@ -555,17 +549,17 @@ Generate the complete fix plan with EXACT JSON values from the real GCP state.`,

async generateAzureFixPlan(finding: FindingContext): Promise<AzureFixPlan> {
try {
let object = await this.requestAzureFixPlan(finding, 0);
let object = await this.requestAzureFixPlan(finding);

// canAutoFix=true with zero fixSteps surfaces as "AI generated an empty
// fix plan" and (with caching) a Retry that does nothing. Generation is
// non-deterministic — retry once at a higher temperature to force a
// genuinely different sample.
// non-deterministic — regenerate once to force a genuinely different
// sample.
if (object.canAutoFix && object.fixSteps.length === 0) {
this.logger.warn(
`Empty Azure fix plan for ${finding.findingKey}; regenerating once at higher temperature`,
`Empty Azure fix plan for ${finding.findingKey}; regenerating once`,
);
const retry = await this.requestAzureFixPlan(finding, 0.5);
const retry = await this.requestAzureFixPlan(finding);
// Prefer a retry that is usable OR correctly non-auto-fixable —
// either beats returning the original empty canAutoFix=true plan.
if (retry.fixSteps.length > 0 || !retry.canAutoFix) object = retry;
Expand All @@ -586,14 +580,13 @@ Generate the complete fix plan with EXACT JSON values from the real GCP state.`,
/** Single Azure fix-plan generation pass. */
private async requestAzureFixPlan(
finding: FindingContext,
temperature: number,
): Promise<AzureFixPlan> {
// MODEL (claude-opus-4-8) rejects `temperature` — do not re-add it.
const { object } = await generateObject({
model: MODEL,
schema: azureFixPlanSchema,
system: AZURE_SYSTEM_PROMPT,
prompt: buildAzureFixPlanPrompt(finding),
temperature,
});
return object;
}
Expand Down Expand Up @@ -622,7 +615,6 @@ IMPORTANT:
3. Make sure all URLs include the correct api-version parameter.

Generate the complete fix plan with EXACT values from the real Azure state.`,
temperature: 0,
});

this.logger.log(`Azure AI refined plan for ${params.finding.findingKey}`);
Expand Down
9 changes: 6 additions & 3 deletions apps/api/src/people/people.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -646,14 +646,17 @@ describe('PeopleService', () => {
});

describe('when skipOffboarding is true', () => {
it('should not set offboardDate', async () => {
it('should set offboardDate to null to clear any pre-existing date', async () => {
await service.deleteById('mem_1', 'org_123', 'usr_actor', {
skipOffboarding: true,
});

const updateCall = (db.member.update as jest.Mock).mock.calls[0]?.[0];
expect(updateCall.data).toEqual({ deactivated: true, isActive: false });
expect(updateCall.data).not.toHaveProperty('offboardDate');
expect(updateCall.data).toEqual({
deactivated: true,
isActive: false,
offboardDate: null,
});
});

it('should not collect assigned items or notify the owner', async () => {
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/people/people.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ export class PeopleService {
deactivated: true,
isActive: false,
...(skipOffboarding
? {}
? { offboardDate: null }
: { offboardDate: member.offboardDate ?? new Date() }),
},
});
Expand Down
Loading
Loading