Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/commands/test.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2667,6 +2667,40 @@ describe('runSteps', () => {
expect(block.match(/error: /g)).toHaveLength(1);
});

it('--run-id run_failed_sample dry-run sample maps the failed step error and contributor flag', async () => {
const out: string[] = [];
const page = await runSteps(
{
profile: 'default',
output: 'json',
debug: false,
dryRun: true,
testId: 'test_fe',
runId: 'run_failed_sample',
},
{
env: {} as NodeJS.ProcessEnv,
credentialsPath: join(tmpdir(), 'testsprite-no-creds'),
stdout: line => out.push(line),
stderr: () => undefined,
},
);

const failing = page.items.find(step => step.stepIndex === 3);
expect(failing).toMatchObject({
status: 'failed',
error: expect.any(String),
stepType: 'assertion',
outcomeContributesToFailure: true,
});
expect(page.items.find(step => step.stepIndex === 1)?.outcomeContributesToFailure).toBe(false);

const printed = JSON.parse(out[0]!) as { items: Array<{ stepIndex: number; error?: string }> };
expect(printed.items.some(step => step.stepIndex === 3 && typeof step.error === 'string')).toBe(
true,
);
});

it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => {
const { credentialsPath } = makeCreds();
// The run-scoped endpoint returns a run whose testId differs from the
Expand Down
55 changes: 49 additions & 6 deletions src/lib/dry-run/samples.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,44 @@ describe('findSample', () => {
expect(body.stepSummary.failedCount).toBe(0);
});

it('GET /runs/run_failed_sample resolves to the sentinel failed run-scoped step sample', () => {
const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_failed_sample');
expect(e?.operationId).toBe('getRun');
const body = e?.body() as {
status: string;
runId: string;
failedStepIndex: number | null;
failureKind: string | null;
error: string | null;
stepSummary: { total: number; completed: number; passedCount: number; failedCount: number };
steps: Array<{
stepIndex: string;
type: string;
status: string | null;
error: string | null;
}>;
};
expect(body.runId).toBe('run_failed_sample');
expect(body.status).toBe('failed');
expect(body.failedStepIndex).toBe(3);
expect(body.failureKind).toBe('assertion');
expect(body.error).toEqual(expect.any(String));
expect(body.stepSummary).toMatchObject({
total: 3,
completed: 3,
passedCount: 2,
failedCount: 1,
});

const failingStep = body.steps.find(step => step.stepIndex === '0003');
expect(failingStep).toMatchObject({
type: 'assertion',
status: 'failed',
error: expect.any(String),
});
expect(failingStep?.error).not.toBe('');
});

it('getTest sample carries priority field (G1a)', () => {
const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc');
expect(e?.operationId).toBe('getTest');
Expand Down Expand Up @@ -524,13 +562,18 @@ describe('findSample', () => {
expect(body.summary.total).toBeGreaterThanOrEqual(1);
});

it('only one getRun entry exists in the registry (no duplicate)', () => {
// Guards against re-introducing the duplicate by ensuring exactly one
// sample is registered for GET /runs/{runId}.
const matches = DRY_RUN_SAMPLE_ENTRIES.filter(
e => e.method === 'GET' && e.operationId === 'getRun',
it('keeps the failed run sentinel before the generic getRun sample', () => {
// findSample is first-match-wins; exact run fixtures must precede
// `/runs/{runId}` so `test wait --dry-run` still gets the passed sample.
const failedRunIndex = DRY_RUN_SAMPLE_ENTRIES.findIndex(
e => e.method === 'GET' && e.pathTemplate === '/runs/run_failed_sample',
);
const genericRunIndex = DRY_RUN_SAMPLE_ENTRIES.findIndex(
e => e.method === 'GET' && e.pathTemplate === '/runs/{runId}',
);
expect(matches).toHaveLength(1);
expect(failedRunIndex).toBeGreaterThanOrEqual(0);
expect(genericRunIndex).toBeGreaterThanOrEqual(0);
expect(failedRunIndex).toBeLessThan(genericRunIndex);
});

// Input-derived sample tests (Fix #1 — dogfood 2026-05-15)
Expand Down
175 changes: 120 additions & 55 deletions src/lib/dry-run/samples.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ const SAMPLE_TEST_ID_FAILED = 'test_8f2a4d10';
const SAMPLE_TEST_ID_PASSED = 'test_3a91bb02';
const SAMPLE_TEST_ID_BLOCKED = 'test_blocked_4f7a';
export const SAMPLE_RUN_ID = 'run_abc';
// Documented sentinel for `test steps --run-id run_failed_sample --dry-run`:
// keeps wait flows on the default passed sample while still demonstrating a
// run-scoped failed step offline.
const SAMPLE_FAILED_RUN_ID = 'run_failed_sample';
// M3.4 rerun dry-run sample IDs
const SAMPLE_RERUN_ID_BE_NAMED = 'run_rerun_be_named';
const SAMPLE_RERUN_ID_BE_PRODUCER = 'run_rerun_be_producer';
Expand Down Expand Up @@ -341,6 +345,118 @@ const failureSummary: CliFailureSummary = {
recommendedFixTarget: failureContext.failure.recommendedFixTarget,
};

const passedRunSample: RunResponse = {
runId: SAMPLE_RUN_ID,
testId: SAMPLE_TEST_ID_PASSED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'passed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: null,
failureKind: null,
error: null,
videoUrl: null,
stepSummary: {
total: 2,
completed: 2,
passedCount: 2,
failedCount: 0,
},
// Representative per-run steps so `test steps --run-id <id> --dry-run`
// demonstrates real output instead of an empty list (the generic
// `/runs/{runId}` sample is also used by `test wait`, which ignores steps).
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
],
};
Comment thread
coderabbitai[bot] marked this conversation as resolved.

const failedRunSample: RunResponse = {
runId: SAMPLE_FAILED_RUN_ID,
testId: SAMPLE_TEST_ID_FAILED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'failed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: 3,
failureKind: 'assertion',
error: 'Expected billing status badge to be visible, but it was not found.',
videoUrl: null,
stepSummary: {
total: 3,
completed: 3,
passedCount: 2,
failedCount: 1,
},
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
{
stepIndex: '0003',
type: 'assertion',
action: 'assert_visible',
status: 'failed',
description: 'Billing status badge is visible',
error: 'Expected billing status badge to be visible, but it was not found.',
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:30.000Z',
},
],
};

/**
* Dry-run sample lookup keyed by OpenAPI operationId. Order matters in
* {@link findSample}: more specific patterns must precede their generic
Expand Down Expand Up @@ -696,57 +812,8 @@ const ENTRIES: DryRunSampleEntry[] = [
// fix(2026-05-21): a duplicate failed-shape entry that appeared before
// this entry was removed; findSample first-match-wins was always
// returning status: "failed" for `test wait --dry-run`.
entry('getRun', 'GET', '/runs/{runId}', {
runId: SAMPLE_RUN_ID,
testId: SAMPLE_TEST_ID_PASSED,
projectId: SAMPLE_PROJECT_ID,
userId: SAMPLE_USER_ID,
status: 'passed',
source: 'cli',
createdAt: '2026-05-15T19:32:00.000Z',
startedAt: '2026-05-15T19:32:05.000Z',
finishedAt: '2026-05-15T19:34:00.000Z',
codeVersion: 'v1',
targetUrl: SAMPLE_TARGET_URL,
createdFrom: null,
failedStepIndex: null,
failureKind: null,
error: null,
videoUrl: null,
stepSummary: {
total: 8,
completed: 8,
passedCount: 8,
failedCount: 0,
},
// Representative per-run steps so `test steps --run-id <id> --dry-run`
// demonstrates real output instead of an empty list (the generic
// `/runs/{runId}` sample is also used by `test wait`, which ignores steps).
steps: [
{
stepIndex: '0001',
type: 'action',
action: 'navigate',
status: 'passed',
description: 'Open the target URL',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:10.000Z',
},
{
stepIndex: '0002',
type: 'assertion',
action: 'assert_visible',
status: 'passed',
description: 'Dashboard heading is visible',
error: null,
screenshotUrl: null,
htmlSnapshotUrl: null,
createdAt: '2026-05-15T19:32:20.000Z',
},
],
} satisfies RunResponse),
entry('getRun', 'GET', `/runs/${SAMPLE_FAILED_RUN_ID}`, failedRunSample),
entry('getRun', 'GET', '/runs/{runId}', passedRunSample),
];

function entry(
Expand Down Expand Up @@ -798,10 +865,8 @@ export function findSample(
const pathOnly = extractPath(url);
for (const e of ENTRIES) {
if (e.method === upper && e.pattern.test(pathOnly)) {
// Rebind body so callers get the resolved value, not the factory.
// We return a new object with `body` already applied so downstream
// code can keep calling `e.body` as-before (no API break for tests
// that call `findSample` directly).
// Rebind body so downstream code can call `e.body` as before while
// still preserving the original lazy factory semantics.
return { ...e, body: () => e.body(requestBody) };
}
}
Expand Down
Loading