Skip to content

Commit 12467c1

Browse files
authored
feat: add GitHub Actions job summaries for integration tests (#34)
Signed-off-by: Jiaxiao (mossaka) Zhou <duibao55328@gmail.com>
1 parent 497ba7d commit 12467c1

3 files changed

Lines changed: 199 additions & 3 deletions

File tree

.github/workflows/test-integration.yml

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,19 @@ jobs:
6060
run: sudo ./scripts/ci/cleanup.sh
6161

6262
- name: Run basic firewall tests
63-
run: sudo -E npm run test:integration -- basic-firewall.test.ts
63+
id: run-tests
64+
run: |
65+
sudo -E npm run test:integration -- basic-firewall.test.ts 2>&1 | tee test-output.log
66+
continue-on-error: true
67+
68+
- name: Generate test summary
69+
if: always()
70+
run: |
71+
npx tsx scripts/ci/generate-test-summary.ts "basic-firewall.test.ts" "Basic Firewall Tests" test-output.log
72+
73+
- name: Check test results
74+
if: steps.run-tests.outcome == 'failure'
75+
run: exit 1
6476

6577
- name: Post-test cleanup
6678
if: always()
@@ -103,7 +115,19 @@ jobs:
103115
run: sudo ./scripts/ci/cleanup.sh
104116

105117
- name: Run robustness tests
106-
run: sudo -E npm run test:integration -- robustness.test.ts
118+
id: run-tests
119+
run: |
120+
sudo -E npm run test:integration -- robustness.test.ts 2>&1 | tee test-output.log
121+
continue-on-error: true
122+
123+
- name: Generate test summary
124+
if: always()
125+
run: |
126+
npx tsx scripts/ci/generate-test-summary.ts "robustness.test.ts" "Robustness Tests" test-output.log
127+
128+
- name: Check test results
129+
if: steps.run-tests.outcome == 'failure'
130+
run: exit 1
107131

108132
- name: Post-test cleanup
109133
if: always()
@@ -146,7 +170,19 @@ jobs:
146170
run: sudo ./scripts/ci/cleanup.sh
147171

148172
- name: Run docker egress tests
149-
run: sudo -E npm run test:integration -- docker-egress.test.ts
173+
id: run-tests
174+
run: |
175+
sudo -E npm run test:integration -- docker-egress.test.ts 2>&1 | tee test-output.log
176+
continue-on-error: true
177+
178+
- name: Generate test summary
179+
if: always()
180+
run: |
181+
npx tsx scripts/ci/generate-test-summary.ts "docker-egress.test.ts" "Docker Egress Tests" test-output.log
182+
183+
- name: Check test results
184+
if: steps.run-tests.outcome == 'failure'
185+
run: exit 1
150186

151187
- name: Post-test cleanup
152188
if: always()

AGENTS.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,29 @@ This is a firewall for GitHub Copilot CLI (package name: `@github/awf`) that pro
1414

1515
## Development Workflow
1616

17+
### GitHub Actions Best Practices
18+
19+
**IMPORTANT:** When writing or modifying GitHub Actions workflows:
20+
21+
1. **Use TypeScript for workflow scripts, not bash** - All scripts that run in GitHub Actions workflows should be written in TypeScript and executed with `npx tsx`. This ensures:
22+
- Type safety and better IDE support
23+
- Consistency with the rest of the codebase
24+
- Easier testing and maintenance
25+
- Better error handling
26+
27+
2. **Inline script execution** - Run TypeScript scripts directly in workflow steps using `npx tsx path/to/script.ts`, rather than creating bash wrapper scripts. Example:
28+
```yaml
29+
- name: Generate test summary
30+
run: |
31+
npx tsx scripts/ci/generate-test-summary.ts "test-file.ts" "Test Name" test-output.log
32+
```
33+
34+
3. **Place scripts in `scripts/ci/`** - All CI/CD-related scripts should be in the `scripts/ci/` directory and written as TypeScript modules with proper type definitions.
35+
36+
**Example:**
37+
- ❌ Bad: `scripts/ci/generate-summary.sh` (bash script)
38+
- ✅ Good: `scripts/ci/generate-test-summary.ts` (TypeScript script called with `npx tsx`)
39+
1740
### Debugging GitHub Actions Failures
1841

1942
**IMPORTANT:** When GitHub Actions workflows fail, always follow this debugging workflow:
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
#!/usr/bin/env node
2+
/**
3+
* Generate GitHub Actions job summary from Jest test output
4+
* This script parses Jest test output and creates a markdown summary
5+
* showing what scenarios were tested and their results.
6+
*/
7+
8+
import * as fs from 'fs';
9+
import * as path from 'path';
10+
11+
interface TestResults {
12+
passed: number;
13+
failed: number;
14+
total: number;
15+
duration: string;
16+
}
17+
18+
interface TestScenario {
19+
name: string;
20+
passed: boolean;
21+
isGroup?: boolean;
22+
}
23+
24+
function parseTestOutput(output: string): { results: TestResults; scenarios: TestScenario[] } {
25+
const lines = output.split('\n');
26+
27+
// Extract test results from "Tests:" line
28+
const testsLine = lines.find(line => line.startsWith('Tests:'));
29+
let results: TestResults = { passed: 0, failed: 0, total: 0, duration: 'unknown' };
30+
31+
if (testsLine) {
32+
const passedMatch = testsLine.match(/(\d+) passed/);
33+
const failedMatch = testsLine.match(/(\d+) failed/);
34+
const totalMatch = testsLine.match(/(\d+) total/);
35+
36+
results.passed = passedMatch ? parseInt(passedMatch[1], 10) : 0;
37+
results.failed = failedMatch ? parseInt(failedMatch[1], 10) : 0;
38+
results.total = totalMatch ? parseInt(totalMatch[1], 10) : results.passed + results.failed;
39+
}
40+
41+
// Extract duration from "Time:" line
42+
const timeLine = lines.find(line => line.match(/Time:\s+[\d.]+\s*s/));
43+
if (timeLine) {
44+
const timeMatch = timeLine.match(/Time:\s+([\d.]+\s*s)/);
45+
if (timeMatch) {
46+
results.duration = timeMatch[1];
47+
}
48+
}
49+
50+
// Extract test scenarios
51+
const scenarios: TestScenario[] = [];
52+
53+
for (const line of lines) {
54+
// Check for describe blocks (e.g., " 1. Happy-Path Basics")
55+
// These are indented with 4 spaces and don't have ✓ or ✗
56+
if (line.match(/^\s{4}[0-9A-Z]/) && !line.includes('✓') && !line.includes('✗')) {
57+
const groupName = line.trim();
58+
scenarios.push({ name: groupName, passed: true, isGroup: true });
59+
}
60+
// Check for test results (lines with ✓ or ✗)
61+
else if (line.match(/^\s+[]/)) {
62+
const isPassed = line.includes('✓');
63+
// Remove leading whitespace, status symbol, and timing info
64+
const name = line.trim().replace(/^[]\s*/, '').replace(/\s*\(\d+\s*ms\)$/, '');
65+
scenarios.push({ name, passed: isPassed, isGroup: false });
66+
}
67+
}
68+
69+
return { results, scenarios };
70+
}
71+
72+
function generateSummary(testFile: string, testName: string, output: string): string {
73+
const { results, scenarios } = parseTestOutput(output);
74+
75+
// Determine status emoji
76+
const statusEmoji = results.failed === 0 ? '✅' : '❌';
77+
78+
let summary = `## ${statusEmoji} ${testName}\n\n`;
79+
summary += `**Test File:** \`${testFile}\`\n\n`;
80+
summary += `**Results:** ${results.passed} passed, ${results.failed} failed (Total: ${results.total}) in ${results.duration}\n\n`;
81+
82+
if (scenarios.length > 0) {
83+
summary += `### Test Scenarios\n\n`;
84+
85+
for (const scenario of scenarios) {
86+
if (scenario.isGroup) {
87+
summary += `\n**${scenario.name}**\n\n`;
88+
} else {
89+
const emoji = scenario.passed ? '✅' : '❌';
90+
summary += `- ${emoji} ${scenario.name}\n`;
91+
}
92+
}
93+
} else {
94+
summary += `_Test details not available in output_\n\n`;
95+
summary += `<details>\n<summary>Raw Test Output</summary>\n\n\`\`\`\n${output}\n\`\`\`\n\n</details>\n`;
96+
}
97+
98+
return summary;
99+
}
100+
101+
function main() {
102+
const args = process.argv.slice(2);
103+
104+
if (args.length < 2) {
105+
console.error('Usage: generate-test-summary.ts <test-file> <test-name> [<output-file>]');
106+
process.exit(1);
107+
}
108+
109+
const testFile = args[0];
110+
const testName = args[1];
111+
const outputFile = args[2]; // Optional: file containing test output
112+
113+
// Read test output from file or stdin
114+
let testOutput: string;
115+
if (outputFile && fs.existsSync(outputFile)) {
116+
testOutput = fs.readFileSync(outputFile, 'utf-8');
117+
} else {
118+
// Read from stdin
119+
testOutput = fs.readFileSync(0, 'utf-8');
120+
}
121+
122+
// Generate summary
123+
const summary = generateSummary(testFile, testName, testOutput);
124+
125+
// Write to GITHUB_STEP_SUMMARY or stdout
126+
const summaryPath = process.env.GITHUB_STEP_SUMMARY;
127+
if (summaryPath) {
128+
fs.appendFileSync(summaryPath, summary);
129+
console.log('Summary generated successfully');
130+
} else {
131+
console.error('Warning: GITHUB_STEP_SUMMARY not set. Running outside GitHub Actions?');
132+
console.log('\n--- Summary ---');
133+
console.log(summary);
134+
}
135+
}
136+
137+
main();

0 commit comments

Comments
 (0)