Skip to content

Commit a1e0479

Browse files
gertjanmaasgertjanmaas
committed
WIP commit
1 parent 67f80ad commit a1e0479

15 files changed

Lines changed: 509 additions & 49 deletions

File tree

‎examples/default/main.tf‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,14 @@ module "runners" {
2020
Project = "ProjectX"
2121
}
2222

23-
github_app_webhook_secret = random_password.random.result
23+
github_app_webhook_secret = var.github_app_webhook_secret
2424

25+
github_app_client_id = var.github_app_client_id
26+
github_app_client_secret = var.github_app_client_secret
27+
github_app_id = var.github_app_id
28+
github_app_key_base64 = var.github_app_key_base64
29+
30+
enable_organization_runners = var.enable_organization_runners
2531
}
2632

2733

‎examples/default/variables.tf‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
variable "enable_organization_runners" {
2+
type = bool
3+
}
4+
5+
variable "github_app_key_base64" {}
6+
7+
variable "github_app_id" {}
8+
9+
variable "github_app_client_id" {}
10+
11+
variable "github_app_client_secret" {}
12+
13+
variable "github_app_webhook_secret" {}

‎main.tf‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,15 @@ module "runners" {
3030

3131
s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
3232
sqs = module.agent.sqs
33-
}
3433

35-
module "agent" {
36-
source = "./modules/agent"
34+
github_app_client_id = var.github_app_client_id
35+
github_app_client_secret = var.github_app_client_secret
36+
github_app_id = var.github_app_id
37+
github_app_key_base64 = var.github_app_key_base64
3738

38-
aws_region = var.aws_region
39-
environment = var.environment
40-
tags = var.tags
41-
github_app_webhook_secret = "blaat"
39+
enable_organization_runners = var.enable_organization_runners
4240
}
4341

44-
4542
module "agent" {
4643
source = "./modules/agent"
4744

‎modules/agent/webhook_queue.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
resource "aws_sqs_queue" "webhook_events" {
22
name = "${var.environment}-webhook-events.fifo"
3-
delay_seconds = 30
3+
delay_seconds = 0
44
fifo_queue = true
55
receive_wait_time_seconds = 10
66
content_based_deduplication = true

‎modules/runners/lambdas/scale-runners/package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
"dist": "yarn build && cd dist && zip ../scale-runners.zip index.js"
1212
},
1313
"devDependencies": {
14+
"@types/aws-lambda": "^8.10.51",
1415
"@types/express": "^4.17.3",
1516
"@types/jest": "^25.2.1",
1617
"@types/node": "^13.13.4",
1718
"@zeit/ncc": "^0.22.1",
18-
"aws-sdk": "^2.645.0",
19+
"aws-sdk": "^2.671.0",
1920
"body-parser": "^1.19.0",
2021
"express": "^4.17.1",
2122
"jest": "^25.4.0",
@@ -25,6 +26,7 @@
2526
},
2627
"dependencies": {
2728
"@octokit/auth-app": "^2.4.5",
28-
"@octokit/rest": "^17.6.0"
29+
"@octokit/rest": "^17.6.0",
30+
"yn": "^4.0.0"
2931
}
3032
}

‎modules/runners/lambdas/scale-runners/src/lambda.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
import { handle } from './scale-runners/handler';
2+
import { SQSEvent } from 'aws-lambda';
23

3-
module.exports.handler = async (event: any, context: any, callback: any) => {
4+
module.exports.handler = async (event: SQSEvent, context: any, callback: any) => {
5+
console.log(event);
46
try {
5-
await handle(event.eventSource, JSON.parse(event.body));
7+
for (const e of event.Records) {
8+
await handle(e.eventSource, JSON.parse(e.body));
9+
}
610
return callback(null);
711
} catch (e) {
812
console.error(e);

‎modules/runners/lambdas/scale-runners/src/local.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import express from 'express';
22
import bodyParser from 'body-parser';
3-
import { handle } from './scale-runners/handler';
3+
import { handle, ActionRequestMessage } from './scale-runners/handler';
44

55
const app = express();
66

77
app.use(bodyParser.json());
88

99
app.post('/event_handler', (req, res) => {
10-
handle(req.headers, JSON.stringify(req.body))
11-
.then((c) => res.status(c).end())
10+
handle('aws:sqs', JSON.parse(req.body) as ActionRequestMessage)
11+
.then()
1212
.catch((e) => {
1313
console.log(e);
1414
res.status(404);

‎modules/runners/lambdas/scale-runners/src/scale-runners/handler.test.ts‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,14 @@ import { Octokit } from '@octokit/rest';
66
jest.mock('@octokit/auth-app', () => ({
77
createAppAuth: jest.fn().mockImplementation(() => jest.fn().mockImplementation(() => ({ token: 'Blaat' }))),
88
}));
9-
const mockOctokit = { checks: { get: jest.fn() }, actions: { listRepoWorkflowRuns: jest.fn() } };
9+
const mockOctokit = {
10+
checks: { get: jest.fn() },
11+
actions: {
12+
listRepoWorkflowRuns: jest.fn(),
13+
listSelfHostedRunnersForOrg: jest.fn(),
14+
listSelfHostedRunnersForRepo: jest.fn(),
15+
},
16+
};
1017
jest.mock('@octokit/rest', () => ({
1118
Octokit: jest.fn().mockImplementation(() => mockOctokit),
1219
}));
@@ -21,14 +28,31 @@ const TEST_DATA: ActionRequestMessage = {
2128

2229
describe('handler', () => {
2330
beforeEach(() => {
24-
process.env.GITHUB_APP_KEY = 'TEST_CERTIFICATE_DATA';
31+
process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA';
2532
process.env.GITHUB_APP_ID = '1337';
2633
process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID';
2734
process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET';
2835
jest.clearAllMocks();
2936
mockOctokit.actions.listRepoWorkflowRuns.mockImplementation(() => ({
30-
total_count: 1,
37+
data: {
38+
total_count: 1,
39+
},
3140
}));
41+
const mockRunnersReturnValue = {
42+
data: {
43+
total_count: 1,
44+
runners: [
45+
{
46+
id: 23,
47+
name: 'Test Runner',
48+
status: 'online',
49+
os: 'linux',
50+
},
51+
],
52+
},
53+
};
54+
mockOctokit.actions.listSelfHostedRunnersForOrg.mockImplementation(() => mockRunnersReturnValue);
55+
mockOctokit.actions.listSelfHostedRunnersForRepo.mockImplementation(() => mockRunnersReturnValue);
3256
});
3357

3458
it('ignores non-sqs events', async () => {
@@ -44,4 +68,31 @@ describe('handler', () => {
4468
status: 'queued',
4569
});
4670
});
71+
72+
// describe('on org level', () => {
73+
// beforeAll(() => {
74+
// process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
75+
// });
76+
77+
// it('gets the current org level runners', async () => {
78+
// await handle('aws:sqs', TEST_DATA);
79+
// expect(mockOctokit.actions.listSelfHostedRunnersForOrg).toBeCalledWith({
80+
// org: TEST_DATA.repositoryOwner,
81+
// });
82+
// });
83+
// });
84+
85+
// describe('on repo level', () => {
86+
// beforeAll(() => {
87+
// delete process.env.ENABLE_ORGANIZATION_RUNNERS;
88+
// });
89+
90+
// it('gets the current repo level runners', async () => {
91+
// await handle('aws:sqs', TEST_DATA);
92+
// expect(mockOctokit.actions.listSelfHostedRunnersForRepo).toBeCalledWith({
93+
// owner: TEST_DATA.repositoryOwner,
94+
// repo: TEST_DATA.repositoryName,
95+
// });
96+
// });
97+
// });
4798
});

‎modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { createAppAuth } from '@octokit/auth-app';
22
import { Octokit } from '@octokit/rest';
3+
import { AppAuth } from '@octokit/auth-app/dist-types/types';
4+
import yn from 'yn';
35

46
export interface ActionRequestMessage {
57
id: number;
@@ -9,40 +11,52 @@ export interface ActionRequestMessage {
911
installationId: number;
1012
}
1113

12-
async function createGithubClient(installationId: number): Promise<Octokit> {
13-
const privateKey = process.env.GITHUB_APP_KEY as string;
14+
function createGithubAppAuth(installationId: number): AppAuth {
15+
const privateKey = Buffer.from(process.env.GITHUB_APP_KEY_BASE64 as string, 'base64').toString();
1416
const appId: number = parseInt(process.env.GITHUB_APP_ID as string);
1517
const clientId = process.env.GITHUB_APP_CLIENT_ID as string;
1618
const clientSecret = process.env.GITHUB_APP_CLIENT_SECRET as string;
1719

18-
try {
19-
const auth = createAppAuth({
20-
id: appId,
21-
privateKey: privateKey,
22-
installationId: installationId,
23-
clientId: clientId,
24-
clientSecret: clientSecret,
25-
});
26-
const installationAuthentication = await auth({ type: 'installation' });
20+
return createAppAuth({
21+
id: appId,
22+
privateKey: privateKey,
23+
installationId: installationId,
24+
clientId: clientId,
25+
clientSecret: clientSecret,
26+
});
27+
}
2728

28-
return new Octokit({
29-
auth: installationAuthentication.token,
30-
});
31-
} catch (e) {
32-
Promise.reject(e);
33-
}
29+
async function createInstallationClient(githubAppAuth: AppAuth): Promise<Octokit> {
30+
const auth = await githubAppAuth({ type: 'installation' });
31+
return new Octokit({ auth: auth.token });
3432
}
3533

3634
export const handle = async (eventSource: string, payload: ActionRequestMessage): Promise<void> => {
3735
if (eventSource !== 'aws:sqs') throw Error('Cannot handle non-SQS events!');
38-
const githubClient = await createGithubClient(payload.installationId);
39-
const queuedWorkflows = await githubClient.actions.listRepoWorkflowRuns({
36+
const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS);
37+
const githubAppAuth = createGithubAppAuth(payload.installationId);
38+
const githubInstallationClient = await createInstallationClient(githubAppAuth);
39+
const queuedWorkflows = await githubInstallationClient.actions.listRepoWorkflowRuns({
4040
owner: payload.repositoryOwner,
4141
repo: payload.repositoryName,
4242
// @ts-ignore (typing is incorrect)
4343
status: 'queued',
4444
});
4545
console.info(
46-
`Repo ${payload.repositoryOwner}/${payload.repositoryName} has ${queuedWorkflows.total_count} queued workflow runs`,
46+
`Repo ${payload.repositoryOwner}/${payload.repositoryName} has ${queuedWorkflows.data.total_count} queued workflow runs`,
4747
);
48+
49+
if (queuedWorkflows.data.total_count > 0) {
50+
// console.log(enableOrgLevel);
51+
// const currentRunners = enableOrgLevel
52+
// ? await githubInstallationClient.actions.listSelfHostedRunnersForOrg({
53+
// org: payload.repositoryOwner,
54+
// })
55+
// : await githubInstallationClient.actions.listSelfHostedRunnersForRepo({
56+
// owner: payload.repositoryOwner,
57+
// repo: payload.repositoryName,
58+
// });
59+
// // const currentOnlineRunners = currentRunners.data.runners.filter((r) => r.status === 'online');
60+
// // if (currentOnlineRunners.length > 0)
61+
}
4862
};

‎modules/runners/lambdas/scale-runners/src/scale-runners/runners.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { listRunners } from './runners';
2+
import { handle } from './handler';
3+
import { EC2 } from 'aws-sdk';
4+
5+
jest.mock('./handler');
6+
const mockEC2 = { describeInstances: jest.fn() };
7+
jest.mock('aws-sdk', () => ({
8+
EC2: jest.fn().mockImplementation(() => mockEC2),
9+
}));
10+
11+
describe('list instances', () => {
12+
beforeAll(() => {
13+
jest.clearAllMocks();
14+
});
15+
it('returns a list of instances', () => {
16+
listRunners();
17+
});
18+
});

0 commit comments

Comments
 (0)