Make SQS modulair (#7)

* Make queue modulair, refactor and fix naming according architcture

* fix creating fake zip file

* Fix ci for webhook

Author: npalm

.github/workflows/lambda-syncer.yml …kflows/lambda-runner-binaries-syncer.yml.github/workflows/lambda-syncer.yml renamed to .github/workflows/lambda-runner-binaries-syncer.yml .github/workflows/lambda-syncer.yml renamed to .github/workflows/lambda-runner-binaries-syncer.yml

@@ -1,20 +1,20 @@
-name: Lambda Runner Distribution Syncer
+name: Lambda Runner Binaries Syncer
 on:
   push:
     branches:
       - master
   pull_request:
     paths:
       - .github/workflows/lambda-agent-webhook.yml
-      - "modules/action-runner-binary-cache/lambdas/syncer/**"
+      - "modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/**"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     container: node:12
     defaults:
       run:
-        working-directory: modules/action-runner-binary-cache/lambdas/syncer
+        working-directory: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer
 
     steps:
       - uses: actions/checkout@v2

…ithub/workflows/lambda-agent-webhook.yml .github/workflows/lambda-webhook.yml.github/workflows/lambda-agent-webhook.yml renamed to .github/workflows/lambda-webhook.yml .github/workflows/lambda-agent-webhook.yml renamed to .github/workflows/lambda-webhook.yml

@@ -5,16 +5,16 @@ on:
       - master
   pull_request:
     paths:
-      - .github/workflows/lambda-agent-webhook.yml
-      - "modules/agent/lambdas/webhook/**"
+      - .github/workflows/lambda-webhook.yml
+      - "modules/webhook/lambdas/webhook/**"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     container: node:12
     defaults:
       run:
-        working-directory: modules/agent/lambdas/webhook
+        working-directory: modules/webhook/lambdas/webhook
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/terraform.yml

@@ -19,9 +19,9 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
-        run: |
-          touch modules/action-runner-binary-cache/lambdas/syncer/syncer.zip
-          touch modules/agent/lambdas/webhook/webhook.zip
+        run:
+          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
+          touch modules/webhook/lambdas/webhook/webhook.zip
       - name: "Terraform Format"
         uses: hashicorp/terraform-github-actions@master
         with:

examples/default/outputs.tf

@@ -5,11 +5,13 @@ output "action_runners" {
 }
 
 
-output "lambda_syncer_function_name" {
-  value = module.runners.lambda_s3_action_runner_dist_syncer.id
+output "lambda_binaries_syncer_name" {
+  value = module.runners.binaries_syncer.lambda.id
 }
 
 
 output "github_app_webhook_secret" {
   value = random_password.random.result
 }
+
+

main.tf

@@ -3,21 +3,34 @@ locals {
     Environment = var.environment
   })
 
+  s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}"
 }
+
 resource "random_string" "random" {
   length  = 24
   special = false
   upper   = false
 }
 
-module "dsitrubtion_cache" {
-  source = "./modules/action-runner-binary-cache"
+resource "aws_sqs_queue" "queued_builds" {
+  name                        = "${var.environment}-queued-builds.fifo"
+  delay_seconds               = 30
+  fifo_queue                  = true
+  receive_wait_time_seconds   = 10
+  content_based_deduplication = true
+
+  tags = var.tags
+}
+
+module "webhook" {
+  source = "./modules/webhook"
 
   aws_region  = var.aws_region
   environment = var.environment
   tags        = local.tags
 
-  distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
+  sqs_build_queue           = aws_sqs_queue.queued_builds
+  github_app_webhook_secret = var.github_app_webhook_secret
 }
 
 module "runners" {
@@ -28,36 +41,18 @@ module "runners" {
   environment = var.environment
   tags        = local.tags
 
-  s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
+  s3_bucket_runner_binaries   = module.runner_binaries.bucket
+  s3_location_runner_binaries = local.s3_action_runner_url
 }
 
-
-module "agent" {
-  source = "./modules/agent"
+module "runner_binaries" {
+  source = "./modules/runner-binaries-syncer"
 
   aws_region  = var.aws_region
   environment = var.environment
   tags        = local.tags
 
-  github_app_webhook_secret = var.github_app_webhook_secret
-}
-
-
-resource "aws_iam_policy" "dist_bucket" {
-  name        = "${var.environment}-gh-distribution-bucket"
-  path        = "/"
-  description = "Policy for the runner to download the github action runner."
-
-  policy = templatefile("${path.module}/policies/action-runner-s3-policy.json",
-    {
-      s3_arn = module.dsitrubtion_cache.distribution_bucket.arn
-    }
-  )
-}
-
-resource "aws_iam_role_policy_attachment" "dist_bucket" {
-  role       = module.runners.role.name
-  policy_arn = aws_iam_policy.dist_bucket.arn
+  distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
 }
 
 resource "aws_resourcegroups_group" "resourcegroups_group" {