diff --git a/README.md b/README.md index 6e91499..38c5795 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,37 @@ This module was created to simplify deploying Gitlab into the EKS with storage o In the above diagram, you can see the components and their relations (PostgreSQL and Redis are not deployed with this module). +## Lean backup CronJob + +The chart deploys a single nightly full backup (blobs included) via the toolbox `backups.cron`. The +optional `lean_backup` input adds a **second, toggleable tier**: an intraday "lean full" backup that +runs `backup-utility` with all object-storage/blob components skipped, so only `db` and +`repositories` are captured. This is useful when the blob data already lives durably in S3 and only +the database and Git repositories need a more frequent off-instance copy. + +It is disabled by default. Enabling it takes a single flag: + +```hcl +lean_backup = { + enabled = true +} +``` + +Everything else has sensible defaults (schedule `0 6,12,18 * * *`, `concurrencyPolicy: Forbid`, +`activeDeadlineSeconds: 2700`, gitlab-base node placement, and the full blob `--skip` list). See the +`lean_backup` input below for the complete set of overridable fields. + +The CronJob is rendered from `templates/lean-backup-cronjob.yaml.tpl` and applied with the +`kubectl` provider. The template is a faithful clone of the chart-managed toolbox backup pod spec +(init containers, projected secrets, volumes and env), with only scheduling, resources and the +`--skip` arguments parameterized. Container/init image repositories are taken from `values`, and the +image tag is resolved automatically from the deployed release's GitLab application version (the +chart's appVersion / `global.gitlabVersion`) — so no version needs to be maintained here. + +> **Chart upgrades:** because the template mirrors the chart's rendered toolbox backup pod spec, it +> must be re-synced after major GitLab chart upgrades that change the toolbox secret/volume topology. +> Regenerate it from the live CronJob: `kubectl -n get cronjob -toolbox-backup -o yaml`. + ## Requirements @@ -26,9 +57,10 @@ In the above diagram, you can see the components and their relations (PostgreSQL | Name | Version | |------|---------| -| [aws](#provider\_aws) | 6.36.0 | +| [aws](#provider\_aws) | 6.55.0 | | [helm](#provider\_helm) | 2.11.0 | -| [kubernetes](#provider\_kubernetes) | 3.0.1 | +| [kubectl](#provider\_kubectl) | 2.4.1 | +| [kubernetes](#provider\_kubernetes) | 3.2.1 | ## Modules @@ -43,6 +75,7 @@ In the above diagram, you can see the components and their relations (PostgreSQL | Name | Type | |------|------| | [helm_release.gitlab](https://registry.terraform.io/providers/hashicorp/helm/2.11.0/docs/resources/release) | resource | +| [kubectl_manifest.lean_backup](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | | [kubernetes_namespace_v1.gitlab](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | | [kubernetes_secret_v1.gitlab_omniauth_providers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | | [kubernetes_secret_v1.gitlab_rails_storage](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | @@ -67,6 +100,7 @@ In the above diagram, you can see the components and their relations (PostgreSQL | [database\_password](#input\_database\_password) | Password to access PostgreSQL database | `string` | n/a | yes | | [gitlab\_chart\_version](#input\_gitlab\_chart\_version) | Version of the gitlab chart | `string` | `"7.8.1"` | no | | [ldap\_password](#input\_ldap\_password) | LDAP password | `string` | `""` | no | +| [lean\_backup](#input\_lean\_backup) | Optional intraday "lean full" backup CronJob (db + repositories only; object-storage/blob
components skipped). Rendered as a clone of the chart's toolbox backup CronJob with only the
scheduling, resources and `--skip` arguments changed, so env/secrets/volumes stay faithful to
the chart. Disabled by default; set `enabled = true` to create it.

Images default to the toolbox/certificates/gitlab-base repositories taken from `values`, tagged
with the GitLab application version resolved from the deployed Helm release (the chart's
appVersion, or `global.gitlabVersion` if set) — so there is no image version to maintain here.
Supply the full `*_image` fields only to override. `name` defaults to
"-toolbox-backup-lean" and `service_account_name` to "-toolbox". | 
object({
    enabled                       = optional(bool, false)
    schedule                      = optional(string, "0 6,12,18 * * *")
    name                          = optional(string, null)
    toolbox_image                 = optional(string, null)
    certificates_image            = optional(string, null)
    configure_image               = optional(string, null)
    service_account_name          = optional(string, null)
    rails_secret_name             = optional(string, null)
    concurrency_policy            = optional(string, "Forbid")
    restart_policy                = optional(string, "Never")
    active_deadline_seconds       = optional(number, 2700)
    backoff_limit                 = optional(number, 0)
    successful_jobs_history_limit = optional(number, 1)
    failed_jobs_history_limit     = optional(number, 3)
    ttl_seconds_after_finished    = optional(number, 86400)
    tmp_storage_size              = optional(string, "30Gi")
    skip = optional(list(string), [
      "registry", "uploads", "pages", "packages", "external_diffs",
      "ci_secure_files", "lfs", "artifacts", "terraform_state",
    ])
    node_selector = optional(map(string), {
      provisioner = "gitlab-base"
      nodetype    = "gitlab-base"
    })
    tolerations = optional(list(object({
      key      = string
      value    = optional(string)
      effect   = string
      operator = optional(string)
    })), [{ key = "gitlab-base", value = "true", effect = "NoSchedule" }])
    pod_annotations = optional(map(string), { "karpenter.sh/do-not-disrupt" = "true" })
    resources       = optional(any, { requests = { cpu = "500m", memory = "1G" } })
  })
| `{}` | no | | [namespace\_labels](#input\_namespace\_labels) | Labels for GitLab namespace | `map(string)` | `{}` | no | | [omniauth\_providers](#input\_omniauth\_providers) | OmniAuth providers | `map(string)` | `{}` | no | | [redis\_password](#input\_redis\_password) | Password to access Redis database | `string` | n/a | yes | @@ -86,6 +120,7 @@ In the above diagram, you can see the components and their relations (PostgreSQL | Name | Description | |------|-------------| | [buckets](#output\_buckets) | List of buckets created | +| [lean\_backup\_cronjob\_name](#output\_lean\_backup\_cronjob\_name) | Name of the lean backup CronJob, or null when lean\_backup is disabled | | [role\_arn](#output\_role\_arn) | ARN of IAM role | | [role\_name](#output\_role\_name) | Name of IAM role | diff --git a/examples/main.tf b/examples/main.tf index 23aff72..566fc2f 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -70,6 +70,23 @@ EOF backup = true } + # Optional intraday "lean full" backup CronJob (db + repositories only; object-storage blobs + # skipped) in addition to the chart-managed nightly full. Disabled by default. The image tag is + # resolved automatically from the deployed chart's GitLab app version, so only `enabled` is + # required; the rest below are optional overrides shown with their defaults. + lean_backup = { + enabled = true + + # schedule = "0 6,12,18 * * *" + # concurrency_policy = "Forbid" + # active_deadline_seconds = 2700 + # tmp_storage_size = "30Gi" + # skip = [ + # "registry", "uploads", "pages", "packages", "external_diffs", + # "ci_secure_files", "lfs", "artifacts", "terraform_state", + # ] + } + values = [ templatefile("values.yaml", { database_host = "gitlab.xxxxxxxxxxxx.eu-central-1.rds.amazonaws.com" diff --git a/locals.tf b/locals.tf index c76b7ad..cfc4189 100644 --- a/locals.tf +++ b/locals.tf @@ -54,4 +54,50 @@ locals { for bucket, config in local.decoded_buckets_lifecycles : bucket => config if length(try(config.lifecycle_rule, [])) > 0 } + + # --- Lean backup CronJob ------------------------------------------------ + lean = var.lean_backup + + lean_name = coalesce(local.lean.name, "${var.release_name}-toolbox-backup-lean") + lean_service_account = coalesce(local.lean.service_account_name, "${var.release_name}-toolbox") + lean_rails_secret_name = coalesce(local.lean.rails_secret_name, try(local.values.global.railsSecrets.secret, "${var.release_name}-rails-secret")) + + # Container/init image tag is resolved automatically (no manual version to maintain), mirroring the + # chart's `gitlab.versionTag` helper: take `global.gitlabVersion` if set, else the chart appVersion + # exposed by the deployed helm_release, and prepend "v" when it is a plain semver (branch-name + # versions are used verbatim). Repositories come from `values`; full `*_image` overrides win. + lean_app_version = coalesce(try(local.values.global.gitlabVersion, null), try(helm_release.gitlab.metadata[0].app_version, "")) + lean_image_tag = can(regex("^\\d+\\.\\d+\\.\\d+(-rc\\d+)?(-pre)?$", local.lean_app_version)) ? "v${local.lean_app_version}" : local.lean_app_version + + lean_toolbox_image = coalesce(local.lean.toolbox_image, "${try(local.values.gitlab.toolbox.image.repository, "")}:${local.lean_image_tag}") + lean_certificates_image = coalesce(local.lean.certificates_image, "${try(local.values.global.certificates.image.repository, "")}:${local.lean_image_tag}") + lean_configure_image = coalesce(local.lean.configure_image, "${try(local.values.global.gitlabBase.image.repository, "")}:${local.lean_image_tag}") + + # s3 backend: the toolbox copies its rendered .s3cfg into $HOME before running backup-utility. + lean_backup_command = "cp /etc/gitlab/.s3cfg $HOME/.s3cfg && backup-utility ${join(" ", [for s in local.lean.skip : "--skip ${s}"])}" + + # Drop null value/operator fields so the rendered tolerations stay clean. + lean_tolerations = [for t in local.lean.tolerations : merge( + { key = t.key, effect = t.effect }, + t.value != null ? { value = t.value } : {}, + t.operator != null ? { operator = t.operator } : {}, + )] + + # Baked pod annotations mirror the chart toolbox pod (logging + safe-to-evict); user-supplied + # annotations (default: karpenter do-not-disrupt) merge on top. + lean_pod_annotations = merge({ + "cluster-autoscaler.kubernetes.io/safe-to-evict" = "false" + "logging/source" = "service" + "logging/stderr-parser" = "json" + "logging/stdout-dt-field" = "time" + "logging/stdout-dt-format" = "%Y-%m-%dT%H:%M:%SZ" + "logging/stdout-parser" = "json" + "logging/type" = "gitlab" + }, local.lean.pod_annotations) + + # Pre-indented YAML fragments injected into the CronJob template. + lean_node_selector_yaml = indent(12, trimspace(yamlencode(local.lean.node_selector))) + lean_tolerations_yaml = indent(12, trimspace(yamlencode(local.lean_tolerations))) + lean_pod_annotations_yaml = indent(12, trimspace(yamlencode(local.lean_pod_annotations))) + lean_resources_yaml = indent(14, trimspace(yamlencode(local.lean.resources))) } diff --git a/main.tf b/main.tf index d6c8317..1157c6a 100644 --- a/main.tf +++ b/main.tf @@ -275,6 +275,46 @@ resource "helm_release" "gitlab" { ] } +# Intraday "lean full" backup CronJob (db + repositories only). Cloned from the chart-managed +# toolbox backup CronJob so its env/secrets/volumes stay faithful; only scheduling, resources and +# the backup-utility --skip arguments differ. Created only when var.lean_backup.enabled is true. +resource "kubectl_manifest" "lean_backup" { + count = var.lean_backup.enabled ? 1 : 0 + + yaml_body = templatefile("${path.module}/templates/lean-backup-cronjob.yaml.tpl", { + cronjob_name = local.lean_name + namespace = local.release_namespace + release_name = var.release_name + bucket_prefix = var.bucket_prefix + schedule = local.lean.schedule + concurrency_policy = local.lean.concurrency_policy + restart_policy = local.lean.restart_policy + active_deadline_seconds = local.lean.active_deadline_seconds + backoff_limit = local.lean.backoff_limit + successful_jobs_history_limit = local.lean.successful_jobs_history_limit + failed_jobs_history_limit = local.lean.failed_jobs_history_limit + ttl_seconds_after_finished = local.lean.ttl_seconds_after_finished + tmp_storage_size = local.lean.tmp_storage_size + backup_command = local.lean_backup_command + toolbox_image = local.lean_toolbox_image + certificates_image = local.lean_certificates_image + configure_image = local.lean_configure_image + service_account = local.lean_service_account + rails_secret_name = local.lean_rails_secret_name + node_selector_yaml = local.lean_node_selector_yaml + tolerations_yaml = local.lean_tolerations_yaml + pod_annotations_yaml = local.lean_pod_annotations_yaml + resources_yaml = local.lean_resources_yaml + }) + + server_side_apply = true + wait = false + + # The chart-generated secrets/configmaps the CronJob projects must exist first; the release also + # supplies the app version used to tag the images. + depends_on = [helm_release.gitlab] +} + module "gitlab_policy" { source = "terraform-aws-modules/iam/aws//modules/iam-policy" version = "v6.4.0" diff --git a/outputs.tf b/outputs.tf index a8ea634..fff6aa3 100644 --- a/outputs.tf +++ b/outputs.tf @@ -14,4 +14,9 @@ output "buckets" { value = tomap({ for k, v in module.s3_bucket : k => v.s3_bucket_arn }) +} + +output "lean_backup_cronjob_name" { + description = "Name of the lean backup CronJob, or null when lean_backup is disabled" + value = var.lean_backup.enabled ? local.lean_name : null } \ No newline at end of file diff --git a/templates/lean-backup-cronjob.yaml.tpl b/templates/lean-backup-cronjob.yaml.tpl new file mode 100644 index 0000000..4f5f4a0 --- /dev/null +++ b/templates/lean-backup-cronjob.yaml.tpl @@ -0,0 +1,323 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ${cronjob_name} + namespace: ${namespace} + labels: + app: toolbox + release: ${release_name} + backup-tier: lean +spec: + concurrencyPolicy: ${concurrency_policy} + failedJobsHistoryLimit: ${failed_jobs_history_limit} + successfulJobsHistoryLimit: ${successful_jobs_history_limit} + schedule: "${schedule}" + suspend: false + jobTemplate: + metadata: {} + spec: + backoffLimit: ${backoff_limit} + activeDeadlineSeconds: ${active_deadline_seconds} + ttlSecondsAfterFinished: ${ttl_seconds_after_finished} + template: + metadata: + annotations: + ${pod_annotations_yaml} + labels: + app: toolbox + release: ${release_name} + backup-tier: lean + spec: + restartPolicy: ${restart_policy} + serviceAccountName: ${service_account} + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: certificates + image: ${certificates_image} + imagePullPolicy: IfNotPresent + env: + - name: TZ + value: UTC + - name: GITLAB_LOG_LEVEL + value: ERROR + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1000 + resources: + requests: + cpu: 50m + volumeMounts: + - mountPath: /etc/ssl/certs + name: etc-ssl-certs + - mountPath: /etc/pki/ca-trust/extracted/pem + name: etc-pki-ca-trust-extracted-pem + - name: configure + command: + - sh + - /config/configure + image: ${configure_image} + imagePullPolicy: IfNotPresent + env: + - name: TZ + value: UTC + - name: GITLAB_LOG_LEVEL + value: ERROR + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1000 + resources: + requests: + cpu: 50m + volumeMounts: + - mountPath: /config + name: toolbox-config + readOnly: true + - mountPath: /init-config + name: init-toolbox-secrets + readOnly: true + - mountPath: /init-secrets + name: toolbox-secrets + containers: + - name: toolbox-backup + args: + - /bin/bash + - -c + - ${backup_command} + image: ${toolbox_image} + imagePullPolicy: IfNotPresent + env: + - name: ARTIFACTS_BUCKET_NAME + value: ${bucket_prefix}-artifacts + - name: REGISTRY_BUCKET_NAME + value: ${bucket_prefix}-registry + - name: LFS_BUCKET_NAME + value: ${bucket_prefix}-lfs + - name: UPLOADS_BUCKET_NAME + value: ${bucket_prefix}-uploads + - name: PACKAGES_BUCKET_NAME + value: ${bucket_prefix}-packages + - name: EXTERNAL_DIFFS_BUCKET_NAME + value: ${bucket_prefix}-mr-diffs + - name: TERRAFORM_STATE_BUCKET_NAME + value: ${bucket_prefix}-terraform-state + - name: CI_SECURE_FILES_BUCKET_NAME + value: ${bucket_prefix}-ci-secure-files + - name: AGENT_PLAN_CONTENT_BUCKET_NAME + value: ${release_name}-agent-plan-content + - name: BACKUP_BUCKET_NAME + value: ${bucket_prefix}-backup-storage + - name: BACKUP_BACKEND + value: s3 + - name: TMP_BUCKET_NAME + value: ${bucket_prefix}-backup-tmp-storage + - name: PAGES_BUCKET_NAME + value: ${bucket_prefix}-pages + - name: CONFIG_TEMPLATE_DIRECTORY + value: /var/opt/gitlab/templates + - name: CONFIG_DIRECTORY + value: /srv/gitlab/config + - name: TZ + value: UTC + - name: GITLAB_LOG_LEVEL + value: ERROR + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1000 + resources: + ${resources_yaml} + volumeMounts: + - mountPath: /etc/gitlab/registry-db/ + name: registry-db-config + readOnly: true + - mountPath: /etc/gitlab/openbao-db/ + name: openbao-db-config + readOnly: true + - mountPath: /var/opt/gitlab/templates + name: toolbox-config + - mountPath: /etc/gitlab + name: toolbox-secrets + readOnly: true + - mountPath: /srv/gitlab/config/secrets.yml + name: toolbox-secrets + subPath: rails-secrets/secrets.yml + - mountPath: /srv/gitlab/tmp + name: toolbox-tmp + - mountPath: /etc/ssl/certs/ + name: etc-ssl-certs + readOnly: true + - mountPath: /etc/pki/ca-trust/extracted/pem + name: etc-pki-ca-trust-extracted-pem + readOnly: true + volumes: + - name: registry-db-config + projected: + defaultMode: 288 + sources: + - configMap: + items: + - key: db-connection.env + path: connection.env + name: ${release_name}-registry-db-connection-config + optional: true + - configMap: + items: + - key: backup-user + path: backup-user.env + - key: restore-user + path: restore-user.env + name: ${release_name}-toolbox-registry-db-backuprestore-users + optional: true + - secret: + items: + - key: backupPassword + path: backup-pass + - key: restorePassword + path: restore-pass + name: ${release_name}-toolbox-registry-database-password + optional: true + - name: openbao-db-config + projected: + defaultMode: 288 + sources: + - configMap: + items: + - key: db-connection.env + path: connection.env + name: ${release_name}-toolbox-openbao-db-connection-config + optional: true + - configMap: + items: + - key: backup-user + path: backup-user.env + - key: restore-user + path: restore-user.env + name: ${release_name}-toolbox-openbao-db-backuprestore-users + optional: true + - secret: + items: + - key: backupPassword + path: backup-pass + - key: restorePassword + path: restore-pass + name: ${release_name}-toolbox-openbao-database-password + optional: true + - name: toolbox-config + projected: + defaultMode: 420 + sources: + - configMap: + name: ${release_name}-toolbox + - name: toolbox-tmp + ephemeral: + volumeClaimTemplate: + metadata: + labels: + app: toolbox + release: ${release_name} + backup-tier: lean + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${tmp_storage_size} + volumeMode: Filesystem + - name: init-toolbox-secrets + projected: + defaultMode: 256 + sources: + - secret: + items: + - key: secrets.yml + path: rails-secrets/secrets.yml + name: ${rails_secret_name} + - secret: + items: + - key: secret + path: shell/.gitlab_shell_secret + name: ${release_name}-gitlab-shell-secret + - secret: + items: + - key: token + path: gitaly/gitaly_token + name: ${release_name}-gitaly-secret + - secret: + items: + - key: secret + path: redis/redis-password + name: ${release_name}-redis-password + - secret: + items: + - key: postgresql-password + path: postgres/psql-password-ci + name: ${release_name}-postgresql-password + - secret: + items: + - key: postgresql-password + path: postgres/psql-password-main + name: ${release_name}-postgresql-password + - secret: + items: + - key: registry-auth.key + path: registry/gitlab-registry.key + name: ${release_name}-registry-secret + - secret: + items: + - key: secret + path: registry/notificationSecret + name: ${release_name}-registry-notification + - secret: + items: + - key: config + path: objectstorage/.s3cfg + name: ${release_name}-rails-storage + - secret: + items: + - key: shared_secret + path: pages/secret + name: ${release_name}-gitlab-pages-secret + - secret: + items: + - key: connection + path: objectstorage/object_store + name: ${release_name}-rails-storage + - secret: + items: + - key: connection + path: objectstorage/ci_secure_files + name: ${release_name}-rails-storage + - secret: + items: + - key: provider + path: omniauth/gitlab-google-oauth2/provider + name: gitlab-google-oauth2 + - name: toolbox-secrets + emptyDir: + medium: Memory + - name: etc-ssl-certs + emptyDir: + medium: Memory + - name: etc-pki-ca-trust-extracted-pem + emptyDir: + medium: Memory + nodeSelector: + ${node_selector_yaml} + tolerations: + ${tolerations_yaml} diff --git a/variables.tf b/variables.tf index bee8099..692c5cb 100644 --- a/variables.tf +++ b/variables.tf @@ -118,4 +118,56 @@ variable "role_suffix" { type = string description = "Optional suffix for GitLab role" default = "access-aws" +} + +variable "lean_backup" { + description = <<-EOT + Optional intraday "lean full" backup CronJob (db + repositories only; object-storage/blob + components skipped). Rendered as a clone of the chart's toolbox backup CronJob with only the + scheduling, resources and `--skip` arguments changed, so env/secrets/volumes stay faithful to + the chart. Disabled by default; set `enabled = true` to create it. + + Images default to the toolbox/certificates/gitlab-base repositories taken from `values`, tagged + with the GitLab application version resolved from the deployed Helm release (the chart's + appVersion, or `global.gitlabVersion` if set) — so there is no image version to maintain here. + Supply the full `*_image` fields only to override. `name` defaults to + "-toolbox-backup-lean" and `service_account_name` to "-toolbox". + EOT + + type = object({ + enabled = optional(bool, false) + schedule = optional(string, "0 6,12,18 * * *") + name = optional(string, null) + toolbox_image = optional(string, null) + certificates_image = optional(string, null) + configure_image = optional(string, null) + service_account_name = optional(string, null) + rails_secret_name = optional(string, null) + concurrency_policy = optional(string, "Forbid") + restart_policy = optional(string, "Never") + active_deadline_seconds = optional(number, 2700) + backoff_limit = optional(number, 0) + successful_jobs_history_limit = optional(number, 1) + failed_jobs_history_limit = optional(number, 3) + ttl_seconds_after_finished = optional(number, 86400) + tmp_storage_size = optional(string, "30Gi") + skip = optional(list(string), [ + "registry", "uploads", "pages", "packages", "external_diffs", + "ci_secure_files", "lfs", "artifacts", "terraform_state", + ]) + node_selector = optional(map(string), { + provisioner = "gitlab-base" + nodetype = "gitlab-base" + }) + tolerations = optional(list(object({ + key = string + value = optional(string) + effect = string + operator = optional(string) + })), [{ key = "gitlab-base", value = "true", effect = "NoSchedule" }]) + pod_annotations = optional(map(string), { "karpenter.sh/do-not-disrupt" = "true" }) + resources = optional(any, { requests = { cpu = "500m", memory = "1G" } }) + }) + + default = {} } \ No newline at end of file