From 991118722180cebac3fe623943e741c14d7a446d Mon Sep 17 00:00:00 2001 From: janli Date: Wed, 21 Jan 2026 15:06:48 -0800 Subject: [PATCH 1/6] fix: added new variable to support use existing SQS queue for s3 logs bucket data event --- CHANGELOG.md | 4 ++++ VARIABLES.md | 1 + common.tf | 1 + s3-other.tf | 2 +- sns.tf | 2 +- variables.tf | 6 ++++++ 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3666ef..2c4f070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [7.12.6] - 2026-01-22 +### Added +- Added var `apiary_managed_s3_logs_queue_arn` to support use existing SQS queue for s3 logs bucket data event. + ## [7.12.5] - 2025-11-12 ### Added - Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` permission in LakeFormation on s3 locations of all databases. diff --git a/VARIABLES.md b/VARIABLES.md index b78d092..98954b8 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -17,6 +17,7 @@ | apiary\_governance\_iamroles | AWS IAM governance roles allowed read and tagging access to managed Apiary S3 buckets. | `list(string)` | `[]` | no | | apiary\_log\_bucket | Bucket for Apiary logs.If this is blank, module will create a bucket. | `string` | `""` | no | | apiary\_log\_prefix | Prefix for Apiary logs. | `string` | `""` | no | +| apiary\_managed\_s3\_logs_queue\_arn | Apiary Managed S3 Logs SQS Queue ARN. | `string` | `""` | no | | apiary\_managed\_schemas | List of maps, each map contains schema name from which S3 bucket names will be derived, and various properties. The corresponding S3 bucket will be named as apiary\_instance-aws\_account-aws\_region-schema\_name. | `list(map(string))` | `[]` | no | | apiary\_producer\_iamroles | AWS IAM roles allowed write access to managed Apiary S3 buckets. | `map(any)` | `{}` | no | | apiary\_rds\_additional\_sg | Comma-separated string containing additional security groups to attach to RDS. | `list(any)` | `[]` | no | diff --git a/common.tf b/common.tf index 792f359..0357233 100644 --- a/common.tf +++ b/common.tf @@ -41,6 +41,7 @@ locals { create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false + create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn != "" && local.enable_apiary_s3_log_management ? true : false apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : "" apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : "" apiary_system_bucket = "${local.apiary_bucket_prefix}-${replace(var.system_schema_name, "_", "-")}" diff --git a/s3-other.tf b/s3-other.tf index 5ce7573..4564022 100644 --- a/s3-other.tf +++ b/s3-other.tf @@ -208,7 +208,7 @@ resource "aws_s3_bucket_notification" "apiary_managed_logs_bucket" { bucket = aws_s3_bucket.apiary_managed_logs_bucket[0].bucket queue { - queue_arn = aws_sqs_queue.apiary_managed_logs_queue[0].arn + queue_arn = local.create_sqs_s3_logs_queue ? aws_sqs_queue.apiary_managed_logs_queue[0].arn : var.apiary_managed_s3_logs_queue_arn events = ["s3:ObjectCreated:*"] } } diff --git a/sns.tf b/sns.tf index 7c604b3..d275fd1 100644 --- a/sns.tf +++ b/sns.tf @@ -77,7 +77,7 @@ POLICY } resource "aws_sqs_queue" "apiary_managed_logs_queue" { - count = local.enable_apiary_s3_log_management ? 1 : 0 + count = local.create_sqs_s3_logs_queue ? 1 : 0 name = "${local.instance_alias}-s3-logs-queue" tags = var.apiary_tags diff --git a/variables.tf b/variables.tf index 156a8df..f0b5072 100644 --- a/variables.tf +++ b/variables.tf @@ -1251,6 +1251,12 @@ variable "additional_s3_log_buckets" { default = [] } +variable "apiary_managed_s3_logs_queue_arn" { + description = "Apiary Managed Log SQS Queue ARN" + type = string + default = "" +} + variable "hms_ro_k8s_log4j_properties" { description = "Custom Log4j properties for apiary readonly metastore." type = string From fd3f972c5f48924f1609c92fccf9d979bc49bda3 Mon Sep 17 00:00:00 2001 From: Diego Armando Vargas Salazar Date: Wed, 21 Jan 2026 19:19:14 -0500 Subject: [PATCH 2/6] =?UTF-8?q?Decouple=20the=20=C2=B4apiary=5Flog=5Fbucke?= =?UTF-8?q?t=C2=B4=20variable=20for=20s3=20logs=20and=20s3=20logs=20hive?= =?UTF-8?q?=20buckets=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will decouple the old apiary_log_bucket variable and split out in two, this would decouple the behavior of creating the s3 logs and the s3 logs hive bucket depending on the same variable. --- README.md | 3 ++- common.tf | 4 ++-- k8s-readwrite.tf | 2 +- s3.tf | 2 +- templates.tf | 2 +- variables.tf | 10 ++++++++-- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 27418ab..444e45b 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,8 @@ module "apiary" { hms_rw_cpu = "2048" hms_ro_heapsize = "8192" hms_rw_heapsize = "8192" - apiary_log_bucket = "s3-logs-bucket" + apiary_s3_log_bucket = "s3-logs-bucket" + apiary_s3_log_hive_bucket= "s3-logs-hive-bucket" db_instance_class = "db.t2.medium" db_backup_retention = "7" apiary_managed_schemas = [ diff --git a/common.tf b/common.tf index 0357233..b864380 100644 --- a/common.tf +++ b/common.tf @@ -39,8 +39,8 @@ locals { s3_inventory_prefix = "EntireBucketDaily" s3_inventory_bucket = var.s3_enable_inventory ? "${local.apiary_bucket_prefix}-s3-inventory" : "" create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false - enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false - enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false + enable_apiary_s3_log_management = var.apiary_s3_log_bucket == "" ? true : false + enable_apiary_s3_log_hive = var.apiary_s3_log_hive_bucket == "" && var.enable_apiary_s3_log_hive ? true : false create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn != "" && local.enable_apiary_s3_log_management ? true : false apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : "" apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : "" diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf index 849c16b..6fd1438 100644 --- a/k8s-readwrite.tf +++ b/k8s-readwrite.tf @@ -262,7 +262,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" { value = var.s3_enable_inventory ? "1" : "" } env { - # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. + # If user sets "apiary_s3_log_hive_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. name = "ENABLE_S3_LOGS" value = local.enable_apiary_s3_log_hive ? "1" : "" } diff --git a/s3.tf b/s3.tf index 86a75c8..dc6d5a2 100644 --- a/s3.tf +++ b/s3.tf @@ -47,7 +47,7 @@ resource "aws_s3_bucket" "apiary_data_bucket" { jsondecode(lookup(each.value, "tags", "{}"))) logging { - target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket + target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_s3_log_bucket target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" } } diff --git a/templates.tf b/templates.tf index 7874bd4..e96f90d 100644 --- a/templates.tf +++ b/templates.tf @@ -54,7 +54,7 @@ locals{ docker_auth = "${var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("", data.aws_secretsmanager_secret.docker_registry.*.arn))}" s3_enable_inventory = var.s3_enable_inventory ? "1" : "" - # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. + # If user sets "apiary_s3_log_hive_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. s3_enable_logs = local.enable_apiary_s3_log_hive ? "1" : "" # Template vars for init container diff --git a/variables.tf b/variables.tf index f0b5072..6bbecbf 100644 --- a/variables.tf +++ b/variables.tf @@ -66,8 +66,14 @@ variable "hive_metastore_port" { default = 9083 } -variable "apiary_log_bucket" { - description = "Bucket for Apiary logs.If this is blank, module will create a bucket." +variable "apiary_s3_log_bucket" { + description = "Bucket for Apiary s3 logs.If this is blank, module will create a bucket." + type = string + default = "" +} + +variable "apiary_s3_log_hive_bucket" { + description = "Bucket for Apiary s3 logs hive.If this is blank, module will create a bucket." type = string default = "" } From 03d9a83833bb5d181ed1417dfe9e846d327894b6 Mon Sep 17 00:00:00 2001 From: Diego Armando Vargas Salazar Date: Wed, 21 Jan 2026 19:23:10 -0500 Subject: [PATCH 3/6] =?UTF-8?q?Revert=20"Decouple=20the=20=C2=B4apiary=5Fl?= =?UTF-8?q?og=5Fbucket=C2=B4=20variable=20for=20s3=20logs=20and=20s3=20log?= =?UTF-8?q?s=20hive=20buckets=20creation"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit fd3f972c5f48924f1609c92fccf9d979bc49bda3. --- README.md | 3 +-- common.tf | 4 ++-- k8s-readwrite.tf | 2 +- s3.tf | 2 +- templates.tf | 2 +- variables.tf | 10 ++-------- 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 444e45b..27418ab 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,7 @@ module "apiary" { hms_rw_cpu = "2048" hms_ro_heapsize = "8192" hms_rw_heapsize = "8192" - apiary_s3_log_bucket = "s3-logs-bucket" - apiary_s3_log_hive_bucket= "s3-logs-hive-bucket" + apiary_log_bucket = "s3-logs-bucket" db_instance_class = "db.t2.medium" db_backup_retention = "7" apiary_managed_schemas = [ diff --git a/common.tf b/common.tf index b864380..0357233 100644 --- a/common.tf +++ b/common.tf @@ -39,8 +39,8 @@ locals { s3_inventory_prefix = "EntireBucketDaily" s3_inventory_bucket = var.s3_enable_inventory ? "${local.apiary_bucket_prefix}-s3-inventory" : "" create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false - enable_apiary_s3_log_management = var.apiary_s3_log_bucket == "" ? true : false - enable_apiary_s3_log_hive = var.apiary_s3_log_hive_bucket == "" && var.enable_apiary_s3_log_hive ? true : false + enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false + enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn != "" && local.enable_apiary_s3_log_management ? true : false apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : "" apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : "" diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf index 6fd1438..849c16b 100644 --- a/k8s-readwrite.tf +++ b/k8s-readwrite.tf @@ -262,7 +262,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" { value = var.s3_enable_inventory ? "1" : "" } env { - # If user sets "apiary_s3_log_hive_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. + # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. name = "ENABLE_S3_LOGS" value = local.enable_apiary_s3_log_hive ? "1" : "" } diff --git a/s3.tf b/s3.tf index dc6d5a2..86a75c8 100644 --- a/s3.tf +++ b/s3.tf @@ -47,7 +47,7 @@ resource "aws_s3_bucket" "apiary_data_bucket" { jsondecode(lookup(each.value, "tags", "{}"))) logging { - target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_s3_log_bucket + target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" } } diff --git a/templates.tf b/templates.tf index e96f90d..7874bd4 100644 --- a/templates.tf +++ b/templates.tf @@ -54,7 +54,7 @@ locals{ docker_auth = "${var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("", data.aws_secretsmanager_secret.docker_registry.*.arn))}" s3_enable_inventory = var.s3_enable_inventory ? "1" : "" - # If user sets "apiary_s3_log_hive_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. + # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt. s3_enable_logs = local.enable_apiary_s3_log_hive ? "1" : "" # Template vars for init container diff --git a/variables.tf b/variables.tf index 6bbecbf..f0b5072 100644 --- a/variables.tf +++ b/variables.tf @@ -66,14 +66,8 @@ variable "hive_metastore_port" { default = 9083 } -variable "apiary_s3_log_bucket" { - description = "Bucket for Apiary s3 logs.If this is blank, module will create a bucket." - type = string - default = "" -} - -variable "apiary_s3_log_hive_bucket" { - description = "Bucket for Apiary s3 logs hive.If this is blank, module will create a bucket." +variable "apiary_log_bucket" { + description = "Bucket for Apiary logs.If this is blank, module will create a bucket." type = string default = "" } From c48a0657ca2d19e5e895388b5f15a22fcc300674 Mon Sep 17 00:00:00 2001 From: githubjianli <51385385+githubjianli@users.noreply.github.com> Date: Wed, 21 Jan 2026 19:56:05 -0800 Subject: [PATCH 4/6] fix: update condition --- common.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.tf b/common.tf index 0357233..538730c 100644 --- a/common.tf +++ b/common.tf @@ -41,7 +41,7 @@ locals { create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false - create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn != "" && local.enable_apiary_s3_log_management ? true : false + create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn == "" && local.enable_apiary_s3_log_management ? true : false apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : "" apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : "" apiary_system_bucket = "${local.apiary_bucket_prefix}-${replace(var.system_schema_name, "_", "-")}" From e8c319cceb7eef190710832ac78357f3f41c1182 Mon Sep 17 00:00:00 2001 From: Diego Armando Vargas Salazar Date: Thu, 22 Jan 2026 09:12:34 -0500 Subject: [PATCH 5/6] Added flag to enable or disable s3 access logging of the buckets --- s3.tf | 9 ++++++--- variables.tf | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/s3.tf b/s3.tf index 86a75c8..62c9da6 100644 --- a/s3.tf +++ b/s3.tf @@ -46,9 +46,12 @@ resource "aws_s3_bucket" "apiary_data_bucket" { var.apiary_tags, jsondecode(lookup(each.value, "tags", "{}"))) - logging { - target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket - target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" + dynamic "logging" { + for_each = var.enable_s3_access_logging ? [1] : [] + content { + target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket + target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" + } } } diff --git a/variables.tf b/variables.tf index f0b5072..b4ea2c4 100644 --- a/variables.tf +++ b/variables.tf @@ -1274,3 +1274,8 @@ variable "hms_housekeeper_k8s_log4j_properties" { type = string default = "" } + +variable "enable_s3_access_logging" { + type = bool + default = true +} From 6e22e953aad50678b09663032c9be2a12f2ca287 Mon Sep 17 00:00:00 2001 From: janli Date: Thu, 22 Jan 2026 12:22:02 -0800 Subject: [PATCH 6/6] fix: update s3 bucket logging configuration --- s3.tf | 16 ++++++++-------- variables.tf | 5 ----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/s3.tf b/s3.tf index 62c9da6..71841ea 100644 --- a/s3.tf +++ b/s3.tf @@ -45,14 +45,6 @@ resource "aws_s3_bucket" "apiary_data_bucket" { tags = merge(tomap({"Name"=each.value["data_bucket"]}), var.apiary_tags, jsondecode(lookup(each.value, "tags", "{}"))) - - dynamic "logging" { - for_each = var.enable_s3_access_logging ? [1] : [] - content { - target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket - target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" - } - } } resource "aws_s3_bucket_versioning" "apiary_data_bucket_versioning" { @@ -191,6 +183,14 @@ resource "aws_s3_bucket_notification" "data_queue_events" { } } +resource "aws_s3_bucket_logging" "apiary_bucket" { + for_each = { + for schema in local.schemas_info : "${schema["schema_name"]}" => schema + } + bucket = each.value["data_bucket"] + target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket + target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" +} resource "aws_s3_bucket_metric" "paid_metrics" { for_each = var.enable_s3_paid_metrics ? { diff --git a/variables.tf b/variables.tf index b4ea2c4..f0b5072 100644 --- a/variables.tf +++ b/variables.tf @@ -1274,8 +1274,3 @@ variable "hms_housekeeper_k8s_log4j_properties" { type = string default = "" } - -variable "enable_s3_access_logging" { - type = bool - default = true -}