diff --git a/CHANGELOG.md b/CHANGELOG.md index e3666ef..2c4f070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [7.12.6] - 2026-01-22 +### Added +- Added var `apiary_managed_s3_logs_queue_arn` to support use existing SQS queue for s3 logs bucket data event. + ## [7.12.5] - 2025-11-12 ### Added - Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` permission in LakeFormation on s3 locations of all databases. diff --git a/VARIABLES.md b/VARIABLES.md index b78d092..98954b8 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -17,6 +17,7 @@ | apiary\_governance\_iamroles | AWS IAM governance roles allowed read and tagging access to managed Apiary S3 buckets. | `list(string)` | `[]` | no | | apiary\_log\_bucket | Bucket for Apiary logs.If this is blank, module will create a bucket. | `string` | `""` | no | | apiary\_log\_prefix | Prefix for Apiary logs. | `string` | `""` | no | +| apiary\_managed\_s3\_logs_queue\_arn | Apiary Managed S3 Logs SQS Queue ARN. | `string` | `""` | no | | apiary\_managed\_schemas | List of maps, each map contains schema name from which S3 bucket names will be derived, and various properties. The corresponding S3 bucket will be named as apiary\_instance-aws\_account-aws\_region-schema\_name. | `list(map(string))` | `[]` | no | | apiary\_producer\_iamroles | AWS IAM roles allowed write access to managed Apiary S3 buckets. | `map(any)` | `{}` | no | | apiary\_rds\_additional\_sg | Comma-separated string containing additional security groups to attach to RDS. | `list(any)` | `[]` | no | diff --git a/common.tf b/common.tf index 792f359..538730c 100644 --- a/common.tf +++ b/common.tf @@ -41,6 +41,7 @@ locals { create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false + create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn == "" && local.enable_apiary_s3_log_management ? true : false apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : "" apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : "" apiary_system_bucket = "${local.apiary_bucket_prefix}-${replace(var.system_schema_name, "_", "-")}" diff --git a/s3-other.tf b/s3-other.tf index 5ce7573..4564022 100644 --- a/s3-other.tf +++ b/s3-other.tf @@ -208,7 +208,7 @@ resource "aws_s3_bucket_notification" "apiary_managed_logs_bucket" { bucket = aws_s3_bucket.apiary_managed_logs_bucket[0].bucket queue { - queue_arn = aws_sqs_queue.apiary_managed_logs_queue[0].arn + queue_arn = local.create_sqs_s3_logs_queue ? aws_sqs_queue.apiary_managed_logs_queue[0].arn : var.apiary_managed_s3_logs_queue_arn events = ["s3:ObjectCreated:*"] } } diff --git a/s3.tf b/s3.tf index 86a75c8..71841ea 100644 --- a/s3.tf +++ b/s3.tf @@ -45,11 +45,6 @@ resource "aws_s3_bucket" "apiary_data_bucket" { tags = merge(tomap({"Name"=each.value["data_bucket"]}), var.apiary_tags, jsondecode(lookup(each.value, "tags", "{}"))) - - logging { - target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket - target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" - } } resource "aws_s3_bucket_versioning" "apiary_data_bucket_versioning" { @@ -188,6 +183,14 @@ resource "aws_s3_bucket_notification" "data_queue_events" { } } +resource "aws_s3_bucket_logging" "apiary_bucket" { + for_each = { + for schema in local.schemas_info : "${schema["schema_name"]}" => schema + } + bucket = each.value["data_bucket"] + target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket + target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/" +} resource "aws_s3_bucket_metric" "paid_metrics" { for_each = var.enable_s3_paid_metrics ? { diff --git a/sns.tf b/sns.tf index 7c604b3..d275fd1 100644 --- a/sns.tf +++ b/sns.tf @@ -77,7 +77,7 @@ POLICY } resource "aws_sqs_queue" "apiary_managed_logs_queue" { - count = local.enable_apiary_s3_log_management ? 1 : 0 + count = local.create_sqs_s3_logs_queue ? 1 : 0 name = "${local.instance_alias}-s3-logs-queue" tags = var.apiary_tags diff --git a/variables.tf b/variables.tf index 156a8df..f0b5072 100644 --- a/variables.tf +++ b/variables.tf @@ -1251,6 +1251,12 @@ variable "additional_s3_log_buckets" { default = [] } +variable "apiary_managed_s3_logs_queue_arn" { + description = "Apiary Managed Log SQS Queue ARN" + type = string + default = "" +} + variable "hms_ro_k8s_log4j_properties" { description = "Custom Log4j properties for apiary readonly metastore." type = string