diff --git a/charts/observability-stack/Chart.yaml b/charts/observability-stack/Chart.yaml index b8974775..b4338e93 100644 --- a/charts/observability-stack/Chart.yaml +++ b/charts/observability-stack/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: observability-stack description: OpenTelemetry-native observability platform for microservices, web apps, and AI agents type: application -version: 0.2.0 +version: 0.3.0 appVersion: "3.7.0" home: https://github.com/opensearch-project/observability-stack diff --git a/charts/observability-stack/templates/data-prepper-metrics-service.yaml b/charts/observability-stack/templates/data-prepper-metrics-service.yaml new file mode 100644 index 00000000..25d5ffe2 --- /dev/null +++ b/charts/observability-stack/templates/data-prepper-metrics-service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.dataPrepperMetricsService.enabled (index .Values "data-prepper" "enabled") }} +{{- /* The subchart's main Service omits 4900, so DP metrics aren't scrapable. Listing + 4900 in .Values.ports duplicates the `server` containerPort and is rejected; a + separate Service targeting `server` by name is not. */ -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-data-prepper-metrics + labels: + {{- /* Must match the subchart's pod labels. */}} + app.kubernetes.io/name: data-prepper + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: metrics +spec: + type: ClusterIP + ports: + - name: metrics + port: {{ .Values.dataPrepperMetricsService.port }} + targetPort: server + protocol: TCP + selector: + app.kubernetes.io/name: data-prepper + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml b/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml index f95d328b..f6ccb3e0 100644 --- a/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml +++ b/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml @@ -1,4 +1,8 @@ -{{- if index .Values "data-prepper" "enabled" }} +{{- /* Default true when the key is absent; honor an explicit false (Helm's `default` + treats false as empty, so a nil-check is required to let the gate turn off). */ -}} +{{- $manageSecret := true }} +{{- if hasKey .Values "dataPrepperManageSecret" }}{{- $manageSecret = .Values.dataPrepperManageSecret }}{{- end }} +{{- if and (index .Values "data-prepper" "enabled") $manageSecret }} {{- $opensearchHost := printf "https://%s:9200" (.Values.opensearchServiceName | default "opensearch-cluster-master") -}} apiVersion: v1 kind: Secret @@ -63,7 +67,8 @@ stringData: pipeline: name: "otel-traces-pipeline" processor: - - otel_traces: {} + - otel_traces: + trace_flush_interval: {{ .Values.dataPrepperTraceFlushInterval | default 180 }} sink: - opensearch: hosts: [{{ $opensearchHost | quote }}] diff --git a/charts/observability-stack/templates/otel-collector-configmap.yaml b/charts/observability-stack/templates/otel-collector-configmap.yaml index 02c5f1be..c22bf7dd 100644 --- a/charts/observability-stack/templates/otel-collector-configmap.yaml +++ b/charts/observability-stack/templates/otel-collector-configmap.yaml @@ -85,7 +85,7 @@ data: scrape_interval: 15s metrics_path: /metrics/sys static_configs: - - targets: ["{{ .Release.Name }}-data-prepper:4900"] + - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"] relabel_configs: - target_label: service.name replacement: data-prepper @@ -93,7 +93,7 @@ data: scrape_interval: 15s metrics_path: /metrics/prometheus static_configs: - - targets: ["{{ .Release.Name }}-data-prepper:4900"] + - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"] relabel_configs: - target_label: service.name replacement: data-prepper diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 3087c0fd..53e2e7e1 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -181,8 +181,11 @@ data-prepper: port: 21891 - name: otel-logs port: 21892 - - name: metrics - port: 4900 + # Port 4900 (the subchart's hardcoded `server` containerPort, serving + # /metrics/prometheus) is intentionally NOT listed here: adding it would + # duplicate the containerPort and the API server rejects the Deployment. + # It is published instead by a separate metrics Service: see + # templates/data-prepper-metrics-service.yaml and dataPrepperMetricsService. config: data-prepper-config.yaml: | ssl: false @@ -210,6 +213,27 @@ data-prepper: enabled: false existingSecret: data-prepper-pipeline +# -- Data Prepper pipeline Secret ownership. When true (default), the chart renders the +# default logs/traces/service-map pipeline Secret named by data-prepper.pipelineConfig. +# existingSecret. Set false to provide that Secret yourself (same name) with custom +# routing/sinks; the chart renders nothing and Data Prepper mounts your Secret. +dataPrepperManageSecret: true + +# -- Data Prepper otel_traces trace_flush_interval, in seconds: how long the processor +# buffers spans before computing traceGroup and emitting completed traces. 180 matches the +# upstream default. Higher values raise Data Prepper heap; lower values can split +# late-arriving spans into separate traces. +dataPrepperTraceFlushInterval: 180 + +# -- Data Prepper metrics Service. Publishes the admin/metrics port (4900, +# /metrics/prometheus) on a dedicated ClusterIP Service so Prometheus / the collector can +# scrape Data Prepper's pipeline metrics. The subchart only puts 4900 on the pod, not on +# its Service, so without this the pipeline-health dashboard's Data Prepper panels stay +# empty. +dataPrepperMetricsService: + enabled: true + port: 4900 + # -- OpenTelemetry Collector opentelemetry-collector: enabled: true diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index fec3e16a..2490081e 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -192,6 +192,72 @@ resource "helm_release" "observability_stack" { } } + # --- OpenSearch sizing --- + set { + name = "opensearch.replicas" + value = var.opensearch_replicas + } + set { + name = "opensearch.persistence.size" + value = var.opensearch_storage_size + } + set { + name = "opensearch.persistence.storageClass" + value = var.opensearch_storage_class + } + set { + name = "opensearch.resources.requests.memory" + value = var.opensearch_node_memory + } + set { + name = "opensearch.resources.limits.memory" + value = var.opensearch_node_memory + } + set { + name = "opensearch.opensearchJavaOpts" + value = "-Xms${var.opensearch_jvm_heap} -Xmx${var.opensearch_jvm_heap}" + } + + # --- Cortex sizing --- + set { + name = "cortex.persistence.size" + value = var.cortex_storage_size + } + set { + name = "cortex.persistence.storageClass" + value = var.cortex_storage_class + } + + # --- Data Prepper sizing --- + set { + name = "data-prepper.resources.requests.memory" + value = var.data_prepper_memory + } + set { + name = "data-prepper.resources.limits.memory" + value = var.data_prepper_memory + } + # JAVA_OPTS via extraEnvs[0], emitted only when set (else the JVM uses its + # MaxRAMPercentage default). Claims index 0, so don't also set data-prepper.extraEnvs. + dynamic "set" { + for_each = var.data_prepper_jvm_heap == "" ? [] : [1] + content { + name = "data-prepper.extraEnvs[0].name" + value = "JAVA_OPTS" + } + } + dynamic "set" { + for_each = var.data_prepper_jvm_heap == "" ? [] : [1] + content { + name = "data-prepper.extraEnvs[0].value" + value = "-Xms${var.data_prepper_jvm_heap} -Xmx${var.data_prepper_jvm_heap}" + } + } + set { + name = "dataPrepperTraceFlushInterval" + value = var.data_prepper_trace_flush_interval + } + depends_on = [ helm_release.aws_lb_controller, ] diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 4d097899..72d98860 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -93,6 +93,87 @@ variable "tags" { } } +# ============================================================================ +# OpenSearch sizing +# ============================================================================ + +variable "opensearch_replicas" { + description = "Number of OpenSearch nodes (StatefulSet replicas). 3 is the production minimum." + type = number + default = 3 +} + +variable "opensearch_storage_size" { + description = "Per-node OpenSearch PVC size. Total cluster storage is opensearch_replicas times this value." + type = string + default = "100Gi" +} + +variable "opensearch_storage_class" { + description = "EBS storage class for OpenSearch PVCs." + type = string + default = "gp2" +} + +variable "opensearch_node_memory" { + description = "Per-node OpenSearch container memory. The chart sets requests equal to limits." + type = string + default = "4Gi" +} + +variable "opensearch_jvm_heap" { + description = "OpenSearch JVM heap, roughly 50% of opensearch_node_memory, max 31g. JVM size syntax, e.g. '2g' or '512m'." + type = string + default = "2g" + validation { + # JVM -Xms/-Xmx syntax (k/m/g), not Kubernetes 'Gi'. + condition = can(regex("^[0-9]+[kmg]$", var.opensearch_jvm_heap)) + error_message = "Use JVM size syntax like '2g' or '512m', not '2Gi'." + } +} + +# ============================================================================ +# Cortex sizing +# ============================================================================ + +variable "cortex_storage_size" { + description = "Cortex PVC size." + type = string + default = "50Gi" +} + +variable "cortex_storage_class" { + description = "EBS storage class for the Cortex PVC." + type = string + default = "gp2" +} + +# ============================================================================ +# Data Prepper sizing +# ============================================================================ + +variable "data_prepper_memory" { + description = "Data Prepper container memory request and limit. Default matches the subchart." + type = string + default = "1Gi" +} + +variable "data_prepper_jvm_heap" { + description = "Data Prepper JVM heap, roughly 75% of data_prepper_memory. JVM size syntax, e.g. '512m'. Empty leaves the JVM default." + type = string + default = "" + validation { + condition = var.data_prepper_jvm_heap == "" || can(regex("^[0-9]+[kmg]$", var.data_prepper_jvm_heap)) + error_message = "Use JVM size syntax like '2g' or '512m', not '2Gi'." + } +} + +variable "data_prepper_trace_flush_interval" { + description = "Seconds the otel_traces processor buffers spans before computing traceGroup. Higher values raise Data Prepper heap; lower values can mark late-arriving spans as separate traces." + type = number + default = 180 +} + # ============================================================================ # Derived # ============================================================================