[chore] Additional Helm chart and Terraform configurability

charts/observability-stack/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: observability-stack
 description: OpenTelemetry-native observability platform for microservices, web apps, and AI agents
 type: application
-version: 0.2.0
+version: 0.3.0
 appVersion: "3.7.0"
 
 home: https://github.com/opensearch-project/observability-stack

charts/observability-stack/templates/data-prepper-metrics-service.yaml

@@ -0,0 +1,24 @@
+{{- if and .Values.dataPrepperMetricsService.enabled (index .Values "data-prepper" "enabled") }}
+{{- /* The subchart's main Service omits 4900, so DP metrics aren't scrapable. Listing
+       4900 in .Values.ports duplicates the `server` containerPort and is rejected; a
+       separate Service targeting `server` by name is not. */ -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-data-prepper-metrics
+  labels:
+    {{- /* Must match the subchart's pod labels. */}}
+    app.kubernetes.io/name: data-prepper
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: metrics
+spec:
+  type: ClusterIP
+  ports:
+    - name: metrics
+      port: {{ .Values.dataPrepperMetricsService.port }}
+      targetPort: server
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: data-prepper
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

charts/observability-stack/templates/data-prepper-pipeline-secret.yaml

@@ -1,4 +1,8 @@
-{{- if index .Values "data-prepper" "enabled" }}
+{{- /* Default true when the key is absent; honor an explicit false (Helm's `default`
+       treats false as empty, so a nil-check is required to let the gate turn off). */ -}}
+{{- $manageSecret := true }}
+{{- if hasKey .Values "dataPrepperManageSecret" }}{{- $manageSecret = .Values.dataPrepperManageSecret }}{{- end }}
+{{- if and (index .Values "data-prepper" "enabled") $manageSecret }}
 {{- $opensearchHost := printf "https://%s:9200" (.Values.opensearchServiceName | default "opensearch-cluster-master") -}}
 apiVersion: v1
 kind: Secret
@@ -63,7 +67,8 @@ stringData:
         pipeline:
           name: "otel-traces-pipeline"
       processor:
-        - otel_traces: {}
+        - otel_traces:
+            trace_flush_interval: {{ .Values.dataPrepperTraceFlushInterval | default 180 }}
       sink:
         - opensearch:
             hosts: [{{ $opensearchHost | quote }}]

charts/observability-stack/templates/otel-collector-configmap.yaml

@@ -85,15 +85,15 @@ data:
               scrape_interval: 15s
               metrics_path: /metrics/sys
               static_configs:
-                - targets: ["{{ .Release.Name }}-data-prepper:4900"]
+                - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"]
               relabel_configs:
                 - target_label: service.name
                   replacement: data-prepper
             - job_name: data-prepper-pipelines
               scrape_interval: 15s
               metrics_path: /metrics/prometheus
               static_configs:
-                - targets: ["{{ .Release.Name }}-data-prepper:4900"]
+                - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"]
               relabel_configs:
                 - target_label: service.name
                   replacement: data-prepper

charts/observability-stack/values.yaml

@@ -181,8 +181,11 @@ data-prepper:
       port: 21891
     - name: otel-logs
       port: 21892
-    - name: metrics
-      port: 4900
+    # Port 4900 (the subchart's hardcoded `server` containerPort, serving
+    # /metrics/prometheus) is intentionally NOT listed here: adding it would
+    # duplicate the containerPort and the API server rejects the Deployment.
+    # It is published instead by a separate metrics Service: see
+    # templates/data-prepper-metrics-service.yaml and dataPrepperMetricsService.
   config:
     data-prepper-config.yaml: |
       ssl: false
@@ -210,6 +213,27 @@ data-prepper:
     enabled: false
     existingSecret: data-prepper-pipeline
 
+# -- Data Prepper pipeline Secret ownership. When true (default), the chart renders the
+# default logs/traces/service-map pipeline Secret named by data-prepper.pipelineConfig.
+# existingSecret. Set false to provide that Secret yourself (same name) with custom
+# routing/sinks; the chart renders nothing and Data Prepper mounts your Secret.
+dataPrepperManageSecret: true
+
+# -- Data Prepper otel_traces trace_flush_interval, in seconds: how long the processor
+# buffers spans before computing traceGroup and emitting completed traces. 180 matches the
+# upstream default. Higher values raise Data Prepper heap; lower values can split
+# late-arriving spans into separate traces.
+dataPrepperTraceFlushInterval: 180
+
+# -- Data Prepper metrics Service. Publishes the admin/metrics port (4900,
+# /metrics/prometheus) on a dedicated ClusterIP Service so Prometheus / the collector can
+# scrape Data Prepper's pipeline metrics. The subchart only puts 4900 on the pod, not on
+# its Service, so without this the pipeline-health dashboard's Data Prepper panels stay
+# empty.
+dataPrepperMetricsService:
+  enabled: true
+  port: 4900
+
 # -- OpenTelemetry Collector
 opentelemetry-collector:
   enabled: true

terraform/aws/observability-stack.tf

@@ -192,6 +192,72 @@ resource "helm_release" "observability_stack" {
     }
   }
 
+  # --- OpenSearch sizing ---
+  set {
+    name  = "opensearch.replicas"
+    value = var.opensearch_replicas
+  }
+  set {
+    name  = "opensearch.persistence.size"
+    value = var.opensearch_storage_size
+  }
+  set {
+    name  = "opensearch.persistence.storageClass"
+    value = var.opensearch_storage_class
+  }
+  set {
+    name  = "opensearch.resources.requests.memory"
+    value = var.opensearch_node_memory
+  }
+  set {
+    name  = "opensearch.resources.limits.memory"
+    value = var.opensearch_node_memory
+  }
+  set {
+    name  = "opensearch.opensearchJavaOpts"
+    value = "-Xms${var.opensearch_jvm_heap} -Xmx${var.opensearch_jvm_heap}"
+  }
+
+  # --- Cortex sizing ---
+  set {
+    name  = "cortex.persistence.size"
+    value = var.cortex_storage_size
+  }
+  set {
+    name  = "cortex.persistence.storageClass"
+    value = var.cortex_storage_class
+  }
+
+  # --- Data Prepper sizing ---
+  set {
+    name  = "data-prepper.resources.requests.memory"
+    value = var.data_prepper_memory
+  }
+  set {
+    name  = "data-prepper.resources.limits.memory"
+    value = var.data_prepper_memory
+  }
+  # JAVA_OPTS via extraEnvs[0], emitted only when set (else the JVM uses its
+  # MaxRAMPercentage default). Claims index 0, so don't also set data-prepper.extraEnvs.
+  dynamic "set" {
+    for_each = var.data_prepper_jvm_heap == "" ? [] : [1]
+    content {
+      name  = "data-prepper.extraEnvs[0].name"
+      value = "JAVA_OPTS"
+    }
+  }
+  dynamic "set" {
+    for_each = var.data_prepper_jvm_heap == "" ? [] : [1]
+    content {
+      name  = "data-prepper.extraEnvs[0].value"
+      value = "-Xms${var.data_prepper_jvm_heap} -Xmx${var.data_prepper_jvm_heap}"
+    }
+  }
+  set {
+    name  = "dataPrepperTraceFlushInterval"
+    value = var.data_prepper_trace_flush_interval
+  }
+
   depends_on = [
     helm_release.aws_lb_controller,
   ]

terraform/aws/variables.tf

@@ -93,6 +93,87 @@ variable "tags" {
   }
 }
 
+# ============================================================================
+# OpenSearch sizing
+# ============================================================================
+
+variable "opensearch_replicas" {
+  description = "Number of OpenSearch nodes (StatefulSet replicas). 3 is the production minimum."
+  type        = number
+  default     = 3
+}
+
+variable "opensearch_storage_size" {
+  description = "Per-node OpenSearch PVC size. Total cluster storage is opensearch_replicas times this value."
+  type        = string
+  default     = "100Gi"
+}
+
+variable "opensearch_storage_class" {
+  description = "EBS storage class for OpenSearch PVCs."
+  type        = string
+  default     = "gp2"
+}
+
+variable "opensearch_node_memory" {
+  description = "Per-node OpenSearch container memory. The chart sets requests equal to limits."
+  type        = string
+  default     = "4Gi"
+}
+
+variable "opensearch_jvm_heap" {
+  description = "OpenSearch JVM heap, roughly 50% of opensearch_node_memory, max 31g. JVM size syntax, e.g. '2g' or '512m'."
+  type        = string
+  default     = "2g"
+  validation {
+    # JVM -Xms/-Xmx syntax (k/m/g), not Kubernetes 'Gi'.
+    condition     = can(regex("^[0-9]+[kmg]$", var.opensearch_jvm_heap))
+    error_message = "Use JVM size syntax like '2g' or '512m', not '2Gi'."
+  }
+}
+
+# ============================================================================
+# Cortex sizing
+# ============================================================================
+
+variable "cortex_storage_size" {
+  description = "Cortex PVC size."
+  type        = string
+  default     = "50Gi"
+}
+
+variable "cortex_storage_class" {
+  description = "EBS storage class for the Cortex PVC."
+  type        = string
+  default     = "gp2"
+}
+
+# ============================================================================
+# Data Prepper sizing
+# ============================================================================
+
+variable "data_prepper_memory" {
+  description = "Data Prepper container memory request and limit. Default matches the subchart."
+  type        = string
+  default     = "1Gi"
+}
+
+variable "data_prepper_jvm_heap" {
+  description = "Data Prepper JVM heap, roughly 75% of data_prepper_memory. JVM size syntax, e.g. '512m'. Empty leaves the JVM default."
+  type        = string
+  default     = ""
+  validation {
+    condition     = var.data_prepper_jvm_heap == "" || can(regex("^[0-9]+[kmg]$", var.data_prepper_jvm_heap))
+    error_message = "Use JVM size syntax like '2g' or '512m', not '2Gi'."
+  }
+}
+
+variable "data_prepper_trace_flush_interval" {
+  description = "Seconds the otel_traces processor buffers spans before computing traceGroup. Higher values raise Data Prepper heap; lower values can mark late-arriving spans as separate traces."
+  type        = number
+  default     = 180
+}
+
 # ============================================================================
 # Derived
 # ============================================================================