Openrca bench overlay

charts/observability-stack/files/init-opensearch-dashboards.py

@@ -778,8 +778,10 @@ def create_opensearch_datasource(workspace_id):
     # OSD_DATASOURCE_ENDPOINT lets operators override the endpoint written
     # onto the saved object — useful when OSD runs outside the compose
     # network and cannot resolve the `opensearch` service name. Falls back
-    # to the intra-network hostname when unset.
-    opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT", OPENSEARCH_ENDPOINT)
+    # to the intra-network hostname when unset or blank. Treat empty as
+    # unset — os.getenv's default only fires when the key is absent, not
+    # empty, and the env var may be set to "".
+    opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT") or OPENSEARCH_ENDPOINT
 
     payload = {
         "attributes": {
@@ -1135,6 +1137,87 @@ def set_default_dashboard(workspace_id, dashboard_id):
         print(f"⚠️  Error setting default dashboard: {e}")
 
 
+def set_trace_analytics_indices(workspace_id, span_indices, service_indices):
+    """Configure Trace Analytics span/service-map index patterns.
+
+    The OSD Observability plugin's Trace Analytics view defaults to
+    `otel-v1-apm-span-*` / `otel-v1-apm-service-map*`. Eval-suite data lives
+    under the `eval-*` prefix; pointing the plugin at a comma-separated
+    pattern list lets operators smoke-test eval ingest from the UI without
+    losing visibility of standard otel-* traffic.
+
+    Stored as user-editable advanced settings (saved objects), not as
+    `uiSettings.overrides` in opensearch_dashboards.yml — operators can
+    still flip patterns from Stack Management → Advanced Settings.
+
+    Writes at BOTH global scope and workspace scope. OSD's settings API
+    is keyed off the URL: `/api/opensearch-dashboards/settings` writes
+    the global config saved-object, `/w/<id>/api/opensearch-dashboards/settings`
+    writes a workspace-scoped one. Without the global write, operators
+    on the home page (no workspace context) see plugin defaults; without
+    the workspace write, operators inside the workspace see plugin
+    defaults. Both are needed for consistent behavior across the UI.
+
+    Idempotent: POSTing the same payload re-applies the same values.
+    Verified: a follow-up GET confirms the userValue matches what we
+    asked for; a mismatch is logged loudly so silent drift surfaces.
+    """
+    print(f"🔭 Setting Trace Analytics indices: spans={span_indices}, service-map={service_indices}")
+
+    expected = {
+        "observability:traceAnalyticsSpanIndices": span_indices,
+        "observability:traceAnalyticsServiceIndices": service_indices,
+        "observability:traceAnalyticsCustomModeDefault": True,
+    }
+
+    targets = [("global", f"{BASE_URL}/api/opensearch-dashboards/settings")]
+    if workspace_id and workspace_id != "default":
+        targets.append((
+            f"workspace {workspace_id}",
+            f"{BASE_URL}/w/{workspace_id}/api/opensearch-dashboards/settings",
+        ))
+
+    for scope, url in targets:
+        try:
+            response = requests.post(
+                url,
+                auth=(USERNAME, PASSWORD),
+                headers={"Content-Type": "application/json", "osd-xsrf": "true"},
+                json={"changes": expected},
+                verify=False,
+                timeout=10,
+            )
+
+            if response.status_code != 200:
+                print(f"⚠️  Setting Trace Analytics indices ({scope}) failed: {response.status_code} {response.text}")
+                continue
+
+            # POST returns 200 even when the value wasn't applied (plugin
+            # not loaded yet, role restriction). Re-read to confirm.
+            verify = requests.get(
+                url, auth=(USERNAME, PASSWORD), verify=False, timeout=10,
+            )
+            if verify.status_code != 200:
+                print(f"⚠️  Trace Analytics verify GET ({scope}) failed: {verify.status_code}")
+                continue
+
+            settings = verify.json().get("settings", {})
+            mismatches = []
+            for key, want in expected.items():
+                got = settings.get(key, {}).get("userValue")
+                if got != want:
+                    mismatches.append(f"{key}: want={want!r} got={got!r}")
+
+            if mismatches:
+                print(f"⚠️  Trace Analytics settings ({scope}) did not stick:")
+                for m in mismatches:
+                    print(f"     {m}")
+            else:
+                print(f"✅ Trace Analytics indices configured ({scope}, verified)")
+        except requests.exceptions.RequestException as e:
+            print(f"⚠️  Error setting Trace Analytics indices ({scope}): {e}")
+
+
 def create_agent_observability_dashboard(workspace_id, traces_pattern_id):
     """Create or update Agent Observability dashboard with visualizations"""
     import json
@@ -1833,8 +1916,30 @@ def main():
         workspace_id, "otel-v2-apm-service-map*", "timestamp"
     )
 
+    # Eval-suite (OpenRCA, etc.) index patterns. Created so operators get
+    # a ready-made Discover/Explore view of the eval-* prefix. The Trace
+    # Analytics plugin reads its own pattern setting (set below); this is
+    # for the standard Discover/Explore views.
+    create_index_pattern(
+        workspace_id, "eval-otel-v1-apm-span-openrca-*", "endTime", "traces",
+        display_name="Eval Trace Dataset - OpenRCA"
+    )
+    create_index_pattern(
+        workspace_id, "eval-otel-v1-logs-openrca-*", "time", "logs", logs_schema_mappings,
+        display_name="Eval Log Dataset - OpenRCA"
+    )
+
     print("📊 Created index patterns for spans, logs, and service map")
 
+    # Point Trace Analytics at both the eval-* prefix and the default otel-*
+    # prefix. Comma-separated patterns are supported by the plugin's
+    # helper_functions.tsx — the value is passed verbatim to OpenSearch.
+    set_trace_analytics_indices(
+        workspace_id,
+        span_indices="eval-otel-v1-apm-span-openrca-*,otel-v1-apm-span-*",
+        service_indices="otel-v2-apm-service-map*,otel-v1-apm-service-map*",
+    )
+
     # Set logs as the default index pattern
     if logs_pattern_id:
         set_default_index_pattern(workspace_id, logs_pattern_id)

charts/observability-stack/templates/data-prepper-metrics-service.yaml

@@ -0,0 +1,35 @@
+{{- if .Values.dataPrepperMetricsService.enabled }}
+{{- /*
+Data Prepper exposes its Micrometer Prometheus metrics on the admin
+server port (default 4900) at /metrics/prometheus and /metrics/sys. The
+data-prepper subchart hardcodes that port as the pod containerPort
+`server` but renders the Service only from `.Values.ports`, so 4900 is
+never published on the main Service — the collector's scrape of
+data-prepper:4900 is refused and the pipeline-health dashboard's Data
+Prepper panels stay empty.
+
+Adding 4900 to the subchart's `.Values.ports` would duplicate the
+containerPort (the subchart already declares `server`) and the API
+server rejects the Deployment. A separate Service avoids that: it
+targets the existing `server` containerPort by name, so no pod change.
+*/ -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-data-prepper-metrics
+  labels:
+    {{- /* Match the subchart's selector so this Service routes to DP pods. */}}
+    app.kubernetes.io/name: data-prepper
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: metrics
+spec:
+  type: ClusterIP
+  ports:
+    - name: metrics
+      port: {{ .Values.dataPrepperMetricsService.port }}
+      targetPort: server
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: data-prepper
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

charts/observability-stack/templates/data-prepper-pipeline-secret.yaml

@@ -39,7 +39,54 @@ stringData:
             entries:
               - from_key: "time"
                 to_key: "@timestamp"
+        # DP route's JsonPointer parser treats dots as nesting separators,
+        # so `/resource/attributes/openrca.dataset` looks up resource →
+        # attributes → openrca → dataset (which doesn't exist) instead of
+        # the literal-dot leaf key. Surface the dotted keys as flat
+        # top-level fields the route can address unambiguously.
+        - add_entries:
+            entries:
+              - key: openrca_dataset
+                value_expression: '/resource/attributes/openrca.dataset'
+                overwrite_if_key_exists: true
+              - key: openrca_cloudbed
+                value_expression: '/resource/attributes/openrca.cloudbed'
+                overwrite_if_key_exists: true
+      route:
+        - openrca_bank: '/openrca_dataset == "bank"'
+        # Market splits by cloudbed; the converter emits dataset="market"
+        # + cloudbed="cb1"|"cb2" as separate resource attributes.
+        - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"'
+        - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"'
       sink:
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            # `index_type: management_disabled` lets us write to a literal
+            # `index:` name. The `log-analytics-plain` index_type ignores
+            # the override and writes to its managed alias instead.
+            index_type: management_disabled
+            index: "eval-otel-v1-logs-openrca-bank"
+            routes: [openrca_bank]
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            index_type: management_disabled
+            index: "eval-otel-v1-logs-openrca-market-cb1"
+            routes: [openrca_market_cb1]
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            index_type: management_disabled
+            index: "eval-otel-v1-logs-openrca-market-cb2"
+            routes: [openrca_market_cb2]
+        # Catch-all for non-OpenRCA traffic (live demo, ad-hoc OTLP).
         - opensearch:
             hosts: [{{ $opensearchHost | quote }}]
             username: {{ .Values.opensearchUsername | quote }}
@@ -63,8 +110,64 @@ stringData:
         pipeline:
           name: "otel-traces-pipeline"
       processor:
-        - otel_traces: {}
+        - otel_traces:
+            trace_flush_interval: {{ .Values.dataPrepperTraceFlushInterval | default 180 }}
+        # DP route's JsonPointer parser treats dots as nesting separators,
+        # so `/resource/attributes/openrca.dataset` looks up resource →
+        # attributes → openrca → dataset (which doesn't exist) instead of
+        # the literal-dot leaf key. Surface the dotted keys as flat
+        # top-level fields the route can address unambiguously.
+        - add_entries:
+            entries:
+              - key: openrca_dataset
+                value_expression: '/resource/attributes/openrca.dataset'
+                overwrite_if_key_exists: true
+              - key: openrca_cloudbed
+                value_expression: '/resource/attributes/openrca.cloudbed'
+                overwrite_if_key_exists: true
+      route:
+        - openrca_telecom: '/openrca_dataset == "telecom"'
+        - openrca_bank: '/openrca_dataset == "bank"'
+        # Market splits by cloudbed.
+        - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"'
+        - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"'
       sink:
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            # `index_type: management_disabled` lets us write to a literal
+            # `index:` name. The `trace-analytics-plain-raw` index_type
+            # ignores the override and writes to its managed alias instead.
+            index_type: management_disabled
+            index: "eval-otel-v1-apm-span-openrca-telecom"
+            routes: [openrca_telecom]
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            index_type: management_disabled
+            index: "eval-otel-v1-apm-span-openrca-bank"
+            routes: [openrca_bank]
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            index_type: management_disabled
+            index: "eval-otel-v1-apm-span-openrca-market-cb1"
+            routes: [openrca_market_cb1]
+        - opensearch:
+            hosts: [{{ $opensearchHost | quote }}]
+            username: {{ .Values.opensearchUsername | quote }}
+            password: {{ .Values.opensearchPassword | quote }}
+            insecure: true
+            index_type: management_disabled
+            index: "eval-otel-v1-apm-span-openrca-market-cb2"
+            routes: [openrca_market_cb2]
+        # Catch-all for non-OpenRCA traffic (live demo, ad-hoc OTLP).
         - opensearch:
             hosts: [{{ $opensearchHost | quote }}]
             username: {{ .Values.opensearchUsername | quote }}

charts/observability-stack/templates/otel-collector-configmap.yaml

@@ -85,15 +85,15 @@ data:
               scrape_interval: 15s
               metrics_path: /metrics/sys
               static_configs:
-                - targets: ["{{ .Release.Name }}-data-prepper:4900"]
+                - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"]
               relabel_configs:
                 - target_label: service.name
                   replacement: data-prepper
             - job_name: data-prepper-pipelines
               scrape_interval: 15s
               metrics_path: /metrics/prometheus
               static_configs:
-                - targets: ["{{ .Release.Name }}-data-prepper:4900"]
+                - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"]
               relabel_configs:
                 - target_label: service.name
                   replacement: data-prepper

charts/observability-stack/values.yaml

@@ -181,8 +181,11 @@ data-prepper:
       port: 21891
     - name: otel-logs
       port: 21892
-    - name: metrics
-      port: 4900
+    # Port 4900 (the subchart's hardcoded `server` containerPort, serving
+    # /metrics/prometheus) is intentionally NOT listed here: adding it would
+    # duplicate the containerPort and the API server rejects the Deployment.
+    # It is published instead by a separate metrics Service — see
+    # templates/data-prepper-metrics-service.yaml and dataPrepperMetricsService.
   config:
     data-prepper-config.yaml: |
       ssl: false
@@ -210,6 +213,23 @@ data-prepper:
     enabled: false
     existingSecret: data-prepper-pipeline
 
+# -- Data Prepper otel_traces processor: trace_flush_interval (seconds).
+# How long the processor buffers spans before computing traceGroup and
+# emitting completed traces. Default 180s matches upstream. Lower at the
+# cost of marking late-arriving traces incomplete; raise at the cost of
+# DP heap usage. See trace_duration_distribution.md for OpenRCA-bench
+# sizing math.
+dataPrepperTraceFlushInterval: 180
+
+# -- Data Prepper metrics Service. Publishes the admin/metrics port (4900,
+# /metrics/prometheus) on a dedicated ClusterIP Service so Prometheus /
+# the collector can scrape Data Prepper's pipeline metrics. The subchart
+# only puts 4900 on the pod, not on its Service, so without this the
+# pipeline-health dashboard's Data Prepper panels stay empty.
+dataPrepperMetricsService:
+  enabled: true
+  port: 4900
+
 # -- OpenTelemetry Collector
 opentelemetry-collector:
   enabled: true