diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 5fc342ed..091f3cb4 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -778,8 +778,10 @@ def create_opensearch_datasource(workspace_id): # OSD_DATASOURCE_ENDPOINT lets operators override the endpoint written # onto the saved object — useful when OSD runs outside the compose # network and cannot resolve the `opensearch` service name. Falls back - # to the intra-network hostname when unset. - opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT", OPENSEARCH_ENDPOINT) + # to the intra-network hostname when unset or blank. Treat empty as + # unset — os.getenv's default only fires when the key is absent, not + # empty, and the env var may be set to "". + opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT") or OPENSEARCH_ENDPOINT payload = { "attributes": { @@ -1135,6 +1137,87 @@ def set_default_dashboard(workspace_id, dashboard_id): print(f"⚠️ Error setting default dashboard: {e}") +def set_trace_analytics_indices(workspace_id, span_indices, service_indices): + """Configure Trace Analytics span/service-map index patterns. + + The OSD Observability plugin's Trace Analytics view defaults to + `otel-v1-apm-span-*` / `otel-v1-apm-service-map*`. Eval-suite data lives + under the `eval-*` prefix; pointing the plugin at a comma-separated + pattern list lets operators smoke-test eval ingest from the UI without + losing visibility of standard otel-* traffic. + + Stored as user-editable advanced settings (saved objects), not as + `uiSettings.overrides` in opensearch_dashboards.yml — operators can + still flip patterns from Stack Management → Advanced Settings. + + Writes at BOTH global scope and workspace scope. OSD's settings API + is keyed off the URL: `/api/opensearch-dashboards/settings` writes + the global config saved-object, `/w//api/opensearch-dashboards/settings` + writes a workspace-scoped one. Without the global write, operators + on the home page (no workspace context) see plugin defaults; without + the workspace write, operators inside the workspace see plugin + defaults. Both are needed for consistent behavior across the UI. + + Idempotent: POSTing the same payload re-applies the same values. + Verified: a follow-up GET confirms the userValue matches what we + asked for; a mismatch is logged loudly so silent drift surfaces. + """ + print(f"🔭 Setting Trace Analytics indices: spans={span_indices}, service-map={service_indices}") + + expected = { + "observability:traceAnalyticsSpanIndices": span_indices, + "observability:traceAnalyticsServiceIndices": service_indices, + "observability:traceAnalyticsCustomModeDefault": True, + } + + targets = [("global", f"{BASE_URL}/api/opensearch-dashboards/settings")] + if workspace_id and workspace_id != "default": + targets.append(( + f"workspace {workspace_id}", + f"{BASE_URL}/w/{workspace_id}/api/opensearch-dashboards/settings", + )) + + for scope, url in targets: + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json={"changes": expected}, + verify=False, + timeout=10, + ) + + if response.status_code != 200: + print(f"⚠️ Setting Trace Analytics indices ({scope}) failed: {response.status_code} {response.text}") + continue + + # POST returns 200 even when the value wasn't applied (plugin + # not loaded yet, role restriction). Re-read to confirm. + verify = requests.get( + url, auth=(USERNAME, PASSWORD), verify=False, timeout=10, + ) + if verify.status_code != 200: + print(f"⚠️ Trace Analytics verify GET ({scope}) failed: {verify.status_code}") + continue + + settings = verify.json().get("settings", {}) + mismatches = [] + for key, want in expected.items(): + got = settings.get(key, {}).get("userValue") + if got != want: + mismatches.append(f"{key}: want={want!r} got={got!r}") + + if mismatches: + print(f"⚠️ Trace Analytics settings ({scope}) did not stick:") + for m in mismatches: + print(f" {m}") + else: + print(f"✅ Trace Analytics indices configured ({scope}, verified)") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error setting Trace Analytics indices ({scope}): {e}") + + def create_agent_observability_dashboard(workspace_id, traces_pattern_id): """Create or update Agent Observability dashboard with visualizations""" import json @@ -1833,8 +1916,30 @@ def main(): workspace_id, "otel-v2-apm-service-map*", "timestamp" ) + # Eval-suite (OpenRCA, etc.) index patterns. Created so operators get + # a ready-made Discover/Explore view of the eval-* prefix. The Trace + # Analytics plugin reads its own pattern setting (set below); this is + # for the standard Discover/Explore views. + create_index_pattern( + workspace_id, "eval-otel-v1-apm-span-openrca-*", "endTime", "traces", + display_name="Eval Trace Dataset - OpenRCA" + ) + create_index_pattern( + workspace_id, "eval-otel-v1-logs-openrca-*", "time", "logs", logs_schema_mappings, + display_name="Eval Log Dataset - OpenRCA" + ) + print("📊 Created index patterns for spans, logs, and service map") + # Point Trace Analytics at both the eval-* prefix and the default otel-* + # prefix. Comma-separated patterns are supported by the plugin's + # helper_functions.tsx — the value is passed verbatim to OpenSearch. + set_trace_analytics_indices( + workspace_id, + span_indices="eval-otel-v1-apm-span-openrca-*,otel-v1-apm-span-*", + service_indices="otel-v2-apm-service-map*,otel-v1-apm-service-map*", + ) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) diff --git a/charts/observability-stack/templates/data-prepper-metrics-service.yaml b/charts/observability-stack/templates/data-prepper-metrics-service.yaml new file mode 100644 index 00000000..dfc9299f --- /dev/null +++ b/charts/observability-stack/templates/data-prepper-metrics-service.yaml @@ -0,0 +1,35 @@ +{{- if .Values.dataPrepperMetricsService.enabled }} +{{- /* +Data Prepper exposes its Micrometer Prometheus metrics on the admin +server port (default 4900) at /metrics/prometheus and /metrics/sys. The +data-prepper subchart hardcodes that port as the pod containerPort +`server` but renders the Service only from `.Values.ports`, so 4900 is +never published on the main Service — the collector's scrape of +data-prepper:4900 is refused and the pipeline-health dashboard's Data +Prepper panels stay empty. + +Adding 4900 to the subchart's `.Values.ports` would duplicate the +containerPort (the subchart already declares `server`) and the API +server rejects the Deployment. A separate Service avoids that: it +targets the existing `server` containerPort by name, so no pod change. +*/ -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-data-prepper-metrics + labels: + {{- /* Match the subchart's selector so this Service routes to DP pods. */}} + app.kubernetes.io/name: data-prepper + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: metrics +spec: + type: ClusterIP + ports: + - name: metrics + port: {{ .Values.dataPrepperMetricsService.port }} + targetPort: server + protocol: TCP + selector: + app.kubernetes.io/name: data-prepper + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml b/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml index f95d328b..62a26a76 100644 --- a/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml +++ b/charts/observability-stack/templates/data-prepper-pipeline-secret.yaml @@ -39,7 +39,54 @@ stringData: entries: - from_key: "time" to_key: "@timestamp" + # DP route's JsonPointer parser treats dots as nesting separators, + # so `/resource/attributes/openrca.dataset` looks up resource → + # attributes → openrca → dataset (which doesn't exist) instead of + # the literal-dot leaf key. Surface the dotted keys as flat + # top-level fields the route can address unambiguously. + - add_entries: + entries: + - key: openrca_dataset + value_expression: '/resource/attributes/openrca.dataset' + overwrite_if_key_exists: true + - key: openrca_cloudbed + value_expression: '/resource/attributes/openrca.cloudbed' + overwrite_if_key_exists: true + route: + - openrca_bank: '/openrca_dataset == "bank"' + # Market splits by cloudbed; the converter emits dataset="market" + # + cloudbed="cb1"|"cb2" as separate resource attributes. + - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"' + - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"' sink: + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + # `index_type: management_disabled` lets us write to a literal + # `index:` name. The `log-analytics-plain` index_type ignores + # the override and writes to its managed alias instead. + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-bank" + routes: [openrca_bank] + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-market-cb1" + routes: [openrca_market_cb1] + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-market-cb2" + routes: [openrca_market_cb2] + # Catch-all for non-OpenRCA traffic (live demo, ad-hoc OTLP). - opensearch: hosts: [{{ $opensearchHost | quote }}] username: {{ .Values.opensearchUsername | quote }} @@ -63,8 +110,64 @@ stringData: pipeline: name: "otel-traces-pipeline" processor: - - otel_traces: {} + - otel_traces: + trace_flush_interval: {{ .Values.dataPrepperTraceFlushInterval | default 180 }} + # DP route's JsonPointer parser treats dots as nesting separators, + # so `/resource/attributes/openrca.dataset` looks up resource → + # attributes → openrca → dataset (which doesn't exist) instead of + # the literal-dot leaf key. Surface the dotted keys as flat + # top-level fields the route can address unambiguously. + - add_entries: + entries: + - key: openrca_dataset + value_expression: '/resource/attributes/openrca.dataset' + overwrite_if_key_exists: true + - key: openrca_cloudbed + value_expression: '/resource/attributes/openrca.cloudbed' + overwrite_if_key_exists: true + route: + - openrca_telecom: '/openrca_dataset == "telecom"' + - openrca_bank: '/openrca_dataset == "bank"' + # Market splits by cloudbed. + - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"' + - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"' sink: + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + # `index_type: management_disabled` lets us write to a literal + # `index:` name. The `trace-analytics-plain-raw` index_type + # ignores the override and writes to its managed alias instead. + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-telecom" + routes: [openrca_telecom] + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-bank" + routes: [openrca_bank] + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-market-cb1" + routes: [openrca_market_cb1] + - opensearch: + hosts: [{{ $opensearchHost | quote }}] + username: {{ .Values.opensearchUsername | quote }} + password: {{ .Values.opensearchPassword | quote }} + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-market-cb2" + routes: [openrca_market_cb2] + # Catch-all for non-OpenRCA traffic (live demo, ad-hoc OTLP). - opensearch: hosts: [{{ $opensearchHost | quote }}] username: {{ .Values.opensearchUsername | quote }} diff --git a/charts/observability-stack/templates/otel-collector-configmap.yaml b/charts/observability-stack/templates/otel-collector-configmap.yaml index 02c5f1be..c22bf7dd 100644 --- a/charts/observability-stack/templates/otel-collector-configmap.yaml +++ b/charts/observability-stack/templates/otel-collector-configmap.yaml @@ -85,7 +85,7 @@ data: scrape_interval: 15s metrics_path: /metrics/sys static_configs: - - targets: ["{{ .Release.Name }}-data-prepper:4900"] + - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"] relabel_configs: - target_label: service.name replacement: data-prepper @@ -93,7 +93,7 @@ data: scrape_interval: 15s metrics_path: /metrics/prometheus static_configs: - - targets: ["{{ .Release.Name }}-data-prepper:4900"] + - targets: ["{{ .Release.Name }}-data-prepper-metrics:4900"] relabel_configs: - target_label: service.name replacement: data-prepper diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 3087c0fd..78c38caf 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -181,8 +181,11 @@ data-prepper: port: 21891 - name: otel-logs port: 21892 - - name: metrics - port: 4900 + # Port 4900 (the subchart's hardcoded `server` containerPort, serving + # /metrics/prometheus) is intentionally NOT listed here: adding it would + # duplicate the containerPort and the API server rejects the Deployment. + # It is published instead by a separate metrics Service — see + # templates/data-prepper-metrics-service.yaml and dataPrepperMetricsService. config: data-prepper-config.yaml: | ssl: false @@ -210,6 +213,23 @@ data-prepper: enabled: false existingSecret: data-prepper-pipeline +# -- Data Prepper otel_traces processor: trace_flush_interval (seconds). +# How long the processor buffers spans before computing traceGroup and +# emitting completed traces. Default 180s matches upstream. Lower at the +# cost of marking late-arriving traces incomplete; raise at the cost of +# DP heap usage. See trace_duration_distribution.md for OpenRCA-bench +# sizing math. +dataPrepperTraceFlushInterval: 180 + +# -- Data Prepper metrics Service. Publishes the admin/metrics port (4900, +# /metrics/prometheus) on a dedicated ClusterIP Service so Prometheus / +# the collector can scrape Data Prepper's pipeline metrics. The subchart +# only puts 4900 on the pod, not on its Service, so without this the +# pipeline-health dashboard's Data Prepper panels stay empty. +dataPrepperMetricsService: + enabled: true + port: 4900 + # -- OpenTelemetry Collector opentelemetry-collector: enabled: true diff --git a/docker-compose/data-prepper/pipelines.template.yaml b/docker-compose/data-prepper/pipelines.template.yaml index 9627119f..b16a564f 100644 --- a/docker-compose/data-prepper/pipelines.template.yaml +++ b/docker-compose/data-prepper/pipelines.template.yaml @@ -41,15 +41,57 @@ otel-logs-pipeline: entries: - from_key: "time" to_key: "@timestamp" + # DP route's JsonPointer parser treats dots as nesting separators, + # so `/resource/attributes/openrca.dataset` looks up resource → + # attributes → openrca → dataset (which doesn't exist) instead of + # the literal-dot leaf key. Surface the dotted keys as flat + # top-level fields the route can address unambiguously. + - add_entries: + entries: + - key: openrca_dataset + value_expression: '/resource/attributes/openrca.dataset' + overwrite_if_key_exists: true + - key: openrca_cloudbed + value_expression: '/resource/attributes/openrca.cloudbed' + overwrite_if_key_exists: true + route: + - openrca_bank: '/openrca_dataset == "bank"' + # Market splits by cloudbed; the converter emits dataset="market" + # + cloudbed="cb1"|"cb2" as separate resource attributes. + - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"' + - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"' # Write processed logs to OpenSearch sink: - opensearch: hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] username: OPENSEARCH_USER password: OPENSEARCH_PASSWORD - # Disable SSL verification for development insecure: true - # Use log analytics index type for automatic index management + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-bank" + routes: [openrca_bank] + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-market-cb1" + routes: [openrca_market_cb1] + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true + index_type: management_disabled + index: "eval-otel-v1-logs-openrca-market-cb2" + routes: [openrca_market_cb2] + # Catch-all for non-OpenRCA traffic. + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true index_type: log-analytics-plain # Trace processing pipeline @@ -75,13 +117,64 @@ traces-raw-pipeline: processor: # Process raw trace data for OpenSearch storage - otel_traces: + # DP route's JsonPointer parser treats dots as nesting separators, + # so `/resource/attributes/openrca.dataset` looks up resource → + # attributes → openrca → dataset (which doesn't exist) instead of + # the literal-dot leaf key. Surface the dotted keys as flat + # top-level fields the route can address unambiguously. + - add_entries: + entries: + - key: openrca_dataset + value_expression: '/resource/attributes/openrca.dataset' + overwrite_if_key_exists: true + - key: openrca_cloudbed + value_expression: '/resource/attributes/openrca.cloudbed' + overwrite_if_key_exists: true + route: + - openrca_telecom: '/openrca_dataset == "telecom"' + - openrca_bank: '/openrca_dataset == "bank"' + # Market splits by cloudbed. + - openrca_market_cb1: '/openrca_dataset == "market" and /openrca_cloudbed == "cb1"' + - openrca_market_cb2: '/openrca_dataset == "market" and /openrca_cloudbed == "cb2"' sink: - opensearch: hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] username: OPENSEARCH_USER password: OPENSEARCH_PASSWORD insecure: true - # Use trace analytics index type for automatic index management + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-telecom" + routes: [openrca_telecom] + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-bank" + routes: [openrca_bank] + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-market-cb1" + routes: [openrca_market_cb1] + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true + index_type: management_disabled + index: "eval-otel-v1-apm-span-openrca-market-cb2" + routes: [openrca_market_cb2] + # Catch-all for non-OpenRCA traffic (live demo, ad-hoc OTLP). + - opensearch: + hosts: ["OPENSEARCH_PROTOCOL://OPENSEARCH_HOST:OPENSEARCH_PORT"] + username: OPENSEARCH_USER + password: OPENSEARCH_PASSWORD + insecure: true index_type: trace-analytics-plain-raw # Service map generation pipeline (APM) diff --git a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py index f1980d90..091f3cb4 100644 --- a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py +++ b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py @@ -8,15 +8,15 @@ _dashboards_host = os.getenv("OPENSEARCH_DASHBOARDS_HOST", "opensearch-dashboards") _dashboards_port = os.getenv("OPENSEARCH_DASHBOARDS_PORT", "5601") _dashboards_protocol = os.getenv("OPENSEARCH_DASHBOARDS_PROTOCOL", "http") -BASE_URL = f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}" +BASE_URL = os.getenv("BASE_URL", f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}") USERNAME = os.getenv("OPENSEARCH_USER", "admin") PASSWORD = os.getenv("OPENSEARCH_PASSWORD", "My_password_123!@#") -PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus.observability-stack-network") +PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus") PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090") ALERTMANAGER_HOST = os.getenv("ALERTMANAGER_HOST", "alertmanager") ALERTMANAGER_PORT = os.getenv("ALERTMANAGER_PORT", "9093") _opensearch_protocol = os.getenv("OPENSEARCH_PROTOCOL", "https") -OPENSEARCH_ENDPOINT = f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}" +OPENSEARCH_ENDPOINT = os.getenv("OPENSEARCH_ENDPOINT", f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}") ISM_RETENTION_DAYS = int(os.getenv("ISM_RETENTION_DAYS", "7")) @@ -179,6 +179,43 @@ def create_workspace(): return "default" +def set_default_workspace(workspace_id): + """Set the default workspace so all users land here on login. + + When workspace.enabled is true, users see a workspace picker on first load. + Setting defaultWorkspace directs all users (including anonymous) straight + to the Observability Stack workspace instead. + """ + if not workspace_id or workspace_id == "default": + print("⏭️ Skipping default workspace (using default)") + return False + + print(f"⭐ Setting default workspace: {workspace_id}") + + url = f"{BASE_URL}/api/opensearch-dashboards/settings" + payload = {"changes": {"defaultWorkspace": workspace_id}} + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + print("✅ Default workspace set") + return True + else: + print(f"⚠️ Failed to set default workspace: {response.status_code} {response.text}") + return False + except requests.exceptions.RequestException as e: + print(f"⚠️ Error setting default workspace: {e}") + return False + + def get_existing_index_pattern(workspace_id, title): """Check if an index pattern with the given title already exists""" try: @@ -342,25 +379,59 @@ def create_prometheus_datasource(workspace_id): # Check if datasource already exists existing_id = get_existing_prometheus_datasource(datasource_name) if existing_id: - print(f"✅ Prometheus datasource already exists: {existing_id}") - reconciled = reconcile_prometheus_datasource_properties( - datasource_name, desired_properties - ) - # Reconciliation goes through DELETE + POST, so the saved-object - # id may have changed — re-read before associating. - datasource_id = existing_id - if reconciled: - datasource_id = get_existing_prometheus_datasource(datasource_name) or existing_id - # Associate with workspace if provided - if workspace_id and workspace_id != "default": - associate_prometheus_with_workspace(workspace_id, datasource_id) - return datasource_id + # Verify the datasource is functional (masterkey can decrypt it). + # On helm upgrades the plugins.query.datasources.encryption.masterkey + # may differ from the key used to encrypt the stored secret; the + # SQL plugin then surfaces a decryption error on every query. Detect + # and wipe so the fresh POST below re-creates it with the current key. + broken = False + try: + verify = requests.get( + f"{OPENSEARCH_ENDPOINT}/_plugins/_query/_datasources/{datasource_name}", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json"}, + verify=False, + timeout=10, + ) + if verify.status_code == 200 and "error" not in verify.json(): + pass + else: + print(f"⚠️ Prometheus datasource exists but is broken (encryption key mismatch). Deleting to recreate...") + requests.delete( + f"{OPENSEARCH_ENDPOINT}/_plugins/_query/_datasources/{datasource_name}", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json"}, + verify=False, + timeout=10, + ) + broken = True + except requests.exceptions.RequestException: + pass + + if not broken: + print(f"✅ Prometheus datasource already exists: {existing_id}") + reconciled = reconcile_prometheus_datasource_properties( + datasource_name, desired_properties + ) + # Reconciliation goes through DELETE + POST, so the saved-object + # id may have changed — re-read before associating. + datasource_id = existing_id + if reconciled: + datasource_id = get_existing_prometheus_datasource(datasource_name) or existing_id + # Associate with workspace if provided + if workspace_id and workspace_id != "default": + associate_prometheus_with_workspace(workspace_id, datasource_id) + return datasource_id print("🔧 Creating Prometheus datasource...") + # Grant anonymous users access to the Prometheus datasource when anonymous auth is enabled + anonymous_auth = os.getenv("ANONYMOUS_AUTH_ENABLED", "false").lower() == "true" + allowed_roles = ["all_access", "opendistro_security_anonymous_role"] if anonymous_auth else ["all_access"] + payload = { "name": datasource_name, - "allowedRoles": [], + "allowedRoles": allowed_roles, "connector": "prometheus", "properties": desired_properties, } @@ -707,8 +778,10 @@ def create_opensearch_datasource(workspace_id): # OSD_DATASOURCE_ENDPOINT lets operators override the endpoint written # onto the saved object — useful when OSD runs outside the compose # network and cannot resolve the `opensearch` service name. Falls back - # to the intra-network hostname when unset. - opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT", OPENSEARCH_ENDPOINT) + # to the intra-network hostname when unset or blank. Treat empty as + # unset — os.getenv's default only fires when the key is absent, not + # empty, and the env var may be set to "". + opensearch_endpoint = os.getenv("OSD_DATASOURCE_ENDPOINT") or OPENSEARCH_ENDPOINT payload = { "attributes": { @@ -1064,6 +1137,87 @@ def set_default_dashboard(workspace_id, dashboard_id): print(f"⚠️ Error setting default dashboard: {e}") +def set_trace_analytics_indices(workspace_id, span_indices, service_indices): + """Configure Trace Analytics span/service-map index patterns. + + The OSD Observability plugin's Trace Analytics view defaults to + `otel-v1-apm-span-*` / `otel-v1-apm-service-map*`. Eval-suite data lives + under the `eval-*` prefix; pointing the plugin at a comma-separated + pattern list lets operators smoke-test eval ingest from the UI without + losing visibility of standard otel-* traffic. + + Stored as user-editable advanced settings (saved objects), not as + `uiSettings.overrides` in opensearch_dashboards.yml — operators can + still flip patterns from Stack Management → Advanced Settings. + + Writes at BOTH global scope and workspace scope. OSD's settings API + is keyed off the URL: `/api/opensearch-dashboards/settings` writes + the global config saved-object, `/w//api/opensearch-dashboards/settings` + writes a workspace-scoped one. Without the global write, operators + on the home page (no workspace context) see plugin defaults; without + the workspace write, operators inside the workspace see plugin + defaults. Both are needed for consistent behavior across the UI. + + Idempotent: POSTing the same payload re-applies the same values. + Verified: a follow-up GET confirms the userValue matches what we + asked for; a mismatch is logged loudly so silent drift surfaces. + """ + print(f"🔭 Setting Trace Analytics indices: spans={span_indices}, service-map={service_indices}") + + expected = { + "observability:traceAnalyticsSpanIndices": span_indices, + "observability:traceAnalyticsServiceIndices": service_indices, + "observability:traceAnalyticsCustomModeDefault": True, + } + + targets = [("global", f"{BASE_URL}/api/opensearch-dashboards/settings")] + if workspace_id and workspace_id != "default": + targets.append(( + f"workspace {workspace_id}", + f"{BASE_URL}/w/{workspace_id}/api/opensearch-dashboards/settings", + )) + + for scope, url in targets: + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json={"changes": expected}, + verify=False, + timeout=10, + ) + + if response.status_code != 200: + print(f"⚠️ Setting Trace Analytics indices ({scope}) failed: {response.status_code} {response.text}") + continue + + # POST returns 200 even when the value wasn't applied (plugin + # not loaded yet, role restriction). Re-read to confirm. + verify = requests.get( + url, auth=(USERNAME, PASSWORD), verify=False, timeout=10, + ) + if verify.status_code != 200: + print(f"⚠️ Trace Analytics verify GET ({scope}) failed: {verify.status_code}") + continue + + settings = verify.json().get("settings", {}) + mismatches = [] + for key, want in expected.items(): + got = settings.get(key, {}).get("userValue") + if got != want: + mismatches.append(f"{key}: want={want!r} got={got!r}") + + if mismatches: + print(f"⚠️ Trace Analytics settings ({scope}) did not stick:") + for m in mismatches: + print(f" {m}") + else: + print(f"✅ Trace Analytics indices configured ({scope}, verified)") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error setting Trace Analytics indices ({scope}): {e}") + + def create_agent_observability_dashboard(workspace_id, traces_pattern_id): """Create or update Agent Observability dashboard with visualizations""" import json @@ -1744,6 +1898,9 @@ def main(): else: workspace_id = create_workspace() + # Set as default workspace so users skip the workspace picker + set_default_workspace(workspace_id) + # Create index patterns (idempotent - will skip if already exist) # Titles must match exactly what the APM plugin expects logs_schema_mappings = '{"otelLogs":{"timestamp":"time","traceId":"traceId","spanId":"spanId","serviceName":"resource.attributes.service.name"}}' @@ -1759,8 +1916,30 @@ def main(): workspace_id, "otel-v2-apm-service-map*", "timestamp" ) + # Eval-suite (OpenRCA, etc.) index patterns. Created so operators get + # a ready-made Discover/Explore view of the eval-* prefix. The Trace + # Analytics plugin reads its own pattern setting (set below); this is + # for the standard Discover/Explore views. + create_index_pattern( + workspace_id, "eval-otel-v1-apm-span-openrca-*", "endTime", "traces", + display_name="Eval Trace Dataset - OpenRCA" + ) + create_index_pattern( + workspace_id, "eval-otel-v1-logs-openrca-*", "time", "logs", logs_schema_mappings, + display_name="Eval Log Dataset - OpenRCA" + ) + print("📊 Created index patterns for spans, logs, and service map") + # Point Trace Analytics at both the eval-* prefix and the default otel-* + # prefix. Comma-separated patterns are supported by the plugin's + # helper_functions.tsx — the value is passed verbatim to OpenSearch. + set_trace_analytics_indices( + workspace_id, + span_indices="eval-otel-v1-apm-span-openrca-*,otel-v1-apm-span-*", + service_indices="otel-v2-apm-service-map*,otel-v1-apm-service-map*", + ) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) @@ -1777,9 +1956,13 @@ def main(): create_overview_dashboard(workspace_id) # Create self-monitoring dashboards (PromQL explore panels) + create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-k8s-cluster-health.yaml") create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-pipeline-health.yaml") create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-opensearch-health.yaml") + # Create saved queries for common agent observability patterns + create_default_saved_queries(workspace_id) + # Create datasources (must happen before ndjson import so Prometheus references resolve) prometheus_datasource_id = create_prometheus_datasource(workspace_id) create_opensearch_datasource(workspace_id) @@ -1794,9 +1977,6 @@ def main(): ndjson_id_mappings["54f4c1f0-2938-11f1-84ad-e734b5ac5a91"] = traces_pattern_id import_ndjson_dashboard(workspace_id, "/config/dashboard-astronomy-shop.ndjson", ndjson_id_mappings) - # Create saved queries for common agent observability patterns - create_default_saved_queries(workspace_id) - # Create APM config correlation (ties traces + service map + Prometheus) if traces_pattern_id and service_map_pattern_id: # Resolve Prometheus data-connection saved object ID @@ -1808,8 +1988,6 @@ def main(): # Output summary print() print("🎉 Observability Stack Ready!") - print(f"👤 Username: {USERNAME}") - print(f"🔑 Password: {PASSWORD}") # Generate appropriate dashboard URL if workspace_id and workspace_id != "default": @@ -1896,7 +2074,6 @@ def delayed_field_refresh(workspace_id, patterns): main() # Re-read workspace and pattern IDs for the delayed refresh. - # main() already printed success, so this is a background follow-up. workspace_id = get_existing_workspace() logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index fec3e16a..cc941207 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -192,6 +192,74 @@ resource "helm_release" "observability_stack" { } } + # --- OpenSearch sizing --- + set { + name = "opensearch.replicas" + value = var.opensearch_replicas + } + set { + name = "opensearch.persistence.size" + value = var.opensearch_storage_size + } + set { + name = "opensearch.persistence.storageClass" + value = var.opensearch_storage_class + } + set { + name = "opensearch.resources.requests.memory" + value = var.opensearch_node_memory + } + set { + name = "opensearch.resources.limits.memory" + value = var.opensearch_node_memory + } + set { + name = "opensearch.opensearchJavaOpts" + value = "-Xms${var.opensearch_jvm_heap} -Xmx${var.opensearch_jvm_heap}" + } + + # --- Cortex sizing --- + set { + name = "cortex.persistence.size" + value = var.cortex_storage_size + } + set { + name = "cortex.persistence.storageClass" + value = var.cortex_storage_class + } + + # --- Data Prepper sizing --- + set { + name = "data-prepper.resources.requests.memory" + value = var.data_prepper_memory + } + set { + name = "data-prepper.resources.limits.memory" + value = var.data_prepper_memory + } + # JAVA_OPTS via the subchart's extraEnvs[]. Only emit when an override is + # given; otherwise the JVM picks heap from MaxRAMPercentage defaults. + dynamic "set" { + for_each = var.data_prepper_jvm_heap == "" ? [] : [1] + content { + name = "data-prepper.extraEnvs[0].name" + value = "JAVA_OPTS" + } + } + dynamic "set" { + for_each = var.data_prepper_jvm_heap == "" ? [] : [1] + content { + name = "data-prepper.extraEnvs[0].value" + value = "-Xms${var.data_prepper_jvm_heap} -Xmx${var.data_prepper_jvm_heap}" + } + } + # The pipeline secret template reads .Values.dataPrepperTraceFlushInterval + # to parameterize the otel_traces processor flush window. + set { + name = "dataPrepperTraceFlushInterval" + value = var.data_prepper_trace_flush_interval + } + depends_on = [ helm_release.aws_lb_controller, ] diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 4d097899..8c4e7466 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -93,6 +93,81 @@ variable "tags" { } } +# ============================================================================ +# OpenSearch sizing — bump up for high-volume ingest workloads (e.g. RCA +# benchmark dataset at ~290 GB raw NDJSON / ~1 TB indexed with 1 replica) +# ============================================================================ + +variable "opensearch_replicas" { + description = "Number of OpenSearch nodes (StatefulSet replicas). 3 is the production minimum." + type = number + default = 3 +} + +variable "opensearch_storage_size" { + description = "Per-node OpenSearch PVC size, e.g. 100Gi for default, 500Gi for large-ingest workloads. Total cluster storage = opensearch_replicas × this value." + type = string + default = "100Gi" +} + +variable "opensearch_storage_class" { + description = "EBS storage class for OpenSearch PVCs. gp3 is cheaper and gives provisioned-IOPS knobs vs gp2; both are valid." + type = string + default = "gp2" +} + +variable "opensearch_node_memory" { + description = "Per-node OpenSearch container memory request/limit (e.g. 4Gi default, 16Gi for high-volume). The chart sets requests=limits, so set to whatever the node should reserve." + type = string + default = "4Gi" +} + +variable "opensearch_jvm_heap" { + description = "OpenSearch JVM heap size. Should be ~50% of opensearch_node_memory, max 31g. Use the matching G/g suffix the chart expects (e.g. '2g', '8g', '16g')." + type = string + default = "2g" +} + +# ============================================================================ +# Cortex sizing — usually small, but exposing for parity +# ============================================================================ + +variable "cortex_storage_size" { + description = "Cortex PVC size. 50Gi handles a year of OTLP-demo traffic; bump for higher cardinality fleets." + type = string + default = "50Gi" +} + +variable "cortex_storage_class" { + description = "EBS storage class for Cortex PVC." + type = string + default = "gp2" +} + +# ============================================================================ +# Data Prepper sizing — bump for sustained high-throughput trace ingest. The +# default 180s trace_flush_interval buffers all in-flight spans for traceGroup +# inference; at ~30K spans/sec sustained that's ~5M spans (~10 GB heap) at peak. +# ============================================================================ + +variable "data_prepper_memory" { + description = "Data Prepper container memory request/limit. Defaults to 1Gi (subchart default). Bump to 4Gi+ for sustained ingest workloads." + type = string + default = "1Gi" +} + +variable "data_prepper_jvm_heap" { + description = "Data Prepper JVM heap size. Should be ~75% of data_prepper_memory. Use the G/g suffix Java expects (e.g. '512m', '2g', '8g'). Empty string disables JAVA_OPTS override." + type = string + default = "" +} + +variable "data_prepper_trace_flush_interval" { + description = "Seconds the otel_traces processor buffers spans before computing traceGroup. Default 180s buffers ~5M spans at 30K/sec; lower to ~90 for ingest-time-bounded workloads at the cost of marking late-arriving traces incomplete." + type = number + default = 180 +} + # ============================================================================ # Derived # ============================================================================