Add OpenTelemetry distributed tracing (#60386)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

config/moda/configuration/default/env.yaml

@@ -9,3 +9,7 @@ data:
   RATE_LIMIT_MAX: '21'
   # Moda uses a non-default port for sending datadog metrics
   DD_DOGSTATSD_PORT: '28125'
+  # OTel distributed tracing — sends spans to OTel Collector via OTLP/HTTP (proto).
+  # Uses stamp address (not mesh) since docs-internal is not on the service mesh.
+  # See https://thehub.github.com/epd/engineering/dev-practicals/observability/distributed-tracing/instrumentation/
+  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'https://otelcol.service.%stamp%.github.net/v1/traces'

config/moda/configuration/production/env.yaml

@@ -12,3 +12,7 @@ data:
   # Identifies the service deployment environment as production
   # Equivalent to HEAVEN_DEPLOYED_ENV === 'production'
   MODA_PROD_SERVICE_ENV: 'true'
+  # OTel distributed tracing — sends spans to OTel Collector via OTLP/HTTP (proto).
+  # Uses stamp address (not mesh) since docs-internal is not on the service mesh.
+  # See https://thehub.github.com/epd/engineering/dev-practicals/observability/distributed-tracing/instrumentation/
+  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'https://otelcol.service.%stamp%.github.net/v1/traces'

config/moda/secrets/production/secrets.yml

@@ -64,3 +64,8 @@ secrets:
     type: hydro_api_key
     owner: '@github/docs-engineering'
     externally_usable: true
+  OTEL_EXPORTER_OTLP_TRACES_HEADERS:
+    kind: latest_at_deployment_start
+    type: salt
+    owner: '@github/docs-engineering'
+    externally_usable: true

next.config.ts

@@ -15,6 +15,15 @@ const config: NextConfig = {
   // Transpile @primer/react so Next's webpack can process its CSS and other assets
   // This ensures CSS in node_modules/@primer/react is handled by the app's loaders.
   transpilePackages: ['@primer/react'],
+  // Keep OTel packages out of the Next.js server bundle.
+  // They must be loaded via native require() for auto-instrumentation to work.
+  serverExternalPackages: [
+    '@opentelemetry/api',
+    '@opentelemetry/auto-instrumentations-node',
+    '@opentelemetry/core',
+    '@opentelemetry/exporter-trace-otlp-proto',
+    '@opentelemetry/sdk-node',
+  ],
   // speed up production `next build` by ignoring typechecking during that step of build.
   // type-checking still occurs in the Dockerfile build
   typescript: {
@@ -48,8 +57,15 @@ const config: NextConfig = {
     })
   },
 
-  webpack: (webpackConfig) => {
+  webpack: (webpackConfig, { isServer }) => {
     webpackConfig.resolve.fallback = { fs: false, async_hooks: false }
+    // OTel is server-only. Alias to empty stub in browser bundles.
+    if (!isServer) {
+      webpackConfig.resolve.alias = {
+        ...webpackConfig.resolve.alias,
+        '@/observability/lib/tracing': path.resolve('./src/observability/lib/tracing.browser.ts'),
+      }
+    }
     return webpackConfig
   },
 
@@ -71,6 +87,9 @@ const config: NextConfig = {
       '@/observability/logger/lib/logger-context': {
         browser: './empty.ts',
       },
+      '@/observability/lib/tracing': {
+        browser: './src/observability/lib/tracing.browser.ts',
+      },
     },
   },