diff --git a/.changeset/span-origin-provenance.md b/.changeset/span-origin-provenance.md new file mode 100644 index 000000000..0e177e564 --- /dev/null +++ b/.changeset/span-origin-provenance.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +Add span origin provenance metadata to Braintrust and OpenTelemetry spans. diff --git a/e2e/helpers/normalize.ts b/e2e/helpers/normalize.ts index 6b10658bd..e57330656 100644 --- a/e2e/helpers/normalize.ts +++ b/e2e/helpers/normalize.ts @@ -263,6 +263,24 @@ function normalizeObject( return []; } + if (key === "span_origin") { + return []; + } + + if (key === "braintrust.context_json" && typeof entry === "string") { + const normalizedContextJson = normalizeJsonString( + entry, + tokenMaps, + options, + ); + if (normalizedContextJson === "{}") { + return []; + } + if (normalizedContextJson) { + return [[key, normalizedContextJson]]; + } + } + if (isNodeInternalCaller) { if (key === "caller_filename") { return [[key, ""]]; @@ -283,6 +301,16 @@ function normalizeObject( return [[key, tokenFor(tokenMaps.ids, entry, "signature")]]; } + if ( + key === "environment" && + entry && + typeof entry === "object" && + !Array.isArray(entry) && + "type" in entry + ) { + return []; + } + return [[key, normalizeValue(entry as Json, tokenMaps, options, key)]]; }), ); @@ -341,6 +369,17 @@ function normalizeValue( } if (typeof value === "string") { + if (currentKey === "braintrust.context_json") { + const normalizedContextJson = normalizeJsonString( + value, + tokenMaps, + options, + ); + if (normalizedContextJson) { + return normalizedContextJson; + } + } + value = normalizeStackLikeString(value); const normalizedUrl = normalizeMockServerUrl(value); @@ -461,6 +500,20 @@ function normalizeValue( return value; } +function normalizeJsonString( + value: string, + tokenMaps: TokenMaps, + options: ResolvedNormalizeOptions, +): string | undefined { + try { + return JSON.stringify( + normalizeValue(JSON.parse(value) as Json, tokenMaps, options), + ); + } catch { + return undefined; + } +} + function resolveNormalizeOptions( options: NormalizeOptions | undefined, ): ResolvedNormalizeOptions { diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json index d649a4b4a..422cb001d 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json @@ -373,7 +373,11 @@ ] }, { - "context": {}, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + }, "created": "", "id": "", "input": [ diff --git a/integrations/otel-js/src/otel.test.ts b/integrations/otel-js/src/otel.test.ts index 5f8d543d9..320bd1a83 100644 --- a/integrations/otel-js/src/otel.test.ts +++ b/integrations/otel-js/src/otel.test.ts @@ -33,6 +33,7 @@ import { } from "../tests/utils"; import { SpanComponentsV3, SpanComponentsV4 } from "braintrust/util"; import { setupOtelCompat, resetOtelCompat } from "."; +import packageJson from "../package.json"; async function withEmptyBraintrustEnvFile( fn: () => T | Promise, @@ -178,6 +179,35 @@ describe("AISpanProcessor", () => { expect(spans.find((s) => s.attributes["foo"] === "bar")).toBeUndefined(); }); + it("should ignore Braintrust system attributes when filtering AI spans", async () => { + const rootSpan = tracer.startSpan("root"); + + const parentContext = trace.setSpanContext( + context.active(), + rootSpan.spanContext(), + ); + const systemAttrSpan = tracer.startSpan( + "some.operation", + { + attributes: { + "braintrust.context_json": JSON.stringify({ + span_origin: { name: "braintrust.sdk.javascript" }, + }), + "braintrust.parent": "project_name:test", + }, + }, + parentContext, + ); + + systemAttrSpan.end(); + rootSpan.end(); + + await provider.forceFlush(); + + const spans = memoryExporter.getFinishedSpans(); + expect(spans.find((s) => s.name === "some.operation")).toBeUndefined(); + }); + it("should support custom filter that keeps root spans using isRootSpan", () => { const customFilter = (span: ReadableSpan) => { return isRootSpan(span) ? true : undefined; @@ -754,6 +784,83 @@ describe("BraintrustSpanProcessor", () => { await expect(processor.forceFlush()).resolves.toBeUndefined(); }); + it("should merge span origin with context_json set after span start", () => { + let endedSpan: ReadableSpan | undefined; + const innerProcessor = { + onStart: vi.fn(), + onEnd: vi.fn((span: ReadableSpan) => { + endedSpan = span; + }), + shutdown: vi.fn(async () => undefined), + forceFlush: vi.fn(async () => undefined), + }; + + const processor = new BraintrustSpanProcessor({ + apiKey: "test-api-key", + _spanProcessor: innerProcessor as any, + }); + + const mockSpan = { + spanContext: () => ({ traceId: "test-trace", spanId: "test-span" }), + setAttributes: vi.fn(), + name: "late-context", + attributes: { + "braintrust.context_json": JSON.stringify({ + metadata: { source: "late-attribute" }, + }), + }, + parentSpanContext: undefined, + } as any; + + processor.onStart(mockSpan, context.active()); + processor.onEnd(mockSpan); + + expect(endedSpan).toBeDefined(); + const contextJson = endedSpan?.attributes["braintrust.context_json"]; + expect(typeof contextJson).toBe("string"); + const parsed = JSON.parse(contextJson as string); + expect(parsed.metadata.source).toBe("late-attribute"); + expect(parsed.span_origin.name).toBe("braintrust.sdk.javascript"); + expect(parsed.span_origin.version).toBe(packageJson.version); + expect(parsed.span_origin.instrumentation.name).toBe("braintrust-otel-js"); + }); + + it("should preserve explicit environment name without type", () => { + process.env.BRAINTRUST_ENVIRONMENT_NAME = "staging"; + delete process.env.BRAINTRUST_ENVIRONMENT_TYPE; + + let endedSpan: ReadableSpan | undefined; + const innerProcessor = { + onStart: vi.fn(), + onEnd: vi.fn((span: ReadableSpan) => { + endedSpan = span; + }), + shutdown: vi.fn(async () => undefined), + forceFlush: vi.fn(async () => undefined), + }; + + const processor = new BraintrustSpanProcessor({ + apiKey: "test-api-key", + _spanProcessor: innerProcessor as any, + }); + + const mockSpan = { + spanContext: () => ({ traceId: "test-trace", spanId: "test-span" }), + setAttributes: vi.fn(), + name: "environment-name-only", + attributes: {}, + parentSpanContext: undefined, + } as any; + + processor.onStart(mockSpan, context.active()); + processor.onEnd(mockSpan); + + const parsed = JSON.parse( + endedSpan?.attributes["braintrust.context_json"] as string, + ); + expect(parsed.span_origin.environment).toEqual({ name: "staging" }); + }); + it("should use default parent when none is provided", () => { process.env.BRAINTRUST_API_KEY = "test-api-key"; delete process.env.BRAINTRUST_PARENT; diff --git a/integrations/otel-js/src/otel.ts b/integrations/otel-js/src/otel.ts index 3a9d0f874..272e27398 100644 --- a/integrations/otel-js/src/otel.ts +++ b/integrations/otel-js/src/otel.ts @@ -24,6 +24,7 @@ import { registerOtelFlush, type Span as BraintrustSpan, } from "braintrust"; +import packageJson from "../package.json"; interface ExportResult { code: number; @@ -38,6 +39,11 @@ const FILTER_PREFIXES = [ "traceloop.", ] as const; +const SYSTEM_ATTRIBUTE_NAMES = new Set([ + "braintrust.parent", + "braintrust.context_json", +]); + /** * Custom filter function type for span filtering. * @param span - The span to evaluate @@ -72,7 +78,9 @@ function isAISpan(span: ReadableSpan): boolean { } const attributes = span.attributes; if (attributes) { - const attributeNames = Object.keys(attributes); + const attributeNames = Object.keys(attributes).filter( + (name) => !SYSTEM_ATTRIBUTE_NAMES.has(name), + ); if ( attributeNames.some((name) => FILTER_PREFIXES.some((prefix) => name.startsWith(prefix)), @@ -192,6 +200,7 @@ interface BraintrustSpanProcessorOptions { * Additional headers to send with telemetry data */ headers?: Record; + environment?: { type?: string; name?: string }; /** * @internal * Internal option for dependency injection during testing. @@ -200,6 +209,121 @@ interface BraintrustSpanProcessorOptions { _spanProcessor?: SpanProcessor; } +const SDK_VERSION = (packageJson as { version: string }).version; + +function spanOriginContext(environment?: { type?: string; name?: string }) { + return { + span_origin: { + name: "braintrust.sdk.javascript", + version: SDK_VERSION, + instrumentation: { name: "braintrust-otel-js" }, + ...(environment ? { environment } : {}), + }, + }; +} + +function mergeSpanOriginContextJson( + existingContextJson: unknown, + environment?: { type?: string; name?: string }, +): string { + const context = + typeof existingContextJson === "string" && existingContextJson + ? parseContextJson(existingContextJson) + : {}; + const existingOrigin = + context.span_origin && + typeof context.span_origin === "object" && + !Array.isArray(context.span_origin) + ? context.span_origin + : {}; + return JSON.stringify({ + ...context, + span_origin: { + ...spanOriginContext(environment).span_origin, + ...existingOrigin, + }, + }); +} + +function parseContextJson(contextJson: string): Record { + try { + const parsed = JSON.parse(contextJson); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? parsed + : {}; + } catch { + return {}; + } +} + +function withSpanOriginAttributes( + span: ReadableSpan, + environment?: { type?: string; name?: string }, +): ReadableSpan { + const attributes = { + ...span.attributes, + "braintrust.context_json": mergeSpanOriginContextJson( + span.attributes["braintrust.context_json"], + environment, + ), + }; + return new Proxy(span, { + get(target, prop, receiver) { + if (prop === "attributes") return attributes; + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +function detectEnvironment(explicit?: { + type?: string; + name?: string; +}): { type?: string; name?: string } | undefined { + if (explicit) return explicit; + const envType = process.env.BRAINTRUST_ENVIRONMENT_TYPE; + const envName = process.env.BRAINTRUST_ENVIRONMENT_NAME; + if (envType || envName) { + return { + ...(envType ? { type: envType } : {}), + ...(envName ? { name: envName } : {}), + }; + } + if (process.env.GITHUB_ACTIONS) return { type: "ci", name: "github_actions" }; + if (process.env.GITLAB_CI) return { type: "ci", name: "gitlab_ci" }; + if (process.env.CIRCLECI) return { type: "ci", name: "circleci" }; + if (process.env.BUILDKITE) return { type: "ci", name: "buildkite" }; + if (process.env.CI) return { type: "ci", name: "ci" }; + if (process.env.VERCEL) return { type: "server", name: "vercel" }; + if (process.env.NETLIFY) return { type: "server", name: "netlify" }; + if ( + process.env.ECS_CONTAINER_METADATA_URI || + process.env.ECS_CONTAINER_METADATA_URI_V4 || + process.env.AWS_EXECUTION_ENV?.startsWith("AWS_ECS_") + ) { + return { type: "server", name: "ecs" }; + } + if ( + process.env.AWS_LAMBDA_FUNCTION_NAME || + process.env.AWS_EXECUTION_ENV?.startsWith("AWS_Lambda_") + ) { + return { type: "server", name: "aws_lambda" }; + } + if ( + process.env.NODE_ENV === "production" || + process.env.NODE_ENV === "staging" + ) { + return { type: "server", name: process.env.NODE_ENV }; + } + if ( + process.env.NODE_ENV === "development" || + process.env.NODE_ENV === "local" + ) { + return { type: "local", name: process.env.NODE_ENV }; + } + return undefined; +} + class LazyBraintrustOTLPTraceExporter implements SpanExporter { private readonly diagLogger = diag.createComponentLogger({ namespace: "@braintrust/otel", @@ -377,8 +501,10 @@ export class BraintrustSpanProcessor implements SpanProcessor { private readonly processor: SpanProcessor; private readonly aiSpanProcessor: SpanProcessor; private readonly exporter?: LazyBraintrustOTLPTraceExporter; + private readonly environment?: { type?: string; name?: string }; constructor(options: BraintrustSpanProcessorOptions = {}) { + this.environment = detectEnvironment(options.environment); // If a processor is injected (for testing), use it directly if (options._spanProcessor) { this.processor = options._spanProcessor; @@ -524,7 +650,9 @@ export class BraintrustSpanProcessor implements SpanProcessor { } onEnd(span: ReadableSpan): void { - this.aiSpanProcessor.onEnd(span); + this.aiSpanProcessor.onEnd( + withSpanOriginAttributes(span, this.environment), + ); } shutdown(): Promise { diff --git a/integrations/otel-js/tsconfig.json b/integrations/otel-js/tsconfig.json index 8cfc6d9ae..ac8c3126e 100644 --- a/integrations/otel-js/tsconfig.json +++ b/integrations/otel-js/tsconfig.json @@ -5,6 +5,7 @@ "module": "commonjs", "target": "es2022", "moduleResolution": "node", + "resolveJsonModule": true, "strict": true, "noUnusedLocals": true, "esModuleInterop": true, diff --git a/js/src/logger.ts b/js/src/logger.ts index ab56bed90..905512568 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -180,6 +180,11 @@ import { lintTemplate as _lintMustacheTemplate } from "./template/mustache-utils import { prettifyXact } from "../util/index"; import { SpanCache, CachedSpan } from "./span-cache"; import type { EvalParameters, InferParameters } from "./eval-parameters"; +import { + detectSpanOriginEnvironment, + mergeSpanOriginContext, + type SpanOriginEnvironment, +} from "./span-origin"; // Manual type definition for inline attachments (not in generated_types) const InlineAttachmentReferenceSchema = z.object({ @@ -752,6 +757,7 @@ export class BraintrustState { private _idGenerator: IDGenerator | null = null; private _contextManager: ContextManager | null = null; private _otelFlushCallback: (() => Promise) | null = null; + public spanOriginEnvironment: SpanOriginEnvironment | undefined; constructor(private loginParams: LoginOptions) { this.id = `${new Date().toLocaleString()}-${stateNonce++}`; // This is for debugging. uuidv4() breaks on platforms like Cloudflare. @@ -813,6 +819,7 @@ export class BraintrustState { }); this.spanCache = new SpanCache({ disabled: loginParams.disableSpanCache }); + this.spanOriginEnvironment = detectSpanOriginEnvironment(); } public resetLoginInfo() { @@ -4610,6 +4617,7 @@ type AsyncFlushArg = { export type InitLoggerOptions = FullLoginOptions & { projectName?: string; projectId?: string; + environment?: SpanOriginEnvironment; setCurrent?: boolean; state?: BraintrustState; orgProjectMetadata?: OrgProjectMetadata; @@ -4643,6 +4651,7 @@ export function initLogger( orgName, forceLogin, debugLogLevel, + environment, fetch, state: stateArg, } = options || {}; @@ -4664,6 +4673,7 @@ export function initLogger( const state = stateArg ?? _globalState; state.setDebugLogLevel(debugLogLevel); + state.spanOriginEnvironment = detectSpanOriginEnvironment(environment); // Enable queue size limit enforcement for initLogger() calls // This ensures production observability doesn't OOM customer processes @@ -7620,7 +7630,11 @@ export class SpanImpl implements Span { metrics: { start: args.startTime ?? getCurrentUnixTimestamp(), }, - context: { ...callerLocation }, + context: mergeSpanOriginContext( + { ...callerLocation }, + "braintrust-js-logger", + this._state.spanOriginEnvironment, + ), span_attributes: { name, type, diff --git a/js/src/node/config.ts b/js/src/node/config.ts index 09c27b807..b7c8d98b6 100644 --- a/js/src/node/config.ts +++ b/js/src/node/config.ts @@ -31,7 +31,17 @@ export function configureNode() { iso.getPastNAncestors = getPastNAncestors; iso.getEnv = (name) => { const value = process.env[name]; - return name === "BRAINTRUST_API_KEY" && !value?.trim() ? undefined : value; + if (name === "BRAINTRUST_API_KEY") { + return value?.trim() ? value : undefined; + } + if ( + (name === "BRAINTRUST_ENVIRONMENT_TYPE" || + name === "BRAINTRUST_ENVIRONMENT_NAME") && + !value?.trim() + ) { + return getNearestBraintrustEnvValue(name); + } + return value; }; iso.getBraintrustApiKey = async () => { const value = process.env.BRAINTRUST_API_KEY; @@ -152,3 +162,31 @@ export function configureNode() { // Enable auto-instrumentation registry.enable(); } + +function getNearestBraintrustEnvValue(name: string): string | undefined { + for ( + let dir = process.cwd(), depth = 0; + depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; + dir = path.dirname(dir), depth++ + ) { + const envPath = path.join(dir, ".env.braintrust"); + try { + const parsed = dotenv.parse(fsSync.readFileSync(envPath, "utf8")); + const value = parsed[name]; + return value?.trim() ? value : undefined; + } catch (e) { + if ( + typeof e !== "object" || + e === null || + !("code" in e) || + e.code !== "ENOENT" + ) { + return undefined; + } + } + if (path.dirname(dir) === dir) { + break; + } + } + return undefined; +} diff --git a/js/src/span-origin.test.ts b/js/src/span-origin.test.ts new file mode 100644 index 000000000..b6ea0a391 --- /dev/null +++ b/js/src/span-origin.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; + +import packageJson from "../package.json"; +import { mergeSpanOriginContext } from "./span-origin"; + +describe("mergeSpanOriginContext", () => { + it("loads the SDK version from package.json", () => { + const context = mergeSpanOriginContext(undefined, "test-instrumentation"); + + expect(context.span_origin).toMatchObject({ + name: "braintrust.sdk.javascript", + version: packageJson.version, + instrumentation: { name: "test-instrumentation" }, + }); + }); +}); diff --git a/js/src/span-origin.ts b/js/src/span-origin.ts new file mode 100644 index 000000000..b2c45b66c --- /dev/null +++ b/js/src/span-origin.ts @@ -0,0 +1,130 @@ +import iso from "./isomorph"; +import packageJson from "../package.json"; + +export type SpanOriginEnvironment = { + type?: string; + name?: string; +}; + +type SpanOrigin = { + name: string; + version: string; + instrumentation: { name: string }; + environment?: SpanOriginEnvironment; +}; + +const SDK_VERSION = (packageJson as { version: string }).version; + +export function detectSpanOriginEnvironment( + explicit?: SpanOriginEnvironment, +): SpanOriginEnvironment | undefined { + if (explicit) return explicit; + + const envType = iso.getEnv("BRAINTRUST_ENVIRONMENT_TYPE"); + const envName = iso.getEnv("BRAINTRUST_ENVIRONMENT_NAME"); + if (envType || envName) { + return { + ...(envType ? { type: envType } : {}), + ...(envName ? { name: envName } : {}), + }; + } + + const ci = firstPresent([ + ["GITHUB_ACTIONS", "github_actions"], + ["GITLAB_CI", "gitlab_ci"], + ["CIRCLECI", "circleci"], + ["BUILDKITE", "buildkite"], + ["JENKINS_URL", "jenkins"], + ["JENKINS_HOME", "jenkins"], + ["TF_BUILD", "azure_pipelines"], + ["TEAMCITY_VERSION", "teamcity"], + ["TRAVIS", "travis"], + ["BITBUCKET_BUILD_NUMBER", "bitbucket"], + ]); + if (ci) return { type: "ci", name: ci }; + if (iso.getEnv("CI")) return { type: "ci", name: "ci" }; + + const earlyServer = firstPresent([ + ["VERCEL", "vercel"], + ["NETLIFY", "netlify"], + ]); + if (earlyServer) return { type: "server", name: earlyServer }; + if ( + iso.getEnv("ECS_CONTAINER_METADATA_URI") || + iso.getEnv("ECS_CONTAINER_METADATA_URI_V4") + ) { + return { type: "server", name: "ecs" }; + } + const awsExecutionEnv = iso.getEnv("AWS_EXECUTION_ENV"); + if (awsExecutionEnv?.startsWith("AWS_ECS_")) { + return { type: "server", name: "ecs" }; + } + if (awsExecutionEnv?.startsWith("AWS_Lambda_")) { + return { type: "server", name: "aws_lambda" }; + } + if (iso.getEnv("AWS_LAMBDA_FUNCTION_NAME")) { + return { type: "server", name: "aws_lambda" }; + } + + const server = firstPresent([ + ["K_SERVICE", "cloud_run"], + ["FUNCTION_TARGET", "gcp_functions"], + ["KUBERNETES_SERVICE_HOST", "kubernetes"], + ["DYNO", "heroku"], + ["FLY_APP_NAME", "fly"], + ["RAILWAY_ENVIRONMENT", "railway"], + ["RENDER_SERVICE_NAME", "render"], + ]); + if (server) return { type: "server", name: server }; + + return deploymentModeEnvironment("NODE_ENV", iso.getEnv("NODE_ENV")); +} + +function makeSpanOrigin( + instrumentationName: string, + environment?: SpanOriginEnvironment, +): SpanOrigin { + return { + name: "braintrust.sdk.javascript", + version: SDK_VERSION, + instrumentation: { name: instrumentationName }, + ...(environment ? { environment } : {}), + }; +} + +export function mergeSpanOriginContext( + context: Record | undefined, + instrumentationName: string, + environment?: SpanOriginEnvironment, +): Record { + const next = { ...(context ?? {}) }; + const current = isObject(next.span_origin) ? { ...next.span_origin } : {}; + next.span_origin = { + ...makeSpanOrigin(instrumentationName, environment), + ...current, + }; + return next; +} + +function firstPresent(entries: Array<[string, string]>): string | undefined { + return entries.find(([key]) => Boolean(iso.getEnv(key)))?.[1]; +} + +function deploymentModeEnvironment( + _key: string, + value: string | undefined, +): SpanOriginEnvironment | undefined { + if (!value) return undefined; + const normalized = value.toLowerCase(); + if (normalized === "production" || normalized === "staging") { + return { type: "server", name: normalized }; + } + if (normalized === "development" || normalized === "local") { + return { type: "local", name: normalized }; + } + return undefined; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/js/tsconfig.json b/js/tsconfig.json index de86073ac..6509d4c1f 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -5,6 +5,7 @@ "module": "commonjs", "target": "es2022", "moduleResolution": "node", + "resolveJsonModule": true, "baseUrl": ".", "paths": { "@apm-js-collab/code-transformer": [