From b7aa981fe200e3bf117bf17713930069f486405b Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Thu, 27 Aug 2026 12:44:25 +0200 Subject: [PATCH] feat(server-utils): Emit low-cardinality gen_ai inference span names when streaming When span streaming is on, inference spans use `{operation} {model}`, or `{operation}` when the model is missing or empty. `gen_ai.execute_tool` drops the tool name from the span name, which stays on `gen_ai.tool.name`. Static lifecycle names are unchanged. Every instrumented method supplies an operation, so the convention fallback `Generative AI model operation` is unreachable and is not wired up at any site. --- MIGRATION.md | 6 +- .../server-utils/src/ai/anthropic-ai/index.ts | 67 +++++++------ .../server-utils/src/ai/google-genai/index.ts | 16 +++- .../src/ai/langchain/embeddings.ts | 16 +++- .../server-utils/src/ai/langchain/index.ts | 26 +++-- packages/server-utils/src/ai/openai/index.ts | 9 +- .../server-utils/src/ai/workers-ai/index.ts | 11 ++- .../src/integrations/anthropic.ts | 6 +- .../src/integrations/google-genai.ts | 6 +- .../server-utils/src/integrations/openai.ts | 6 +- .../test/ai/lib/tracing/anthropic-ai.test.ts | 91 ++++++++++++++++++ .../test/ai/lib/tracing/google-genai.test.ts | 95 +++++++++++++++++++ .../ai/lib/tracing/langchain-chat.test.ts | 65 +++++++++++++ .../lib/tracing/langchain-embeddings.test.ts | 50 +++++++++- .../test/ai/lib/tracing/openai.test.ts | 93 ++++++++++++++++++ .../test/ai/lib/tracing/workers-ai.test.ts | 58 +++++++++++ .../vercel-ai/inference-span-names.test.ts | 52 ++++++++++ 17 files changed, 617 insertions(+), 56 deletions(-) create mode 100644 packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/google-genai.test.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/langchain-chat.test.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/openai.test.ts create mode 100644 packages/server-utils/test/integrations/vercel-ai/inference-span-names.test.ts diff --git a/MIGRATION.md b/MIGRATION.md index 7ddc981d29af..430f30f9a138 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -823,7 +823,7 @@ The LangGraph instrumentation no longer emits `gen_ai.create_agent` spans when a Affected SDKs: All SDKs. -With [span streaming](#span-streaming-is-now-the-default) enabled(the default), span names are now **low cardinality**, following the [Sentry span name conventions](https://getsentry.github.io/sentry-conventions/names/). +With [span streaming](#span-streaming-is-now-the-default) enabled (the default), span names are now **low cardinality**, following the [Sentry span name conventions](https://getsentry.github.io/sentry-conventions/names/). If you [opt out of span streaming](#opting-out-of-span-streaming), span names remain unchanged. @@ -836,12 +836,15 @@ The following span names were adjusted: | `http.server` | The request method and route, or the raw URL path if the SDK couldn't resolve one (`GET /users/123`) | `GET /users/:id` when a route is known, otherwise just the request method (`GET`) | | `router` | Framework-specific, sometimes containing the raw URL (`/users/123`, `SvelteKit Route Change`) | The span's `http.route`, or `Router` if the SDK has none | | `graphql` | The graphql phase and, for operations, the operation name (`query GetUser`, `graphql.parse`, `graphql.resolve user.0.name`) | The operation type, or the processing type where there is none (`GraphQL query`, `GraphQL parse`, `GraphQL resolve`) | +| `gen_ai.chat`, `gen_ai.embeddings`, `gen_ai.generate_content` | `{operation} {model}`, or `{operation} unknown` if the model is missing (`chat unknown`) | `{operation} {model}`, or `{operation}` if the model is missing (`chat`) | | `resource.*` | The resource URL, relative to the page origin for same-origin resources (`/assets/app.js`) | The resource domain (`cdn.example.com`), or `Resource` if the SDK has none | | `mcp.server` | The method and its target, including the resource URI (`resources/read file:///docs/api.md`) | The method alone for resource methods (`resources/read`). Tool and prompt names are unchanged (`tools/call get-weather`) | | `mcp.notification.client_to_server`, `mcp.notification.server_to_client` | The notification method name (`notifications/tools/list_changed`) | The notification method name, or `MCP notification` if the message carries none | `navigation.redirect` spans are started through the same code path as navigation spans, so they get the same names. +Resolved low-cardinality values are kept in both lifecycles: a known model stays in the name (`chat gpt-4`). + Resource spans now also carry a `url.domain` attribute holding that domain. The full URL remains available on `url.full`. `http.server` requests that resolve to a route are **unchanged** — those names were already low cardinality. Only requests the SDK cannot parameterize are affected. @@ -863,6 +866,7 @@ Child spans of a service or root span carry its name in their `sentry.segment.na `ignoreSpans` is evaluated when a span **starts**, at which point a span might not yet have its final name. For example, an unresolved pageload or navigation span is named `'Pageload'`/`'Navigation'` and might receive its final, resolved route name later. `ignoreSpans` filters matching a URL path no longer apply to them. Another example where filters might need adjustments are `resource.*` spans where their name now only includes the domain the resource was taken from. +Likewise, filters matching `chat unknown` no longer apply to a streamed chat span (`'chat'`). Match on attributes instead: diff --git a/packages/server-utils/src/ai/anthropic-ai/index.ts b/packages/server-utils/src/ai/anthropic-ai/index.ts index 30802331fdb2..a4a09e334043 100644 --- a/packages/server-utils/src/ai/anthropic-ai/index.ts +++ b/packages/server-utils/src/ai/anthropic-ai/index.ts @@ -1,5 +1,12 @@ /* eslint-disable typescript-eslint/no-deprecated */ -import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, SPAN_STATUS_ERROR, startSpan, startSpanManual } from '@sentry/core'; +import { + getClient, + hasSpanStreamingEnabled, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SPAN_STATUS_ERROR, + startSpan, + startSpanManual, +} from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_OPERATION_NAME, @@ -170,21 +177,12 @@ function handleStreamingRequest( target: (...args: T) => R | Promise, invocationThis: unknown, args: T, - requestAttributes: Record, - operationName: string, - methodPath: string, + spanConfig: { name: string; op: string; attributes: Record }, params: Record | undefined, options: AnthropicAiOptions, isStreamRequested: boolean, isStreamingMethod: boolean, ): R | Promise { - const model = requestAttributes[GEN_AI_REQUEST_MODEL] ?? 'unknown'; - const spanConfig = { - name: `${operationName} ${model}`, - op: getGenAiSpanOp(operationName), - attributes: requestAttributes as Record, - }; - // messages.stream() always returns a sync MessageStream, even with stream: true param if (isStreamRequested && !isStreamingMethod) { let originalResult!: Promise; @@ -262,7 +260,17 @@ function instrumentMethod( const operationName = instrumentedMethod.operation || 'unknown'; const requestAttributes = extractRequestAttributes(args, operationName); - const model = requestAttributes[GEN_AI_REQUEST_MODEL] ?? 'unknown'; + const model = requestAttributes[GEN_AI_REQUEST_MODEL] || 'unknown'; + const client = getClient(); + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + const spanConfig = { + name: + (typeof model === 'string' && model !== 'unknown') || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${model}` + : operationName, + op: getGenAiSpanOp(operationName), + attributes: requestAttributes as Record, + }; const params = typeof args[0] === 'object' ? (args[0] as Record) : undefined; const isStreamRequested = Boolean(params?.stream); @@ -272,9 +280,7 @@ function instrumentMethod( target, invocationThis, args, - requestAttributes, - operationName, - methodPath, + spanConfig, params, options, isStreamRequested, @@ -284,25 +290,18 @@ function instrumentMethod( let originalResult!: Promise; - const instrumentedPromise = startSpan( - { - name: `${operationName} ${model}`, - op: getGenAiSpanOp(operationName), - attributes: requestAttributes as Record, - }, - span => { - originalResult = target.apply(invocationThis, args) as Promise; - - if (options.recordInputs && params) { - addPrivateRequestAttributes(span, params); - } - - return originalResult.then(result => { - addResponseAttributes(span, result as AnthropicAiResponse, options.recordOutputs); - return result; - }); - }, - ); + const instrumentedPromise = startSpan(spanConfig, span => { + originalResult = target.apply(invocationThis, args) as Promise; + + if (options.recordInputs && params) { + addPrivateRequestAttributes(span, params); + } + + return originalResult.then(result => { + addResponseAttributes(span, result as AnthropicAiResponse, options.recordOutputs); + return result; + }); + }); return wrapPromiseWithMethods(originalResult, instrumentedPromise); }, diff --git a/packages/server-utils/src/ai/google-genai/index.ts b/packages/server-utils/src/ai/google-genai/index.ts index 7342d3a043bf..29e4d619c635 100644 --- a/packages/server-utils/src/ai/google-genai/index.ts +++ b/packages/server-utils/src/ai/google-genai/index.ts @@ -1,11 +1,13 @@ /* eslint-disable typescript-eslint/no-deprecated */ /* eslint-disable max-lines */ import { + getClient, + handleCallbackErrors, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, SPAN_STATUS_ERROR, startSpan, startSpanManual, - handleCallbackErrors, stringify, } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; @@ -268,14 +270,20 @@ function instrumentMethod( const operationName = instrumentedMethod.operation || 'unknown'; const params = args[0] as Record | undefined; const requestAttributes = extractRequestAttributes(operationName, params, context); - const model = requestAttributes[GEN_AI_REQUEST_MODEL] ?? 'unknown'; + const model = requestAttributes[GEN_AI_REQUEST_MODEL] || 'unknown'; + const client = getClient(); + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + const spanName = + (typeof model === 'string' && model !== 'unknown') || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${model}` + : operationName; // Check if this is a streaming method if (instrumentedMethod.streaming) { // Use startSpanManual for streaming methods to control span lifecycle return startSpanManual( { - name: `${operationName} ${model}`, + name: spanName, op: getGenAiSpanOp(operationName), attributes: requestAttributes, }, @@ -297,7 +305,7 @@ function instrumentMethod( // Single span for both sync and async operations return startSpan( { - name: `${operationName} ${model}`, + name: spanName, op: getGenAiSpanOp(operationName), attributes: requestAttributes, }, diff --git a/packages/server-utils/src/ai/langchain/embeddings.ts b/packages/server-utils/src/ai/langchain/embeddings.ts index cf052e76a684..28229082f0b1 100644 --- a/packages/server-utils/src/ai/langchain/embeddings.ts +++ b/packages/server-utils/src/ai/langchain/embeddings.ts @@ -1,4 +1,11 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, startSpan, stringify } from '@sentry/core'; +import { + getClient, + hasSpanStreamingEnabled, + SEMANTIC_ATTRIBUTE_SENTRY_OP, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + startSpan, + stringify, +} from '@sentry/core'; import type { SpanAttributeValue } from '@sentry/core'; import { GEN_AI_EMBEDDINGS_INPUT, @@ -69,13 +76,18 @@ export function _INTERNAL_getLangChainEmbeddingsSpanOptions( const { recordInputs } = resolveAIRecordingOptions(options); const attributes = extractEmbeddingAttributes(instance); const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; + const client = getClient(); if (recordInputs && input != null) { attributes[GEN_AI_EMBEDDINGS_INPUT] = stringify(input, String); } return { - name: `embeddings ${modelName}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: + (typeof modelName === 'string' && modelName !== 'unknown') || !(client && hasSpanStreamingEnabled(client)) + ? `embeddings ${modelName}` + : 'embeddings', op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, attributes: attributes as Record, }; diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index f7d0de9d4b61..02ad4fa14293 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -1,5 +1,7 @@ /* eslint-disable max-lines */ import { + getClient, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, SPAN_STATUS_ERROR, @@ -99,12 +101,18 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): invocationParams, metadata, ); - const modelName = attributes[GEN_AI_REQUEST_MODEL]; - const operationName = attributes[GEN_AI_OPERATION_NAME]; + const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; + const operationName = + typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; + const client = getClient(); startSpanManual( { - name: `${operationName} ${modelName}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: + (typeof modelName === 'string' && modelName !== 'unknown') || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${modelName}` + : operationName, op: 'gen_ai.chat', attributes: { ...getAgentNameFromMetadata(metadata), @@ -144,12 +152,18 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): attributes[GEN_AI_TOOL_DEFINITIONS] = toolDefsJson; } - const modelName = attributes[GEN_AI_REQUEST_MODEL]; - const operationName = attributes[GEN_AI_OPERATION_NAME]; + const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; + const operationName = + typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; + const client = getClient(); startSpanManual( { - name: `${operationName} ${modelName}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: + (typeof modelName === 'string' && modelName !== 'unknown') || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${modelName}` + : operationName, op: 'gen_ai.chat', attributes: { ...getAgentNameFromMetadata(metadata), diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 83d9aceb9541..701456f5f2af 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -1,6 +1,8 @@ /* eslint-disable typescript-eslint/no-deprecated */ import { DEBUG_BUILD } from '../../debug-build'; import { + getClient, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, SPAN_STATUS_ERROR, startSpan, @@ -143,9 +145,14 @@ function instrumentMethod( const params = args[0] as Record | undefined; const isStreamRequested = params && typeof params === 'object' && params.stream === true; + const client = getClient(); const spanConfig = { - name: `${operationName} ${model}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: + model !== 'unknown' || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${model}` + : operationName, op: getGenAiSpanOp(operationName), attributes: requestAttributes as Record, }; diff --git a/packages/server-utils/src/ai/workers-ai/index.ts b/packages/server-utils/src/ai/workers-ai/index.ts index cf5b0a467484..b9d43010093b 100644 --- a/packages/server-utils/src/ai/workers-ai/index.ts +++ b/packages/server-utils/src/ai/workers-ai/index.ts @@ -1,5 +1,7 @@ import { _INTERNAL_shouldSkipAiProviderWrapping, + getClient, + hasSpanStreamingEnabled, isObjectLike, SPAN_STATUS_ERROR, startSpan, @@ -42,7 +44,8 @@ function instrumentRun( const operationName = getOperationName(inputs); const requestAttributes = extractRequestAttributes(model, inputs, operationName); - const modelName = typeof model === 'string' ? model : 'unknown'; + const modelName = typeof model === 'string' && model ? model : 'unknown'; + const client = getClient(); const isStreamRequested = !!inputs && typeof inputs === 'object' && (inputs as { stream?: unknown }).stream === true; @@ -52,7 +55,11 @@ function instrumentRun( (runOptions.returnRawResponse === true || runOptions.websocket === true); const spanConfig = { - name: `${operationName} ${modelName}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: + modelName !== 'unknown' || !(client && hasSpanStreamingEnabled(client)) + ? `${operationName} ${modelName}` + : operationName, op: `gen_ai.${operationName}`, attributes: requestAttributes, }; diff --git a/packages/server-utils/src/integrations/anthropic.ts b/packages/server-utils/src/integrations/anthropic.ts index 1b6bd0c51aed..b54223925494 100644 --- a/packages/server-utils/src/integrations/anthropic.ts +++ b/packages/server-utils/src/integrations/anthropic.ts @@ -4,6 +4,8 @@ import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { _INTERNAL_shouldSkipAiProviderWrapping, defineIntegration, + getClient, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, startInactiveSpan, } from '@sentry/core'; @@ -99,9 +101,11 @@ function createGenAiSpan( const attributes = extractRequestAttributes(args, operation); const model = (attributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] = ORIGIN; + const client = getClient(); const span = startInactiveSpan({ - name: `${operation} ${model}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: model !== 'unknown' || !(client && hasSpanStreamingEnabled(client)) ? `${operation} ${model}` : operation, op: getGenAiSpanOp(operation), attributes: attributes as Record, }); diff --git a/packages/server-utils/src/integrations/google-genai.ts b/packages/server-utils/src/integrations/google-genai.ts index 9ae699fbb001..33675b0941a8 100644 --- a/packages/server-utils/src/integrations/google-genai.ts +++ b/packages/server-utils/src/integrations/google-genai.ts @@ -5,6 +5,8 @@ import { _INTERNAL_shouldSkipAiProviderWrapping, defineIntegration, getActiveSpan, + getClient, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, spanToJSON, startInactiveSpan, @@ -108,9 +110,11 @@ function createGenAiSpan( const attributes = extractRequestAttributes(operation, params, data.self); const model = (attributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] = ORIGIN; + const client = getClient(); const span = startInactiveSpan({ - name: `${operation} ${model}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: model !== 'unknown' || !(client && hasSpanStreamingEnabled(client)) ? `${operation} ${model}` : operation, op: getGenAiSpanOp(operation), attributes, }); diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index 6c30c612ec76..6e900596aef4 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -3,6 +3,8 @@ import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { _INTERNAL_shouldSkipAiProviderWrapping, defineIntegration, + getClient, + hasSpanStreamingEnabled, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, startInactiveSpan, } from '@sentry/core'; @@ -82,9 +84,11 @@ function createGenAiSpan(data: OpenAiChatChannelContext, operation: string, opti const attributes = extractRequestAttributes(args, operation); attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] = ORIGIN; const model = (params?.model as string) || 'unknown'; + const client = getClient(); const span = startInactiveSpan({ - name: `${operation} ${model}`, + // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. + name: model !== 'unknown' || !(client && hasSpanStreamingEnabled(client)) ? `${operation} ${model}` : operation, op: getGenAiSpanOp(operation), attributes: attributes as Record, }); diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts new file mode 100644 index 000000000000..1a08d660ab7d --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('instrumentAnthropicAiClient span names', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + function fakeClient(): { + messages: { create: ReturnType }; + } { + return { + messages: { + create: vi.fn().mockResolvedValue({ id: 'msg', content: [] }), + }, + }; + } + + it('names the span `{operation} {model}` when a model is present', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentAnthropicAiClient(client); + + await instrumented.messages.create({ + model: 'claude-3-haiku-20240307', + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat claude-3-haiku-20240307'); + }); + + it('keeps `chat unknown` when the model is missing in static mode', async () => { + const endedSpans = setupClient('static'); + const client = fakeClient(); + const instrumented = instrumentAnthropicAiClient(client); + + await instrumented.messages.create({ + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat unknown'); + }); + + it('uses the operation name when the model is missing and span streaming is enabled', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentAnthropicAiClient(client); + + await instrumented.messages.create({ + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); + + it('treats an empty-string model as missing', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentAnthropicAiClient(client); + + await instrumented.messages.create({ + model: '', + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts new file mode 100644 index 000000000000..247228990ad3 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts @@ -0,0 +1,95 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { instrumentGoogleGenAIClient } from '../../../../src/ai/google-genai'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('instrumentGoogleGenAIClient span names', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + function fakeClient(): { + models: { generateContent: ReturnType }; + chats: { create: ReturnType }; + } { + return { + models: { + generateContent: vi.fn().mockResolvedValue({ candidates: [] }), + }, + chats: { + create: vi.fn().mockReturnValue({ sendMessage: vi.fn() }), + }, + }; + } + + it('names the span `{operation} {model}` when a model is present', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentGoogleGenAIClient(client); + + await instrumented.models.generateContent({ model: 'gemini-1.5-pro', contents: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('generate_content gemini-1.5-pro'); + }); + + it('keeps `generate_content unknown` when the model is missing in static mode', async () => { + const endedSpans = setupClient('static'); + const client = fakeClient(); + const instrumented = instrumentGoogleGenAIClient(client); + + await instrumented.models.generateContent({ contents: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('generate_content unknown'); + }); + + it('uses the operation name when the model is missing and span streaming is enabled', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentGoogleGenAIClient(client); + + await instrumented.models.generateContent({ contents: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('generate_content'); + }); + + it('treats an empty-string model as missing', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentGoogleGenAIClient(client); + + await instrumented.models.generateContent({ model: '', contents: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('generate_content'); + }); + + it('does not start a span for chats.create', () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentGoogleGenAIClient(client); + + instrumented.chats.create({ model: 'gemini-1.5-pro' }); + + expect(endedSpans).toHaveLength(0); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-chat.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-chat.test.ts new file mode 100644 index 000000000000..925e0f7e9cba --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/langchain-chat.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('LangChain chat span names', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + function runChatModel(model?: string): void { + const handler = createLangChainCallbackHandler(); + handler.handleChatModelStart?.( + { id: ['langchain', 'chat_models', 'ChatOpenAI'] }, + [[{ content: 'Hello', _getType: () => 'human' }]], + 'run-1', + undefined, + undefined, + (model === undefined ? {} : { invocation_params: { model } }) as unknown as string[], + ); + handler.handleLLMEnd?.({ generations: [] }, 'run-1'); + } + + it('names the span `{operation} {model}` when a model is present', () => { + const endedSpans = setupClient('stream'); + runChatModel('gpt-4'); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat gpt-4'); + }); + + it('uses the operation name when the model is missing and span streaming is enabled', () => { + const endedSpans = setupClient('stream'); + runChatModel(); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); + + it('treats an empty-string model as missing', () => { + const endedSpans = setupClient('stream'); + runChatModel(''); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-embeddings.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-embeddings.test.ts index bf3ce75bfbf8..5ce910d1c68a 100644 --- a/packages/server-utils/test/ai/lib/tracing/langchain-embeddings.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langchain-embeddings.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type * as AiCoreUtils from '../../../../src/ai/core/utils'; import type * as SentryCore from '@sentry/core'; import { @@ -42,11 +42,29 @@ vi.mock('@sentry/core', async importOriginal => { }; }); -import { captureException } from '@sentry/core'; +import { captureException, getMainCarrier, setCurrentClient } from '@sentry/core'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +function setupClient(traceLifecycle: 'static' | 'stream'): void { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); +} describe('instrumentEmbeddingMethod', () => { beforeEach(() => { capturedSpanConfig = undefined; + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; }); it('creates a span with correct attributes', async () => { @@ -103,7 +121,8 @@ describe('instrumentEmbeddingMethod', () => { expect(capturedSpanConfig!.attributes[GEN_AI_PROVIDER_NAME]).toBe('google_genai'); }); - it('handles missing instance properties gracefully', async () => { + it('keeps `embeddings unknown` when the model is missing in static mode', async () => { + setupClient('static'); const original = vi.fn().mockResolvedValue([0.1]); const wrapped = instrumentEmbeddingMethod(original); @@ -114,11 +133,36 @@ describe('instrumentEmbeddingMethod', () => { expect(capturedSpanConfig!.attributes[GEN_AI_PROVIDER_NAME]).toBe('langchain'); expect(capturedSpanConfig!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toBeUndefined(); }); + + it('uses the operation name when the model is missing and span streaming is enabled', async () => { + setupClient('stream'); + const original = vi.fn().mockResolvedValue([0.1]); + const wrapped = instrumentEmbeddingMethod(original); + + await wrapped.call({}, 'test'); + + expect(capturedSpanConfig!.name).toBe('embeddings'); + }); + + it('uses the operation name when the model is not a string and span streaming is enabled', async () => { + setupClient('stream'); + const original = vi.fn().mockResolvedValue([0.1]); + const wrapped = instrumentEmbeddingMethod(original); + + await wrapped.call({ model: { id: 'text-embedding-3-small' } }, 'test'); + + expect(capturedSpanConfig!.name).toBe('embeddings'); + }); }); describe('instrumentLangChainEmbeddings', () => { beforeEach(() => { capturedSpanConfig = undefined; + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; }); it('wraps both embedQuery and embedDocuments on an instance', async () => { diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts new file mode 100644 index 000000000000..0c6738ce86ac --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { instrumentOpenAiClient } from '../../../../src/ai/openai'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('instrumentOpenAiClient span names', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + function fakeClient(): { + chat: { completions: { create: ReturnType } }; + } { + return { + chat: { + completions: { + create: vi.fn().mockResolvedValue({ id: 'test', choices: [] }), + }, + }, + }; + } + + it('names the span `{operation} {model}` when a model is present', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentOpenAiClient(client); + + await instrumented.chat.completions.create({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat gpt-4'); + }); + + it('keeps `chat unknown` when the model is missing in static mode', async () => { + const endedSpans = setupClient('static'); + const client = fakeClient(); + const instrumented = instrumentOpenAiClient(client); + + await instrumented.chat.completions.create({ + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat unknown'); + }); + + it('uses the operation name when the model is missing and span streaming is enabled', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentOpenAiClient(client); + + await instrumented.chat.completions.create({ + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); + + it('treats an empty-string model as missing', async () => { + const endedSpans = setupClient('stream'); + const client = fakeClient(); + const instrumented = instrumentOpenAiClient(client); + + await instrumented.chat.completions.create({ + model: '', + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index 9405fb0d8672..4baced963ec2 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -159,4 +159,62 @@ describe('instrumentWorkersAiClient', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).data).toEqual(expected); }); }); + + describe('span names', () => { + function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + it('names the span `{operation} {model}` when a model is present', async () => { + const endedSpans = setupClient('stream'); + const client = { run: vi.fn().mockResolvedValue({ response: 'ok' }) }; + const instrumented = instrumentWorkersAiClient(client); + + await instrumented.run(MODEL, { prompt: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe(`chat ${MODEL}`); + }); + + it('keeps `chat unknown` when the model is missing in static mode', async () => { + const endedSpans = setupClient('static'); + const client = { run: vi.fn().mockResolvedValue({ response: 'ok' }) }; + const instrumented = instrumentWorkersAiClient(client); + + await instrumented.run({ not: 'a-string' }, { prompt: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat unknown'); + }); + + it('treats an empty-string model as missing', async () => { + const endedSpans = setupClient('stream'); + const client = { run: vi.fn().mockResolvedValue({ response: 'ok' }) }; + const instrumented = instrumentWorkersAiClient(client); + + await instrumented.run('', { prompt: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); + + it('falls back to the operation name when the model is missing and span streaming is enabled', async () => { + const endedSpans = setupClient('stream'); + const client = { run: vi.fn().mockResolvedValue({ response: 'ok' }) }; + const instrumented = instrumentWorkersAiClient(client); + + await instrumented.run({ not: 'a-string' }, { prompt: 'Hello' }); + + expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); + }); + }); }); diff --git a/packages/server-utils/test/integrations/vercel-ai/inference-span-names.test.ts b/packages/server-utils/test/integrations/vercel-ai/inference-span-names.test.ts new file mode 100644 index 000000000000..8d2d5cc97420 --- /dev/null +++ b/packages/server-utils/test/integrations/vercel-ai/inference-span-names.test.ts @@ -0,0 +1,52 @@ +import { setCurrentClient, spanToJSON } from '@sentry/core'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { createSpanFromMessage } from '../../../src/integrations/vercel-ai/vercel-ai-dc-subscriber'; +import { getDefaultTestClientOptions, TestClient } from '../../mocks/client'; + +function setupClient(traceLifecycle: 'static' | 'stream'): void { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle, + }), + ); + setCurrentClient(client); + client.init(); +} + +function spanNameFor(type: string, event: Record): string | undefined { + const span = createSpanFromMessage( + { type, event } as Parameters[0], + {} as Parameters[1], + ); + span?.end(); + + return span && spanToJSON(span).name; +} + +// The channel path never emitted an `unknown` model sentinel, so these names already follow the +// inference templates in both lifecycles. +describe.each(['static', 'stream'] as const)('vercel ai inference span names (%s)', traceLifecycle => { + beforeEach(() => { + setupClient(traceLifecycle); + }); + + it.each([ + ['languageModelCall', 'generate_content'], + ['embed', 'embeddings'], + ['embedMany', 'embeddings'], + ['rerank', 'rerank'], + ])('names a %s span `%s {model}` when a model is present', (type, operation) => { + expect(spanNameFor(type, { modelId: 'gpt-4' })).toBe(`${operation} gpt-4`); + }); + + it.each([ + ['languageModelCall', 'generate_content'], + ['embed', 'embeddings'], + ['embedMany', 'embeddings'], + ['rerank', 'rerank'], + ])('names a %s span `%s` when the model is missing', (type, operation) => { + expect(spanNameFor(type, {})).toBe(operation); + }); +});