From d2612c4c3e25383e7c98f3d658044f72c129f3dd Mon Sep 17 00:00:00 2001 From: zkasuran <289388318+zkasuran@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:22:00 +0530 Subject: [PATCH 1/4] fix(core): Include Gemini reasoning tokens in Vercel AI token usage Gemini reports reasoning ("thoughts") tokens separately from the candidate output count, so the AI SDK's `outputTokens` covers only the visible answer and the count reaches us only through `providerMetadata.google.usageMetadata`. A span built from `ai.usage.*` alone undercounts output, and the total with it. Output is recomputed as `candidatesTokenCount + thoughtsTokenCount` rather than added onto the existing value, so it stays correct if a future SDK version folds reasoning in itself. `candidatesTokenCount` is optional, so output and total are written together or not at all: a thoughts-inclusive total beside a candidate-only output would describe a span whose parts do not add up. An `invoke_agent` span carries the summed usage of every step while `providerMetadata` describes the last step alone, so writing output or total from it would replace the aggregate with one step's figures. Both writes are skipped there. The reasoning count is not an aggregate and nothing else carries it, so it is still recorded. Co-Authored-By: Claude Opus 5 (1M context) --- packages/core/src/tracing/vercel-ai/index.ts | 29 +++ .../tracing/vercel-ai/vercel-ai-attributes.ts | 14 ++ .../vercel-ai-reasoning-tokens.test.ts | 177 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index f9abcb096dc8..dd219567faca 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -601,6 +601,23 @@ export function getProviderMetadataAttributes(providerMetadata: unknown): Record setAttributeIfDefined(attributes, 'gen_ai.usage.input_tokens.cache_miss', metadata.deepseek.promptCacheMissTokens); } + // Google (v5 uses 'google', v6 Vertex AI uses 'vertex'). Gemini reports its reasoning ("thoughts") + // tokens separately from the candidate output count, so the SDK's `outputTokens` covers only the + // visible answer. Recompute output from `candidatesTokenCount + thoughtsTokenCount` rather than + // adding onto the existing value, which stays correct even if a future SDK version already folds + // reasoning in. `candidatesTokenCount` is optional, so output and total are written together or + // not at all: taking the thoughts-inclusive total beside a candidate-only output would report a + // span whose parts do not add up. + const googleUsage = (metadata.google ?? metadata.vertex)?.usageMetadata; + if (googleUsage && typeof googleUsage.thoughtsTokenCount === 'number' && googleUsage.thoughtsTokenCount > 0) { + setAttributeIfDefined(attributes, 'gen_ai.usage.reasoning.output_tokens', googleUsage.thoughtsTokenCount); + if (typeof googleUsage.candidatesTokenCount === 'number') { + attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] = + googleUsage.candidatesTokenCount + googleUsage.thoughtsTokenCount; + setAttributeIfDefined(attributes, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, googleUsage.totalTokenCount); + } + } + return attributes; } @@ -609,6 +626,12 @@ function addProviderMetadataToAttributes(attributes: Record): v if (!providerMetadata) { return; } + // An `invoke_agent` span carries the summed `ai.usage.*` of every step, while `providerMetadata` + // describes the last step alone. Writing output or total from it would replace the aggregate with + // one step's figures, so a two-step call reports output 50 against input 900. The event-processor + // path happens to overwrite it again in `applyAccumulatedTokens`; the streamed path ships it. The + // reasoning count is still worth having, since nothing else carries it and it is not a total. + const lastStepOnly = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE] === 'invoke_agent'; try { const derived = getProviderMetadataAttributes(JSON.parse(providerMetadata) as ProviderMetadata); for (const [key, value] of Object.entries(derived)) { @@ -616,6 +639,12 @@ function addProviderMetadataToAttributes(attributes: Record): v if (key === GEN_AI_CONVERSATION_ID_ATTRIBUTE && attributes[key]) { continue; } + if ( + lastStepOnly && + (key === GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE || key === GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE) + ) { + continue; + } attributes[key] = value; } } catch { diff --git a/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts b/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts index 62d89f50c17c..63547b1f058c 100644 --- a/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts +++ b/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts @@ -446,6 +446,20 @@ export interface GoogleGenerativeAIProviderMetadata { * @see https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters */ safetyRatings?: null | unknown; + + /** + * Raw token usage returned by the Gemini API. Reasoning ("thoughts") tokens are reported here in + * `thoughtsTokenCount`, separately from the candidate output count, so they have to be added back + * into `gen_ai.usage.output_tokens`. + * @see https://ai.google.dev/api/generate-content#UsageMetadata + * @see https://github.com/vercel/ai/blob/main/packages/google/src/google-language-model.ts + */ + usageMetadata?: null | { + promptTokenCount?: number; + candidatesTokenCount?: number; + thoughtsTokenCount?: number; + totalTokenCount?: number; + }; } /** diff --git a/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts b/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts new file mode 100644 index 000000000000..ac4e13b60501 --- /dev/null +++ b/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from 'vitest'; +import { addVercelAiProcessors } from '../../../src/tracing/vercel-ai'; +import type { SpanJSON } from '../../../src/types/span'; +import { getDefaultTestClientOptions, TestClient } from '../../mocks/client'; + +/** + * Real usage from a Gemini reasoning model: the candidate output is one token and the model spent + * the rest of its budget on hidden reasoning ("thoughts"). The AI SDK reports the candidate count + * as `outputTokens` and exposes the reasoning count only through + * `providerMetadata.google.usageMetadata`, so a span built from `ai.usage.*` alone undercounts + * output by 100 and undercounts the total by the same. + */ +const GEMINI_REASONING_METADATA = { + google: { + groundingMetadata: null, + safetyRatings: null, + usageMetadata: { + promptTokenCount: 14, + candidatesTokenCount: 1, + thoughtsTokenCount: 100, + totalTokenCount: 115, + }, + }, +}; + +function processSpans(spans: SpanJSON[]): SpanJSON[] { + const options = getDefaultTestClientOptions({ tracesSampleRate: 1.0 }); + const client = new TestClient(options); + client.init(); + addVercelAiProcessors(client); + + const eventProcessor = client['_eventProcessors'].find(processor => processor.id === 'VercelAiEventProcessor'); + expect(eventProcessor).toBeDefined(); + + return eventProcessor!({ type: 'transaction' as const, spans }, {})!.spans!; +} + +function span(description: string, data: SpanJSON['data'], spanId = 'span-1', parentSpanId?: string): SpanJSON { + return { + description, + span_id: spanId, + parent_span_id: parentSpanId, + trace_id: 'test-trace-id', + start_timestamp: 1000, + timestamp: 2000, + origin: 'auto.vercelai.otel', + data, + }; +} + +describe('vercel-ai Gemini reasoning tokens', () => { + it('adds reasoning tokens into output and total on a model span', () => { + const [processed] = processSpans([ + span('ai.generateText.doGenerate', { + 'ai.usage.promptTokens': 14, + 'ai.usage.completionTokens': 1, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }), + ]); + + expect(processed?.data?.['gen_ai.usage.input_tokens']).toBe(14); + // Output covers the 100 reasoning tokens, not just the single candidate token. + expect(processed?.data?.['gen_ai.usage.output_tokens']).toBe(101); + expect(processed?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + // The real Gemini total, not input plus candidate-only output, which would be 15. + expect(processed?.data?.['gen_ai.usage.total_tokens']).toBe(115); + }); + + it('reads the v6 vertex key the same way', () => { + const [processed] = processSpans([ + span('ai.generateText.doGenerate', { + 'ai.usage.completionTokens': 1, + 'ai.response.providerMetadata': JSON.stringify({ vertex: GEMINI_REASONING_METADATA.google }), + }), + ]); + + expect(processed?.data?.['gen_ai.usage.output_tokens']).toBe(101); + expect(processed?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + }); + + it('leaves output and total alone when candidatesTokenCount is absent', () => { + // `candidatesTokenCount` is optional in the Gemini response. Without it there is no candidate + // count to add reasoning to, so rewriting the total on its own would leave a span whose + // thoughts-inclusive total does not match its candidate-only output. + const [processed] = processSpans([ + span('ai.generateText.doGenerate', { + 'ai.usage.completionTokens': 1, + 'ai.response.providerMetadata': JSON.stringify({ + google: { usageMetadata: { promptTokenCount: 14, thoughtsTokenCount: 100, totalTokenCount: 115 } }, + }), + }), + ]); + + expect(processed?.data?.['gen_ai.usage.output_tokens']).toBe(1); + expect(processed?.data?.['gen_ai.usage.total_tokens']).toBeUndefined(); + // The reasoning count is still reported: nothing else on the span carries it. + expect(processed?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + }); + + it('does not overwrite an invoke_agent parent aggregate from the last step', () => { + // The parent carries the summed usage of both steps; providerMetadata describes step two only. + // Writing output or total from it would report step two's figures against the summed input. + const [parent] = processSpans([ + span('ai.generateText', { + 'operation.name': 'ai.generateText', + 'ai.usage.promptTokens': 900, + 'ai.usage.completionTokens': 350, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }), + ]); + + expect(parent?.data?.['gen_ai.operation.name']).toBe('invoke_agent'); + expect(parent?.data?.['gen_ai.usage.input_tokens']).toBe(900); + expect(parent?.data?.['gen_ai.usage.output_tokens']).toBe(350); + expect(parent?.data?.['gen_ai.usage.total_tokens']).not.toBe(115); + // The reasoning count is not an aggregate, so it survives and is the only place it appears. + expect(parent?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + }); + + it('keeps a multi-step call consistent: the parent sums, each step reports its own reasoning', () => { + const stepOne = { + google: { + usageMetadata: { + promptTokenCount: 400, + candidatesTokenCount: 20, + thoughtsTokenCount: 80, + totalTokenCount: 500, + }, + }, + }; + const processed = processSpans([ + span( + 'ai.generateText', + { + 'operation.name': 'ai.generateText', + 'ai.usage.promptTokens': 900, + 'ai.usage.completionTokens': 350, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }, + 'parent', + ), + span( + 'ai.generateText.doGenerate', + { + 'ai.usage.promptTokens': 400, + 'ai.usage.completionTokens': 20, + 'ai.response.providerMetadata': JSON.stringify(stepOne), + }, + 'step-1', + 'parent', + ), + span( + 'ai.generateText.doGenerate', + { + 'ai.usage.promptTokens': 500, + 'ai.usage.completionTokens': 1, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }, + 'step-2', + 'parent', + ), + ]); + + const [parent, first, second] = processed; + + // Each step gets its own reasoning-inclusive figures. + expect(first?.data?.['gen_ai.usage.output_tokens']).toBe(100); + expect(first?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(80); + expect(second?.data?.['gen_ai.usage.output_tokens']).toBe(101); + expect(second?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + + // The parent keeps an aggregate whose parts add up, rather than step two's 101 and 115. + expect(parent?.data?.['gen_ai.usage.input_tokens']).toBe(900); + expect(parent?.data?.['gen_ai.usage.output_tokens']).not.toBe(101); + expect(parent?.data?.['gen_ai.usage.total_tokens']).not.toBe(115); + }); +}); From f43d12d38102e64b408b324a883e75b7e4a0496e Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Thu, 3 Sep 2026 11:38:43 +0200 Subject: [PATCH 2/4] fix(core): Keep Gemini reasoning tokens a subset of output tokens The conventions define `gen_ai.usage.reasoning.output_tokens` as a subset of `gen_ai.usage.output_tokens`, which is itself reasoning-inclusive. Two spans broke that: a model call whose response was truncated during thinking reported reasoning against the SDK's candidate-only output, and an `invoke_agent` parent reported the last step's reasoning against an output the gate deliberately leaves un-recomputed. Treat an absent `candidatesTokenCount` as zero rather than skipping the recompute. Gemini omits the field when no candidate tokens were produced, so the reasoning tokens belong in output either way; skipping left the span claiming zero output for a call that spent its whole budget thinking. Gate reasoning alongside output and total on `invoke_agent`. It is a subset of an output that span never recomputes, and the accumulator never sums it, so the last step's count would stand in for the whole call. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018TR1cvQA7t6T2saCrHwwUh --- packages/core/src/tracing/vercel-ai/index.ts | 30 ++++----- .../vercel-ai-reasoning-tokens.test.ts | 63 +++++++++++++++---- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index dd219567faca..b6bdfa32fe93 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -605,22 +605,26 @@ export function getProviderMetadataAttributes(providerMetadata: unknown): Record // tokens separately from the candidate output count, so the SDK's `outputTokens` covers only the // visible answer. Recompute output from `candidatesTokenCount + thoughtsTokenCount` rather than // adding onto the existing value, which stays correct even if a future SDK version already folds - // reasoning in. `candidatesTokenCount` is optional, so output and total are written together or - // not at all: taking the thoughts-inclusive total beside a candidate-only output would report a - // span whose parts do not add up. + // reasoning in. `candidatesTokenCount` is omitted when the response is truncated during thinking, + // which means no candidate tokens, so it counts as zero. Reasoning is a subset of output per the + // conventions, so it is only written alongside the reasoning-inclusive output it belongs to. const googleUsage = (metadata.google ?? metadata.vertex)?.usageMetadata; if (googleUsage && typeof googleUsage.thoughtsTokenCount === 'number' && googleUsage.thoughtsTokenCount > 0) { + attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] = + (googleUsage.candidatesTokenCount ?? 0) + googleUsage.thoughtsTokenCount; + setAttributeIfDefined(attributes, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, googleUsage.totalTokenCount); setAttributeIfDefined(attributes, 'gen_ai.usage.reasoning.output_tokens', googleUsage.thoughtsTokenCount); - if (typeof googleUsage.candidatesTokenCount === 'number') { - attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] = - googleUsage.candidatesTokenCount + googleUsage.thoughtsTokenCount; - setAttributeIfDefined(attributes, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, googleUsage.totalTokenCount); - } } return attributes; } +const LAST_STEP_ONLY_USAGE_KEYS = new Set([ + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, + 'gen_ai.usage.reasoning.output_tokens', +]); + function addProviderMetadataToAttributes(attributes: Record): void { const providerMetadata = attributes[AI_RESPONSE_PROVIDER_METADATA_ATTRIBUTE] as string | undefined; if (!providerMetadata) { @@ -629,8 +633,9 @@ function addProviderMetadataToAttributes(attributes: Record): v // An `invoke_agent` span carries the summed `ai.usage.*` of every step, while `providerMetadata` // describes the last step alone. Writing output or total from it would replace the aggregate with // one step's figures, so a two-step call reports output 50 against input 900. The event-processor - // path happens to overwrite it again in `applyAccumulatedTokens`; the streamed path ships it. The - // reasoning count is still worth having, since nothing else carries it and it is not a total. + // path happens to overwrite it again in `applyAccumulatedTokens`; the streamed path ships it. + // Reasoning goes with them: it is a subset of an output the parent never recomputes, and the + // accumulator never sums it, so the last step's count would stand in for the whole call. const lastStepOnly = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE] === 'invoke_agent'; try { const derived = getProviderMetadataAttributes(JSON.parse(providerMetadata) as ProviderMetadata); @@ -639,10 +644,7 @@ function addProviderMetadataToAttributes(attributes: Record): v if (key === GEN_AI_CONVERSATION_ID_ATTRIBUTE && attributes[key]) { continue; } - if ( - lastStepOnly && - (key === GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE || key === GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE) - ) { + if (lastStepOnly && LAST_STEP_ONLY_USAGE_KEYS.has(key)) { continue; } attributes[key] = value; diff --git a/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts b/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts index ac4e13b60501..90d8baa1582e 100644 --- a/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts +++ b/packages/core/test/lib/tracing/vercel-ai-reasoning-tokens.test.ts @@ -78,10 +78,10 @@ describe('vercel-ai Gemini reasoning tokens', () => { expect(processed?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); }); - it('leaves output and total alone when candidatesTokenCount is absent', () => { - // `candidatesTokenCount` is optional in the Gemini response. Without it there is no candidate - // count to add reasoning to, so rewriting the total on its own would leave a span whose - // thoughts-inclusive total does not match its candidate-only output. + it('counts an absent candidatesTokenCount as zero', () => { + // Gemini omits `candidatesTokenCount` when the response is truncated during thinking, which + // means no candidate tokens were produced. Treating it as zero keeps the reasoning tokens in + // the output rather than dropping the whole recompute and reporting the candidate-only count. const [processed] = processSpans([ span('ai.generateText.doGenerate', { 'ai.usage.completionTokens': 1, @@ -91,9 +91,8 @@ describe('vercel-ai Gemini reasoning tokens', () => { }), ]); - expect(processed?.data?.['gen_ai.usage.output_tokens']).toBe(1); - expect(processed?.data?.['gen_ai.usage.total_tokens']).toBeUndefined(); - // The reasoning count is still reported: nothing else on the span carries it. + expect(processed?.data?.['gen_ai.usage.output_tokens']).toBe(100); + expect(processed?.data?.['gen_ai.usage.total_tokens']).toBe(115); expect(processed?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); }); @@ -112,9 +111,10 @@ describe('vercel-ai Gemini reasoning tokens', () => { expect(parent?.data?.['gen_ai.operation.name']).toBe('invoke_agent'); expect(parent?.data?.['gen_ai.usage.input_tokens']).toBe(900); expect(parent?.data?.['gen_ai.usage.output_tokens']).toBe(350); - expect(parent?.data?.['gen_ai.usage.total_tokens']).not.toBe(115); - // The reasoning count is not an aggregate, so it survives and is the only place it appears. - expect(parent?.data?.['gen_ai.usage.reasoning.output_tokens']).toBe(100); + expect(parent?.data?.['gen_ai.usage.total_tokens']).toBe(1250); + // Reasoning is a subset of an output this span never recomputes, so it is left off entirely + // rather than reporting the last step's count against the summed output. + expect(parent?.data?.['gen_ai.usage.reasoning.output_tokens']).toBeUndefined(); }); it('keeps a multi-step call consistent: the parent sums, each step reports its own reasoning', () => { @@ -171,7 +171,46 @@ describe('vercel-ai Gemini reasoning tokens', () => { // The parent keeps an aggregate whose parts add up, rather than step two's 101 and 115. expect(parent?.data?.['gen_ai.usage.input_tokens']).toBe(900); - expect(parent?.data?.['gen_ai.usage.output_tokens']).not.toBe(101); - expect(parent?.data?.['gen_ai.usage.total_tokens']).not.toBe(115); + expect(parent?.data?.['gen_ai.usage.output_tokens']).toBe(350); + expect(parent?.data?.['gen_ai.usage.total_tokens']).toBe(1250); + // The last step's 100 would stand in for the call's real 180, and nothing sums it, so it is + // left off rather than reported. + expect(parent?.data?.['gen_ai.usage.reasoning.output_tokens']).toBeUndefined(); + }); + + it('holds on the streamed path, which has no accumulation pass to repair the parent', () => { + // The event processor sees the whole transaction and re-derives an `invoke_agent` parent from + // its children; `processSpan` sees one span at a time and ships whatever it produced. This is + // the path the gate actually protects. + const client = new TestClient(getDefaultTestClientOptions({ tracesSampleRate: 1.0 })); + client.init(); + addVercelAiProcessors(client); + + const streamed = (attrs: Record): Record => { + const span = { span_id: 's', trace_id: 't', attributes: { 'sentry.origin': 'auto.vercelai.otel', ...attrs } }; + client.emit('processSpan', span as never); + return span.attributes; + }; + + const parent = streamed({ + 'operation.name': 'ai.generateText', + 'ai.usage.inputTokens': 14, + 'ai.usage.outputTokens': 1, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }); + const child = streamed({ + 'operation.name': 'ai.generateText.doGenerate', + 'ai.usage.promptTokens': 14, + 'ai.usage.completionTokens': 1, + 'ai.response.providerMetadata': JSON.stringify(GEMINI_REASONING_METADATA), + }); + + expect(child['gen_ai.usage.output_tokens']).toBe(101); + expect(child['gen_ai.usage.total_tokens']).toBe(115); + expect(child['gen_ai.usage.reasoning.output_tokens']).toBe(100); + + // No reasoning count larger than the output it is meant to be a subset of. + expect(parent['gen_ai.usage.output_tokens']).toBe(1); + expect(parent['gen_ai.usage.reasoning.output_tokens']).toBeUndefined(); }); }); From f3bad94c8c9b984d6f8b145141f02af66a0a084f Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Thu, 3 Sep 2026 12:26:11 +0200 Subject: [PATCH 3/4] fix(server-utils): Gate last-step usage on channel-path `invoke_agent` spans `getProviderMetadataAttributes` now derives `gen_ai.usage.output_tokens` and `gen_ai.usage.total_tokens`, but only one of its three callers dropped them on spans that report usage aggregated across steps. The channel and orchestrion subscribers call it directly rather than through `addProviderMetadataToAttributes`, so a top-level operation's span took the last step's figures over its own aggregate. Reachable on `ai` v4, where `generateText` accumulates `usage` across steps (`addLanguageModelUsage`) while exposing the final step's `providerMetadata`: a multi-step Gemini call reported the last step's output and total against the summed input. On v5+ the result's `usage` is the final step's, so the two agree and nothing changes. Export the key set from core and apply it in `enrichSpanOnEnd`, which both subscribers share, so all three callers follow the same rule. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018TR1cvQA7t6T2saCrHwwUh --- packages/core/src/shared-exports.ts | 2 +- packages/core/src/tracing/vercel-ai/index.ts | 10 +++++++++- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 11 +++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/core/src/shared-exports.ts b/packages/core/src/shared-exports.ts index 4c1100142fec..3ed79e5347aa 100644 --- a/packages/core/src/shared-exports.ts +++ b/packages/core/src/shared-exports.ts @@ -178,7 +178,7 @@ export { export * as metrics from './metrics/public-api'; export type { MetricOptions } from './metrics/public-api'; export { createConsolaReporter } from './integrations/consola'; -export { addVercelAiProcessors, getProviderMetadataAttributes } from './tracing/vercel-ai'; +export { addVercelAiProcessors, getProviderMetadataAttributes, LAST_STEP_ONLY_USAGE_KEYS } from './tracing/vercel-ai'; export { getTruncatedJsonString, shouldEnableTruncation, resolveAIRecordingOptions } from './tracing/ai/utils'; export { GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index b6bdfa32fe93..c6f296b77819 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -619,7 +619,15 @@ export function getProviderMetadataAttributes(providerMetadata: unknown): Record return attributes; } -const LAST_STEP_ONLY_USAGE_KEYS = new Set([ +/** + * Usage attributes that `getProviderMetadataAttributes` derives from `providerMetadata`, which + * describes only the last step of a call. They must not be written onto a span that reports usage + * aggregated across steps (`gen_ai.invoke_agent`), where they would replace the aggregate with one + * step's figures. Exported so the channel/orchestrion subscribers, which call + * `getProviderMetadataAttributes` directly rather than through `addProviderMetadataToAttributes`, + * apply the same rule. + */ +export const LAST_STEP_ONLY_USAGE_KEYS = new Set([ GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, 'gen_ai.usage.reasoning.output_tokens', diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index fead1c746383..9098b7c616e9 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -35,6 +35,7 @@ import { GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE, getClient, getProviderMetadataAttributes, + LAST_STEP_ONLY_USAGE_KEYS, getTruncatedJsonString, isObjectLike, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, @@ -576,6 +577,16 @@ export function enrichSpanOnEnd( // oxlint-disable-next-line typescript/no-dynamic-delete delete providerAttributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]; } + // A top-level operation's span reports usage aggregated across every step, while + // `providerMetadata` describes the last step alone. Dropping the derived usage keeps the + // aggregate intact; the model-call spans still carry the provider-derived figures. Matches the + // OTel path, which applies the same rule in `addProviderMetadataToAttributes`. + if (ROOT_OPERATION_TYPES.has(type)) { + for (const key of LAST_STEP_ONLY_USAGE_KEYS) { + // oxlint-disable-next-line typescript/no-dynamic-delete + delete providerAttributes[key]; + } + } span.setAttributes(providerAttributes); if (recordOutputs) { From a1041e1fc9e78fa74a7714dfdcbe55c052c30b31 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Thu, 3 Sep 2026 12:55:22 +0200 Subject: [PATCH 4/4] fix(server-utils): Drop enrichSpanOnEnd below the complexity limit The last-step usage gate pushed the function to 34. Extracting it keeps the same behavior without tripping oxlint. --- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index 9098b7c616e9..b9c8e0ad1798 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -33,9 +33,9 @@ import { GEN_AI_CONVERSATION_ID_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE, + LAST_STEP_ONLY_USAGE_KEYS, getClient, getProviderMetadataAttributes, - LAST_STEP_ONLY_USAGE_KEYS, getTruncatedJsonString, isObjectLike, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, @@ -131,6 +131,20 @@ export function clearOperationCallId(callId: string): void { invokeAgentSpanByCallId.delete(callId); } +/** + * `providerMetadata` is last-step only; drop derived usage on spans that report an aggregate. + * Matches `addProviderMetadataToAttributes`. + */ +function dropLastStepOnlyUsage(providerAttributes: Record, type: ChannelEventType): void { + if (!ROOT_OPERATION_TYPES.has(type)) { + return; + } + for (const key of LAST_STEP_ONLY_USAGE_KEYS) { + // oxlint-disable-next-line typescript/no-dynamic-delete + delete providerAttributes[key]; + } +} + /** Record tool name → description from an event's `tools`, so tool spans can backfill the description. */ function recordToolDescriptions(callId: string | undefined, tools: unknown): void { if (!callId || !Array.isArray(tools)) { @@ -577,16 +591,7 @@ export function enrichSpanOnEnd( // oxlint-disable-next-line typescript/no-dynamic-delete delete providerAttributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]; } - // A top-level operation's span reports usage aggregated across every step, while - // `providerMetadata` describes the last step alone. Dropping the derived usage keeps the - // aggregate intact; the model-call spans still carry the provider-derived figures. Matches the - // OTel path, which applies the same rule in `addProviderMetadataToAttributes`. - if (ROOT_OPERATION_TYPES.has(type)) { - for (const key of LAST_STEP_ONLY_USAGE_KEYS) { - // oxlint-disable-next-line typescript/no-dynamic-delete - delete providerAttributes[key]; - } - } + dropLastStepOnlyUsage(providerAttributes, type); span.setAttributes(providerAttributes); if (recordOutputs) {