diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index 9dc72d748e1..4fd87951bf9 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -913,6 +913,36 @@ export const mapUsage = ( }) } +// GitHub Copilot lists the filters that ran; the blocking one names its rule in `content_filter_raw`. +const decodeContentFilters = Schema.decodeUnknownOption( + Schema.Array( + Schema.Struct({ + source_type: Schema.optional(Schema.String), + blocked: Schema.optional(Schema.Boolean), + content_filter_raw: Schema.optional( + Schema.Array( + Schema.Struct({ + action: Schema.optional(Schema.String), + label: Schema.optional(Schema.String), + result: Schema.optional(Schema.Unknown), + }), + ), + ), + content_filter_results: Schema.optional(Schema.Unknown), + }), + ), +) + +const contentFilterCategory = (raw: unknown) => { + const blocked = Option.getOrElse(decodeContentFilters(raw), () => []).filter((filter) => filter.blocked) + const filter = blocked.find((item) => item.source_type === "completion") ?? blocked[0] + if (!filter) return undefined + return ( + filter.content_filter_raw?.find((rule) => rule.action === "BLOCK" && rule.result === true)?.label ?? + ProviderShared.contentFilterCategory(filter.content_filter_results) + ) +} + const mapFinishReason = (event: Event, hasFunctionCall: boolean): FinishReason => { const reason = event.response?.incomplete_details?.reason if (reason === undefined) { @@ -1397,10 +1427,12 @@ const onResponseFinish = Effect.fnUntraced(function* (state: ParserState, event: } events.push(...pending.events) } + const normalized = mapFinishReason(event, current.hasFunctionCall) const lifecycle = Lifecycle.finish(current.lifecycle, events, { reason: { - normalized: mapFinishReason(event, current.hasFunctionCall), + normalized, raw: event.response?.incomplete_details?.reason, + ...(normalized === "content-filter" ? { category: contentFilterCategory(event.response?.content_filters) } : {}), }, // Copilot reports its billing on the terminal event, beside `response`. usage: mapUsage(event.response?.usage, current.providerMetadataKey, event, event.response), diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index af97167e240..da0622e226f 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -265,6 +265,8 @@ export const OpenAIChatUsage = Schema.StructWithRest( prompt_cache_hit_tokens: optionalNull(Schema.Number), cache_read_input_tokens: optionalNull(Schema.Number), cache_created_input_tokens: optionalNull(Schema.Number), + // GitHub Copilot reports reasoning at the top level instead of in `completion_tokens_details`. + reasoning_tokens: optionalNull(Schema.Number), prompt_tokens_details: optionalNull( Schema.StructWithRest( Schema.Struct({ @@ -967,7 +969,7 @@ const mapUsage = ( ): Usage | undefined => { if (!usage) return undefined const input = usage.prompt_tokens ?? undefined - const output = usage.completion_tokens ?? undefined + const completion = usage.completion_tokens ?? undefined const cached = usage.prompt_tokens_details?.cached_tokens ?? usage.prompt_cache_hit_tokens ?? @@ -975,7 +977,14 @@ const mapUsage = ( usage.cache_read_input_tokens ?? undefined const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? usage.cache_created_input_tokens ?? undefined - const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? undefined + const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? usage.reasoning_tokens ?? undefined + // Copilot's Gemini counts reasoning beside `completion_tokens` rather than inside it; the total tells which. + const output = + completion !== undefined && + reasoning !== undefined && + usage.total_tokens === ProviderShared.sumTokens(input, completion, reasoning) + ? completion + reasoning + : completion const nonCached = ProviderShared.subtractTokens(input, ProviderShared.sumTokens(cached, cacheWrite)) return new Usage({ inputTokens: input, @@ -1128,6 +1137,14 @@ const step = (state: ParserState, event: OpenAIChatEvent) => ? { normalized: yield* mapFinishReason(event, rawFinishReason), raw: choice?.native_finish_reason ?? rawFinishReason, + ...(rawFinishReason === "content_filter" + ? { + category: ProviderShared.contentFilterCategory( + choice?.delta?.copilot_annotations, + choice?.content_filter_results, + ), + } + : {}), } : state.finishReason const delta = choice?.delta diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index 8f5c9c5c745..9ead5424ba5 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -76,6 +76,15 @@ export const reportedBilling = (usage: unknown, ...frames: ReadonlyArray) => + sources + .flatMap((source) => (isRecord(source) ? Object.entries(source) : [])) + .find(([, value]) => (isRecord(value) && "filtered" in value ? value.filtered === true : Boolean(value)))?.[0] + /** * Accepts stream frames that carry no `type`, alongside the typed event schema. Gateways inject such frames as * keep-alives, so they decode as an ignored event; one with an `error` still decodes as an error event. diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index a97a04b92ff..c3cadaed4e0 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -1264,6 +1264,25 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("reads top-level reasoning tokens whether or not completion tokens include them", () => + Effect.gen(function* () { + const generate = (usage: object) => + LLMClient.generate(request).pipe( + Effect.provide(fixedResponse(sseEvents(deltaChunk({ content: "Hi" }, "stop"), usageChunk(usage)))), + Effect.map((response) => response.usage), + ) + + // Copilot's Gemini: total = prompt + completion + reasoning. + expect( + yield* generate({ prompt_tokens: 7, completion_tokens: 2, total_tokens: 62, reasoning_tokens: 53 }), + ).toMatchObject({ outputTokens: 55, reasoningTokens: 53, totalTokens: 62 }) + // Copilot's Kimi: total = prompt + completion, which already includes reasoning. + expect( + yield* generate({ prompt_tokens: 96, completion_tokens: 50, total_tokens: 146, reasoning_tokens: 35 }), + ).toMatchObject({ outputTokens: 50, reasoningTokens: 35, totalTokens: 146 }) + }), + ) + it.effect("converts the GitHub Copilot billing beside usage to USD", () => Effect.gen(function* () { const response = yield* LLMClient.generate(request).pipe( diff --git a/packages/ai/test/provider/openai-compatible-chat.test.ts b/packages/ai/test/provider/openai-compatible-chat.test.ts index 6d7a1e01444..ee29dfbfa53 100644 --- a/packages/ai/test/provider/openai-compatible-chat.test.ts +++ b/packages/ai/test/provider/openai-compatible-chat.test.ts @@ -657,6 +657,31 @@ describe("OpenAI-compatible Chat route", () => { }), ) + it.effect("names the content filter that blocked a response", () => + Effect.gen(function* () { + const finish = (choice: object) => + LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents({ id: "chatcmpl_fixture", choices: [{ finish_reason: "content_filter", ...choice }] }), + ), + ), + Effect.map((response) => response.finishReason), + ) + + expect(yield* finish({ delta: { copilot_annotations: { TextCopyright: [{ id: 1 }] } } })).toMatchObject({ + normalized: "content-filter", + category: "TextCopyright", + }) + expect( + yield* finish({ + delta: {}, + content_filter_results: { hate: { filtered: false }, violence: { filtered: true } }, + }), + ).toMatchObject({ normalized: "content-filter", category: "violence" }) + }), + ) + it.effect("preserves late text and reasoning for replay without changing the finished tool call", () => Effect.gen(function* () { const details = [{ type: "reasoning.encrypted", data: "opaque", id: "late-reasoning" }] @@ -723,7 +748,11 @@ describe("OpenAI-compatible Chat route", () => { expect(response.reasoning).toBe("Read the readme first. Then the manifest.") expect(response.events.filter(LLMEvent.is.reasoningEnd)).toMatchObject([ - { providerMetadata: { "github-copilot": { reasoningField: "reasoning_text", reasoningOpaque: "opaque-second" } } }, + { + providerMetadata: { + "github-copilot": { reasoningField: "reasoning_text", reasoningOpaque: "opaque-second" }, + }, + }, ]) expect(response.toolCalls.map((call) => call.id)).toEqual(["call_first", "call_second"]) const replay = yield* compileRequest(ai.request({ model: copilot, messages: [response.message] })) diff --git a/packages/ai/test/provider/openai-compatible-responses.test.ts b/packages/ai/test/provider/openai-compatible-responses.test.ts index 481ad1d0d84..b05858e1011 100644 --- a/packages/ai/test/provider/openai-compatible-responses.test.ts +++ b/packages/ai/test/provider/openai-compatible-responses.test.ts @@ -444,6 +444,35 @@ describe("Open Responses-compatible route", () => { }), ) + it.effect("names the GitHub Copilot content filter that blocked a response", () => + Effect.gen(function* () { + const model = configure({ apiKey: "test-key", baseURL: "https://api.githubcopilot.test" }).model("gpt-5") + const response = yield* LLMClient.generate(ai.request({ model, prompt: "Say hello." })).pipe( + Effect.provide( + fixedResponse( + sseEvents({ + type: "response.incomplete", + response: { + id: "resp_1", + incomplete_details: { reason: "content_filter" }, + content_filters: [ + { source_type: "prompt", blocked: false }, + { + source_type: "completion", + blocked: true, + content_filter_raw: [{ action: "BLOCK", label: "TextCopyright", result: true }], + }, + ], + }, + }), + ), + ), + ) + + expect(response.finishReason).toMatchObject({ normalized: "content-filter", category: "TextCopyright" }) + }), + ) + it.effect("converts the GitHub Copilot billing on the completed event to USD", () => Effect.gen(function* () { const model = configure({ apiKey: "test-key", baseURL: "https://api.githubcopilot.test" }).model("gpt-5")