diff --git a/packages/ai/src/testing.ts b/packages/ai/src/testing.ts index 3ccafe9c611..a841f1df36d 100644 --- a/packages/ai/src/testing.ts +++ b/packages/ai/src/testing.ts @@ -7,6 +7,7 @@ import { type FinishReasonDetails, type AIError, type LLMRequest, + type ProviderMetadata, type UsageInput, } from "./schema/index.js" import { Context, Deferred, Effect, Latch, Layer, Queue, Scope, Stream } from "effect" @@ -33,13 +34,22 @@ export interface LayerOptions { export class Service extends Context.Service()("@opencode/ai/TestLLM") {} export const complete = ( - options: { readonly reason: FinishReasonDetails; readonly usage?: UsageInput }, + options: { + readonly reason: FinishReasonDetails + readonly usage?: UsageInput + readonly providerMetadata?: ProviderMetadata + }, ...events: readonly LLMEvent[] ) => [ LLMEvent.stepStart({ index: 0 }), ...events, - LLMEvent.stepFinish({ index: 0, reason: options.reason, usage: options.usage }), - LLMEvent.finish({ reason: options.reason }), + LLMEvent.stepFinish({ + index: 0, + reason: options.reason, + usage: options.usage, + providerMetadata: options.providerMetadata, + }), + LLMEvent.finish({ reason: options.reason, providerMetadata: options.providerMetadata }), ] export const stop = (...events: readonly LLMEvent[]) => complete({ reason: { normalized: "stop" } }, ...events) diff --git a/packages/client/src/effect/api/api.ts b/packages/client/src/effect/api/api.ts index 3e41acd814e..dbcc071fc2c 100644 --- a/packages/client/src/effect/api/api.ts +++ b/packages/client/src/effect/api/api.ts @@ -579,6 +579,8 @@ export type Endpoint5_31Output = readonly sessionID: Session.ID readonly assistantMessageID: SessionMessage.ID readonly finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + readonly rawFinish?: string | undefined + readonly providerState?: SessionMessage.ProviderState | undefined readonly cost: number & Brand.Brand<"Money.USD"> readonly tokens: { readonly input: number @@ -601,6 +603,9 @@ export type Endpoint5_31Output = readonly sessionID: Session.ID readonly assistantMessageID: SessionMessage.ID readonly error: { readonly type: string; readonly message: string; readonly status?: number | undefined } + readonly finish?: "content-filter" | undefined + readonly rawFinish?: string | undefined + readonly providerState?: SessionMessage.ProviderState | undefined readonly cost?: (number & Brand.Brand<"Money.USD">) | undefined readonly tokens?: | { diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 2ae36dc7a38..0d54ac34f99 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -1108,6 +1108,8 @@ export type SessionStepEnded = { sessionID: string assistantMessageID: string finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + rawFinish?: string + providerState?: SessionMessageProviderState1 cost: MoneyUSD tokens: TokenUsageInfo snapshot?: string @@ -1145,6 +1147,9 @@ export type SessionStepFailed = { sessionID: string assistantMessageID: string error: SessionStructuredError + finish?: "content-filter" + rawFinish?: string + providerState?: SessionMessageProviderState1 cost?: MoneyUSD tokens?: TokenUsageInfo snapshot?: string @@ -1921,6 +1926,8 @@ export type SessionMessageAssistant = { content: Array snapshot?: { start?: string; end?: string; files?: Array } finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + rawFinish?: string + providerState?: SessionMessageProviderState cost?: MoneyUSD tokens?: TokenUsageInfo error?: SessionStructuredError @@ -2691,6 +2698,8 @@ export type SessionImportInput = { > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + readonly rawFinish?: string + readonly providerState?: { readonly [x: string]: JsonValue } readonly cost?: number readonly tokens?: { readonly input: number @@ -2958,6 +2967,8 @@ export type SessionImportInput = { > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + readonly rawFinish?: string + readonly providerState?: { readonly [x: string]: JsonValue } readonly cost?: number readonly tokens?: { readonly input: number @@ -3225,6 +3236,8 @@ export type SessionImportInput = { > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" + readonly rawFinish?: string + readonly providerState?: { readonly [x: string]: JsonValue } readonly cost?: number readonly tokens?: { readonly input: number diff --git a/packages/client/src/solid/data.ts b/packages/client/src/solid/data.ts index 593513595ea..32eb7546eb1 100644 --- a/packages/client/src/solid/data.ts +++ b/packages/client/src/solid/data.ts @@ -601,6 +601,8 @@ export function createData(config: CreateDataInput) { existing.retry = undefined existing.error = undefined existing.finish = undefined + existing.rawFinish = undefined + existing.providerState = undefined existing.time.completed = undefined if (event.data.snapshot) existing.snapshot = { ...existing.snapshot, start: event.data.snapshot } return @@ -628,6 +630,8 @@ export function createData(config: CreateDataInput) { if (!currentAssistant) return currentAssistant.time.completed = event.created currentAssistant.finish = event.data.finish + currentAssistant.rawFinish = event.data.rawFinish + currentAssistant.providerState = event.data.providerState currentAssistant.cost = event.data.cost currentAssistant.tokens = event.data.tokens if (event.data.snapshot) @@ -640,7 +644,9 @@ export function createData(config: CreateDataInput) { const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID) if (!currentAssistant) return currentAssistant.time.completed = event.created - currentAssistant.finish = "error" + currentAssistant.finish = event.data.finish ?? "error" + currentAssistant.rawFinish = event.data.rawFinish + currentAssistant.providerState = event.data.providerState currentAssistant.error = event.data.error currentAssistant.retry = undefined if (event.data.cost !== undefined && event.data.tokens !== undefined) { diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index e94f6e3e9c0..a493fadb317 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -195,6 +195,8 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { draft.retry = undefined draft.error = undefined draft.finish = undefined + draft.rawFinish = undefined + draft.providerState = undefined draft.time.completed = undefined if (event.data.snapshot) draft.snapshot = { ...draft.snapshot, start: event.data.snapshot } }), @@ -228,6 +230,8 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { draft.time.completed = created draft.finish = event.data.finish + draft.rawFinish = event.data.rawFinish + draft.providerState = castDraft(event.data.providerState) draft.cost = event.data.cost draft.tokens = event.data.tokens if (event.data.snapshot || event.data.files) @@ -241,7 +245,9 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { "session.step.failed": (event) => { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { draft.time.completed = created - draft.finish = "error" + draft.finish = event.data.finish ?? "error" + draft.rawFinish = event.data.rawFinish + draft.providerState = castDraft(event.data.providerState) draft.error = castDraft(event.data.error) draft.retry = undefined if (event.data.cost !== undefined && event.data.tokens !== undefined) { diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 34577e86a68..e6ad8494699 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -326,6 +326,8 @@ const layer = Layer.effect( sessionID: session.id, assistantMessageID: yield* publisher.startAssistant(), finish: finish.finish, + rawFinish: finish.rawFinish, + providerState: finish.providerState, ...stepUsage(finish), ...end, }) diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 7b17266ac1a..f569f686fb9 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -35,6 +35,8 @@ export interface StepRecord { /** Present once the provider finished the step normally. */ readonly finish?: { readonly finish: Extract["reason"]["normalized"] + readonly rawFinish?: string + readonly providerState?: SessionMessage.ProviderState readonly tokens: ReturnType } readonly calls: ReadonlyArray<{ @@ -364,6 +366,9 @@ export const createLLMEventPublisher = (bus: Pick, inp sessionID: input.sessionID, assistantMessageID, error: stepFailure, + finish: stepSettlement?.finish === "content-filter" ? stepSettlement.finish : undefined, + rawFinish: stepSettlement?.rawFinish, + providerState: stepSettlement?.providerState, ...details, }) }) @@ -517,7 +522,12 @@ export const createLLMEventPublisher = (bus: Pick, inp case "step-finish": yield* flush() if (stepSettlement) return yield* Effect.die(new Error("Duplicate step finish")) - stepSettlement = { finish: event.reason.normalized, tokens: SessionUsage.tokens(event.usage) } + stepSettlement = { + finish: event.reason.normalized, + rawFinish: event.reason.raw, + providerState: providerState(event.providerMetadata), + tokens: SessionUsage.tokens(event.usage), + } if (event.reason.normalized === "content-filter") { providerFailed = true yield* failAssistant({ type: "provider.content-filter", message: "Provider blocked the response" }) diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index 0f36123698b..1ed0f16fac5 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -353,7 +353,12 @@ test("content-filter finish retains failure evidence until step closeout", async publisher.publish( LLMEvent.stepFinish({ index: 0, - reason: { normalized: "content-filter" }, + reason: { normalized: "content-filter", raw: "refusal" }, + providerMetadata: { + anthropic: { + stopDetails: { type: "refusal", category: "safety", explanation: "Blocked" }, + }, + }, usage: { nonCachedInputTokens: 8, outputTokens: 3, @@ -367,6 +372,10 @@ test("content-filter finish retains failure evidence until step closeout", async const settlement = publisher.record().finish expect(settlement).toMatchObject({ finish: "content-filter", + rawFinish: "refusal", + providerState: { + stopDetails: { type: "refusal", category: "safety", explanation: "Blocked" }, + }, tokens: { input: 8, output: 2, reasoning: 1 }, }) if (!settlement) throw new Error("Expected content-filter settlement") @@ -381,6 +390,11 @@ test("content-filter finish retains failure evidence until step closeout", async expect(published.map((event) => event.type)).toEqual(["session.step.started.1", "session.step.failed.1"]) expect(published.at(-1)?.data).toMatchObject({ error: { type: "provider.content-filter", message: "Provider blocked the response" }, + finish: "content-filter", + rawFinish: "refusal", + providerState: { + stopDetails: { type: "refusal", category: "safety", explanation: "Blocked" }, + }, cost: 1.25, tokens: { input: 8, output: 2, reasoning: 1 }, snapshot: "tree-end", diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index c4d31dac420..9e6d60b9d1e 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -4161,13 +4161,49 @@ describe("SessionRunnerLLM", () => { }), ) + it.effect("persists raw finish reasons and provider state", () => + Effect.gen(function* () { + const session = yield* setup + yield* TestLLM.push( + TestLLM.complete( + { + reason: { normalized: "stop", raw: "end_turn" }, + providerMetadata: { openai: { responseId: "response-1", serviceTier: "priority" } }, + }, + LLMEvent.textStart({ id: "answer" }), + LLMEvent.textDelta({ id: "answer", text: "Complete" }), + LLMEvent.textEnd({ id: "answer" }), + ), + ) + + yield* runPrompt(session, "Keep provider finish details") + + expect(yield* session.context(sessionID)).toMatchObject([ + { type: "user" }, + { + type: "assistant", + finish: "stop", + rawFinish: "end_turn", + providerState: { responseId: "response-1", serviceTier: "priority" }, + content: [{ type: "text", text: "Complete" }], + }, + ]) + }), + ) + it.effect("projects content-filter finishes as visible terminal failures", () => Effect.gen(function* () { const session = yield* setup yield* TestLLM.push( TestLLM.complete( { - reason: { normalized: "content-filter" }, + reason: { normalized: "content-filter", raw: "SAFETY" }, + providerMetadata: { + openai: { + responseId: "response-blocked", + refusal: { category: "safety", explanation: "Prompt blocked" }, + }, + }, usage: { nonCachedInputTokens: 8, outputTokens: 3, reasoningTokens: 1 }, }, LLMEvent.textStart({ id: "partial" }), @@ -4182,7 +4218,12 @@ describe("SessionRunnerLLM", () => { { type: "user" }, { type: "assistant", - finish: "error", + finish: "content-filter", + rawFinish: "SAFETY", + providerState: { + responseId: "response-blocked", + refusal: { category: "safety", explanation: "Prompt blocked" }, + }, error: { type: "provider.content-filter" }, cost: 0, tokens: { input: 8, output: 2, reasoning: 1, cache: { read: 0, write: 0 } }, diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index cd07af9ceb6..57865747a8d 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -298,6 +298,8 @@ export namespace Step { ...Base, assistantMessageID: SessionMessage.ID, finish: FinishReason, + rawFinish: Schema.String.pipe(optional), + providerState: SessionMessage.ProviderState.pipe(optional), cost: Money.USD, tokens: TokenUsage.Info, snapshot: Snapshot.ID.pipe(optional), @@ -313,6 +315,9 @@ export namespace Step { ...Base, assistantMessageID: SessionMessage.ID, error: SessionError.Error, + finish: Schema.Literals(["content-filter"]).pipe(optional), + rawFinish: Schema.String.pipe(optional), + providerState: SessionMessage.ProviderState.pipe(optional), cost: Money.USD.pipe(optional), tokens: TokenUsage.Info.pipe(optional), snapshot: Snapshot.ID.pipe(optional), diff --git a/packages/schema/src/session-message.ts b/packages/schema/src/session-message.ts index 0de8d68a04c..9323ca58d65 100644 --- a/packages/schema/src/session-message.ts +++ b/packages/schema/src/session-message.ts @@ -215,6 +215,8 @@ export const Assistant = Schema.Struct({ files: Schema.Array(RelativePath).pipe(optional), }).pipe(optional), finish: FinishReason.pipe(optional), + rawFinish: Schema.String.pipe(optional), + providerState: ProviderState.pipe(optional), cost: Money.USD.pipe(optional), tokens: TokenUsage.Info.pipe(optional), error: SessionError.Error.pipe(optional), diff --git a/packages/schema/test/session-terminal.test.ts b/packages/schema/test/session-terminal.test.ts new file mode 100644 index 00000000000..ef5ef79ae8e --- /dev/null +++ b/packages/schema/test/session-terminal.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test" +import { Schema } from "effect" +import { SessionEvent } from "../src/session-event.js" +import { SessionMessage } from "../src/session-message.js" + +const assistant = { + id: "msg_terminal", + type: "assistant" as const, + agent: "build", + model: { providerID: "openai", id: "gpt-test" }, + content: [], + time: { created: 0 }, +} + +test("assistant terminal diagnostics remain optional and round trip", () => { + const decode = Schema.decodeUnknownSync(SessionMessage.Assistant) + const encode = Schema.encodeSync(SessionMessage.Assistant) + + expect(encode(decode(assistant))).toEqual(assistant) + expect( + encode( + decode({ + ...assistant, + finish: "content-filter", + rawFinish: "SAFETY", + providerState: { promptFeedback: { blockReason: "SAFETY" } }, + }), + ), + ).toMatchObject({ + finish: "content-filter", + rawFinish: "SAFETY", + providerState: { promptFeedback: { blockReason: "SAFETY" } }, + }) +}) + +test("failed steps only override the assistant finish for content filters", () => { + const decode = Schema.decodeUnknownSync(SessionEvent.Step.Failed.data) + const input = { + sessionID: "ses_terminal", + assistantMessageID: "msg_terminal", + error: { type: "provider.content-filter", message: "Blocked" }, + } + + expect(decode(input)).toMatchObject(input) + expect(decode({ ...input, finish: "content-filter", rawFinish: "SAFETY" })).toMatchObject({ + finish: "content-filter", + rawFinish: "SAFETY", + }) + expect(() => decode({ ...input, finish: "stop" })).toThrow() +})