diff --git a/packages/client/src/effect/api/api.ts b/packages/client/src/effect/api/api.ts index 158e0332b4a..a6e4e63ad04 100644 --- a/packages/client/src/effect/api/api.ts +++ b/packages/client/src/effect/api/api.ts @@ -987,6 +987,17 @@ export type ExperimentalSessionLogOutput = readonly started: number } } + | { + readonly id: Event.ID + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } | undefined + readonly type: "session.step.first-token" + readonly durable: { readonly aggregateID: string; readonly seq: Event.Seq; readonly version: Event.Version } + readonly location?: + | { readonly directory: AbsolutePath; readonly workspaceID?: Workspace.ID | undefined } + | undefined + readonly data: { readonly sessionID: Session.ID; readonly assistantMessageID: SessionMessage.ID } + } | { readonly id: Event.ID readonly created: number @@ -2092,6 +2103,7 @@ export type EventSubscribeInput = { | "session.shell.started" | "session.shell.ended" | "session.step.started" + | "session.step.first-token" | "session.step.streamed" | "session.step.ended" | "session.step.failed" diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index bf6b3c9b625..1f039f6619c 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -1422,6 +1422,16 @@ export type SessionStepStarted = { } } +export type SessionStepFirstToken = { + id: EventID + created: number + metadata?: { [x: string]: any } + type: "session.step.first-token" + durable: { aggregateID: string; seq: EventSeq; version: 1 } + location?: LocationRef + data: { sessionID: SessionID; assistantMessageID: SessionMessageID } +} + export type SessionStepStreamed = { id: EventID created: number @@ -2700,7 +2710,7 @@ export type SessionInboxInfo = SessionInboxUser | SessionInboxSynthetic | Sessio export type SessionMessageAssistant = { id: SessionMessageID metadata?: { [x: string]: JsonValue } - time: { created: number; streamed?: number; completed?: number } + time: { created: number; firstToken?: number; streamed?: number; completed?: number } type: "assistant" agent: AgentID model: ModelRef @@ -2832,6 +2842,7 @@ export type SessionEventDurable = | SessionShellStarted | SessionShellEnded | SessionStepStarted + | SessionStepFirstToken | SessionStepStreamed | SessionStepEnded | SessionStepFailed @@ -2890,6 +2901,7 @@ export type V2Event = | SessionShellStarted | SessionShellEnded | SessionStepStarted + | SessionStepFirstToken | SessionStepStreamed | SessionStepEnded | SessionStepFailed @@ -4387,6 +4399,7 @@ export type EventSubscribeInput = { | "session.shell.started" | "session.shell.ended" | "session.step.started" + | "session.step.first-token" | "session.step.streamed" | "session.step.ended" | "session.step.failed" diff --git a/packages/client/src/solid/data.ts b/packages/client/src/solid/data.ts index d128dac68ef..a1c8805bc26 100644 --- a/packages/client/src/solid/data.ts +++ b/packages/client/src/solid/data.ts @@ -982,6 +982,7 @@ export function createData(config: CreateDataInput) { existing.rawFinish = undefined existing.native = undefined existing.time.created = event.data.started + existing.time.firstToken = undefined existing.time.streamed = undefined existing.time.completed = undefined if (event.data.snapshot) existing.snapshot = { ...existing.snapshot, start: event.data.snapshot } @@ -1009,6 +1010,11 @@ export function createData(config: CreateDataInput) { assistant.time.streamed = event.created }) return + case "session.step.first-token": + message.editAssistant(event.data.sessionID, event.data.assistantMessageID, (assistant) => { + assistant.time.firstToken = event.created + }) + return case "session.step.ended": { message.editAssistant(event.data.sessionID, event.data.assistantMessageID, (assistant) => { assistant.time.completed = event.created diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index 6297fe679e4..1e89db3af82 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -231,6 +231,7 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { draft.rawFinish = undefined draft.native = undefined draft.time.created = DateTime.makeUnsafe(event.data.started) + draft.time.firstToken = undefined draft.time.streamed = undefined draft.time.completed = undefined if (event.data.snapshot) draft.snapshot = { ...draft.snapshot, start: event.data.snapshot } @@ -260,6 +261,11 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { }), ) }), + "session.step.first-token": (event) => { + return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { + draft.time.firstToken = created + }) + }, "session.step.streamed": (event) => { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { draft.time.streamed = created diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 35f038b00a6..a66839ca9e4 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -1084,6 +1084,7 @@ const layer = Layer.effectDiscard( yield* touch(db, event) }), ) + yield* bus.project(SessionEvent.Step.FirstToken, (event) => run(db, event)) yield* bus.project(SessionEvent.Step.Streamed, (event) => run(db, event)) yield* bus.project(SessionEvent.Step.Ended, (event) => Effect.gen(function* () { diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 49ef5c7e22e..417650ef287 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -30,6 +30,14 @@ type Input = { const asRecord = (value: unknown): Record => (isRecord(value) ? value : { value }) +/** Provider events that carry a step's first generated token, matching how the stream counts output. */ +const beginsOutput = (event: LLMEvent) => + event.type === "text-start" || + event.type === "reasoning-start" || + event.type === "tool-input-start" || + event.type === "tool-input-error" || + event.type === "tool-call" + const INPUT_EXCERPT_LIMIT = 2048 type InputExcerpt = { @@ -142,6 +150,15 @@ export const createLLMEventPublisher = (bus: Pick, inp assistantMessageID: yield* startAssistant(), }) }) + let firstTokenPublished = false + const firstToken = Effect.fnUntraced(function* () { + if (firstTokenPublished) return + firstTokenPublished = true + yield* bus.publish(SessionEvent.Step.FirstToken, { + sessionID: input.sessionID, + assistantMessageID: yield* startAssistant(), + }) + }) const native = (metadata: ProviderMetadata | undefined) => metadata?.[input.providerMetadataKey] const fragments = ( name: string, @@ -413,6 +430,7 @@ export const createLLMEventPublisher = (bus: Pick, inp ) const publish = Effect.fnUntraced(function* (event: LLMEvent) { + if (beginsOutput(event)) yield* firstToken() switch (event.type) { case "step-start": yield* startAssistant() diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index b6c2ec6264a..2c2d3894239 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -247,6 +247,7 @@ describe("SessionRunnerLLM recorded", () => { "session.instructions.updated.2", "session.inbox.delivered.1", "session.step.started.1", + "session.step.first-token.1", "session.text.started.1", "session.text.ended.1", "session.step.streamed.1", diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index 57ace9e4531..c3dd41bb6c1 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -399,7 +399,7 @@ it.effect("publishes batched deltas before the next block starts", () => const types = () => published .map((event) => event.type) - .filter((type) => type !== "session.step.started.1" && type !== "session.step.streamed") + .filter((type) => type !== "session.step.started.1" && type !== "session.step.first-token.1") yield* Effect.forEach( [ LLMEvent.reasoningStart({ id: "reasoning" }), @@ -418,6 +418,8 @@ it.effect("publishes batched deltas before the next block starts", () => "session.text.delta", "session.tool.input.started.1", ]) + // The first generated token is recorded once, before any of the blocks it belongs to. + expect(published.filter((event) => event.type === "session.step.first-token.1")).toHaveLength(1) // Blocks stay open: later chunks still batch, and nothing is published twice. yield* publisher.publish(LLMEvent.textDelta({ id: "text", text: " more" })) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index d6bb961a25f..824986ac808 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -4229,6 +4229,10 @@ describe("SessionRunnerLLM", () => { expect(DateTime.toEpochMillis(assistant.time.streamed!) - DateTime.toEpochMillis(assistant.time.created)).toBe( 5_000, ) + // The tool call was the first output, recorded two seconds after dispatch and before the stream ended. + expect(DateTime.toEpochMillis(assistant.time.firstToken!) - DateTime.toEpochMillis(assistant.time.created)).toBe( + 2_000, + ) expect(assistant.time.completed).toBeUndefined() expect(assistant.content).toMatchObject([{ type: "tool", state: { status: "running" } }]) diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index 51608b6cf62..ee3a2393f75 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -346,6 +346,17 @@ export namespace Step { }) export type Started = typeof Started.Type + /** Records when the provider streamed the first output token, before the rest of the response. */ + export const FirstToken = Event.durable({ + type: "session.step.first-token", + ...options, + schema: { + ...Base, + assistantMessageID: SessionMessage.ID, + }, + }) + export type FirstToken = typeof FirstToken.Type + /** Records the provider response-body boundary independently of tool settlement. */ export const Streamed = Event.durable({ type: "session.step.streamed", @@ -679,6 +690,7 @@ export const Definitions = Event.inventory( Shell.Started, Shell.Ended, Step.Started, + Step.FirstToken, Step.Streamed, Step.Ended, Step.Failed, diff --git a/packages/schema/src/session-message.ts b/packages/schema/src/session-message.ts index 639ad0472ea..5c1755c45ec 100644 --- a/packages/schema/src/session-message.ts +++ b/packages/schema/src/session-message.ts @@ -239,6 +239,8 @@ export const Assistant = Schema.Struct({ retry: AssistantRetry.pipe(optional), time: Schema.Struct({ created: DateTimeUtcFromMillis, + /** When the provider streamed the first output token, before the rest of the response. */ + firstToken: DateTimeUtcFromMillis.pipe(optional), /** When the provider response body ended, before tool settlement. */ streamed: DateTimeUtcFromMillis.pipe(optional), completed: DateTimeUtcFromMillis.pipe(optional), diff --git a/packages/schema/test/event-manifest.test.ts b/packages/schema/test/event-manifest.test.ts index 70b3789fb8c..81053e74b83 100644 --- a/packages/schema/test/event-manifest.test.ts +++ b/packages/schema/test/event-manifest.test.ts @@ -134,6 +134,7 @@ describe("public event manifest", () => { "session.shell.started.1", "session.shell.ended.1", "session.step.started.1", + "session.step.first-token.1", "session.step.streamed.1", "session.step.ended.1", "session.step.failed.1", diff --git a/packages/tui/src/config/index.tsx b/packages/tui/src/config/index.tsx index 3ca1e0db087..4daecc030e8 100644 --- a/packages/tui/src/config/index.tsx +++ b/packages/tui/src/config/index.tsx @@ -172,7 +172,7 @@ export const Info = Schema.Struct({ description: "Show user attachment and tool-result images in the session transcript", }), tps: Schema.optional(Schema.Boolean).annotate({ - description: "Show average tokens per second", + description: "Show output tokens per second while the model streams", }), markdown: Schema.optional(Schema.Literals(["source", "rendered"])).annotate({ description: "Show Markdown syntax markers or conceal them in rendered transcript content", diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index e41af05a13e..fb3573801bc 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -2031,6 +2031,7 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) { const turn = () => ({ index: ctx.messageIndex(props.message.id), legacy: ctx.legacyTurns() }) const duration = createMemo(() => SessionTurn.duration(messages(), props.message, turn())) const tokensPerSecond = createMemo(() => SessionTurn.tokensPerSecond(messages(), props.message, turn())) + const timeToFirstToken = createMemo(() => SessionTurn.timeToFirstToken(messages(), props.message, turn())) const interrupted = createMemo(() => props.message.error?.message === "Step interrupted") const freeLimit = createMemo(() => props.message.error?.type === "provider.free-limit") const limitNotice = useLimitNotice(() => props.message) @@ -2065,6 +2066,9 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) { {(value) => · {value().toFixed(1)} tok/s} + = 36 && timeToFirstToken()}> + {(value) => · {Locale.duration(value())} ttft} + · interrupted diff --git a/packages/tui/test/app-lifecycle.test.tsx b/packages/tui/test/app-lifecycle.test.tsx index ae6f6415b50..dd45b4e6b81 100644 --- a/packages/tui/test/app-lifecycle.test.tsx +++ b/packages/tui/test/app-lifecycle.test.tsx @@ -770,7 +770,7 @@ test("keeps assistant footer metrics current after prepend, same-length refresh, type: "assistant", agent: session.agent, model: session.model, - time: { created: 2000, streamed: 3000, completed: 5000 }, + time: { created: 2000, firstToken: 2500, streamed: 3000, completed: 5000 }, finish: "stop", cost: 0, tokens: { ...session.tokens, output: 20 }, @@ -792,10 +792,10 @@ test("keeps assistant footer metrics current after prepend, same-length refresh, const initial = await waitForHighlightedFrame( setup, - (frame) => frame.includes("Original answer") && frame.includes("20.0 tok/s"), + (frame) => frame.includes("Original answer") && frame.includes("40.0 tok/s"), ) expect(initial).toContain("Current input") - expect(initial).toContain("4.0s \u00b7 20.0 tok/s") + expect(initial).toContain("4.0s \u00b7 40.0 tok/s \u00b7 500ms ttft") expect(initial).not.toContain("Prepended input") setup.mockInput.pressKey("g", { ctrl: true }) @@ -805,12 +805,12 @@ test("keeps assistant footer metrics current after prepend, same-length refresh, frame.includes("Prepended input") && frame.includes("Prepended instructions") && frame.includes("Original answer") && - frame.includes("20.0 tok/s") && + frame.includes("40.0 tok/s") && !frame.includes("Loading session history…"), ) expect(prepended).toContain("Current input") expect(prepended).toContain("Original answer") - expect(prepended).toContain("4.0s \u00b7 20.0 tok/s") + expect(prepended).toContain("4.0s \u00b7 40.0 tok/s \u00b7 500ms ttft") // Refresh the latest page: length stays four, but the retained assistant moves from index three to one. refresh = true @@ -826,7 +826,7 @@ test("keeps assistant footer metrics current after prepend, same-length refresh, ) expect(refreshed).toContain("Current input") expect(refreshed).toContain("Original answer") - expect(refreshed).toContain("4.0s \u00b7 20.0 tok/s") + expect(refreshed).toContain("4.0s \u00b7 40.0 tok/s \u00b7 500ms ttft") expect(refreshed).toContain("3.0s \u00b7 50.0 tok/s") setup.events.emit({ @@ -841,7 +841,7 @@ test("keeps assistant footer metrics current after prepend, same-length refresh, (frame) => frame.includes("Original answer") && !frame.includes("Later input") && !frame.includes("Later answer"), ) expect(reverted).toContain("Current input") - expect(reverted).toContain("4.0s \u00b7 20.0 tok/s") + expect(reverted).toContain("4.0s \u00b7 40.0 tok/s \u00b7 500ms ttft") expect(reverted).not.toContain("50.0 tok/s") }) diff --git a/packages/util/src/session-turn.test.ts b/packages/util/src/session-turn.test.ts index aa7828d4f64..734422e76db 100644 --- a/packages/util/src/session-turn.test.ts +++ b/packages/util/src/session-turn.test.ts @@ -4,7 +4,7 @@ import { SessionTurn } from "./session-turn.js" type Message = { id: string type: string - time: { created: number; streamed?: number; completed?: number } + time: { created: number; firstToken?: number; streamed?: number; completed?: number } tokens?: { output: number; reasoning: number } } @@ -15,12 +15,34 @@ test("measures turn duration from the user prompt across assistant steps", () => expect(SessionTurn.duration(messages, final)).toBe(29_000) }) +test("measures decode throughput from the first streamed token", () => { + const first = step("assistant-1", { created: 6_000, firstToken: 8_000, streamed: 10_000, completed: 20_000 }, 20, 5) + const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10) + const messages = [input("user-1", 1_000), first, final] + + // 65 tokens over the 5s the provider spent streaming them, not the 10s that include waiting for the first. + expect(SessionTurn.tokensPerSecond(messages, final)).toBe(13) + expect(SessionTurn.timeToFirstToken(messages, final)).toBe(2_000) +}) + +test("falls back to the request window for steps recorded without a first token", () => { + const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5) + const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10) + const messages = [input("user-1", 1_000), first, final] + + // The old step counts its 4s request window, the new one its 3s decode window. + expect(SessionTurn.tokensPerSecond(messages, final)).toBe(65 / 7) + // Only the step that recorded a first token can report how long it waited for one. + expect(SessionTurn.timeToFirstToken(messages, final)).toBe(1_000) +}) + test("measures request throughput including reasoning without tool time", () => { const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5) const final = step("assistant-2", { created: 24_000, streamed: 30_000, completed: 31_000 }, 30, 10) const messages = [input("user-1", 1_000), first, final] expect(SessionTurn.tokensPerSecond(messages, final)).toBe(6.5) + expect(SessionTurn.timeToFirstToken(messages, final)).toBeUndefined() first.time.streamed = undefined expect(SessionTurn.tokensPerSecond(messages, final)).toBeUndefined() }) diff --git a/packages/util/src/session-turn.ts b/packages/util/src/session-turn.ts index ffcb4407922..34c79f0182b 100644 --- a/packages/util/src/session-turn.ts +++ b/packages/util/src/session-turn.ts @@ -4,7 +4,13 @@ export * as SessionTurn from "./session-turn.js" export interface Message { readonly id: string readonly type: string - readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number } + readonly time: { + readonly created: number + /** When the provider streamed the first output token; steps recorded before this existed leave it unset. */ + readonly firstToken?: number + readonly streamed?: number + readonly completed?: number + } readonly tokens?: { readonly output: number; readonly reasoning: number } } @@ -27,14 +33,19 @@ export function duration(messages: readonly Message[], message: Message, options return Math.max(0, message.time.completed - (input?.time.created ?? message.time.created)) } -/** Output and reasoning tokens per second across the turn's steps up to `message`, or undefined until every step has streamed. */ +/** + * Output and reasoning tokens per second while the provider streamed them, or undefined until every step has streamed. + * + * Each step counts from its first streamed token to the end of the response body, so the wait for that first token + * (time to first token) is not divided into the output rate. Steps recorded before `time.firstToken` existed count + * from request dispatch instead, which is what this measured for them before the field existed. + */ export function tokensPerSecond(messages: readonly Message[], message: Message, options: Options = {}) { - const index = options.index ?? messages.findIndex((item) => item.id === message.id) - const end = index === -1 ? messages.length : index + 1 - const start = inputIndex(messages, end, options.legacy ?? legacy(messages)) - const steps = messages.slice(start + 1, end).filter((item) => item.type === "assistant") + const steps = turnSteps(messages, message, options) const durations = steps.flatMap((step) => - step.time.streamed === undefined ? [] : [Math.max(0, step.time.streamed - step.time.created)], + step.time.streamed === undefined + ? [] + : [Math.max(0, step.time.streamed - (step.time.firstToken ?? step.time.created))], ) if (steps.length === 0 || durations.length !== steps.length) return const output = steps.reduce((total, step) => total + (step.tokens?.output ?? 0) + (step.tokens?.reasoning ?? 0), 0) @@ -44,10 +55,27 @@ export function tokensPerSecond(messages: readonly Message[], message: Message, return output / (total / 1_000) } +/** + * Milliseconds from the dispatch of the turn's first token-producing step to its first streamed token, or undefined + * until a step records one. + */ +export function timeToFirstToken(messages: readonly Message[], message: Message, options: Options = {}) { + const step = turnSteps(messages, message, options).find((item) => item.time.firstToken !== undefined) + if (step?.time.firstToken === undefined) return + return Math.max(0, step.time.firstToken - step.time.created) +} + export function legacy(messages: readonly Message[]) { return !messages.some((message) => message.type === "idle") } +function turnSteps(messages: readonly Message[], message: Message, options: Options) { + const index = options.index ?? messages.findIndex((item) => item.id === message.id) + const end = index === -1 ? messages.length : index + 1 + const start = inputIndex(messages, end, options.legacy ?? legacy(messages)) + return messages.slice(start + 1, end).filter((item) => item.type === "assistant") +} + function inputIndex(messages: readonly Message[], end: number, legacy: boolean) { // Reading a sliced prefix subscribes every caller to unrelated historical messages, so walk // back only as far as the turn boundary: the nearest input in legacy sessions, otherwise the