feat(tui): show LLM token throughput

This commit is contained in:
Dax
2026-08-26 09:23:53 -04:00
committed by GitHub
Unverified
parent e82aa92e64
commit 91028a690b
19 changed files with 180 additions and 8 deletions
+9
View File
@@ -663,6 +663,15 @@ export type SessionLogOutput =
readonly snapshot?: (string & Brand.Brand<"Snapshot.ID">) | undefined
}
}
| {
readonly id: Event.ID
readonly created: number
readonly metadata?: { readonly [x: string]: unknown } | undefined
readonly type: "session.step.streamed"
readonly durable: { readonly aggregateID: string; readonly seq: Event.Seq; readonly version: Event.Version }
readonly location?: Location.Ref | undefined
readonly data: { readonly sessionID: Session.ID; readonly assistantMessageID: SessionMessage.ID }
}
| {
readonly id: Event.ID
readonly created: number
+16 -4
View File
@@ -720,6 +720,16 @@ export type SessionStepStarted = {
data: { sessionID: string; assistantMessageID: string; agent: string; model: ModelRef; snapshot?: string }
}
export type SessionStepStreamed = {
id: string
created: number
metadata?: { [x: string]: any }
type: "session.step.streamed"
durable: { aggregateID: string; seq: number; version: 1 }
location?: LocationRef
data: { sessionID: string; assistantMessageID: string }
}
export type SessionTextStarted = {
id: string
created: number
@@ -2073,7 +2083,7 @@ export type SessionInboxEnqueued = {
export type SessionMessageAssistant = {
id: string
metadata?: { [x: string]: JsonValue }
time: { created: number; completed?: number }
time: { created: number; streamed?: number; completed?: number }
type: "assistant"
agent: string
model: ModelRef
@@ -2168,6 +2178,7 @@ export type SessionEventDurable =
| SessionShellStarted
| SessionShellEnded
| SessionStepStarted
| SessionStepStreamed
| SessionStepEnded
| SessionStepFailed
| SessionTextStarted
@@ -2227,6 +2238,7 @@ export type V2Event =
| SessionShellStarted
| SessionShellEnded
| SessionStepStarted
| SessionStepStreamed
| SessionStepEnded
| SessionStepFailed
| SessionTextStarted
@@ -2851,7 +2863,7 @@ export type SessionImportInput = {
| {
readonly id: string
readonly metadata?: { readonly [x: string]: JsonValue }
readonly time: { readonly created: number; readonly completed?: number }
readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number }
readonly type: "assistant"
readonly agent: string
readonly model: { readonly id: string; readonly providerID: string; readonly variant?: string }
@@ -3127,7 +3139,7 @@ export type SessionImportInput = {
| {
readonly id: string
readonly metadata?: { readonly [x: string]: JsonValue }
readonly time: { readonly created: number; readonly completed?: number }
readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number }
readonly type: "assistant"
readonly agent: string
readonly model: { readonly id: string; readonly providerID: string; readonly variant?: string }
@@ -3403,7 +3415,7 @@ export type SessionImportInput = {
| {
readonly id: string
readonly metadata?: { readonly [x: string]: JsonValue }
readonly time: { readonly created: number; readonly completed?: number }
readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number }
readonly type: "assistant"
readonly agent: string
readonly model: { readonly id: string; readonly providerID: string; readonly variant?: string }
+7
View File
@@ -750,6 +750,7 @@ export function createData(config: CreateDataInput) {
existing.finish = undefined
existing.rawFinish = undefined
existing.providerState = undefined
existing.time.streamed = undefined
existing.time.completed = undefined
if (event.data.snapshot) existing.snapshot = { ...existing.snapshot, start: event.data.snapshot }
return
@@ -771,6 +772,12 @@ export function createData(config: CreateDataInput) {
})
})
return
case "session.step.streamed":
message.update(event.data.sessionID, (draft, index) => {
const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID)
if (currentAssistant) currentAssistant.time.streamed = event.created
})
return
case "session.step.ended": {
message.update(event.data.sessionID, (draft, index) => {
const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID)
@@ -210,6 +210,7 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
draft.finish = undefined
draft.rawFinish = undefined
draft.providerState = undefined
draft.time.streamed = undefined
draft.time.completed = undefined
if (event.data.snapshot) draft.snapshot = { ...draft.snapshot, start: event.data.snapshot }
}),
@@ -238,6 +239,11 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
}),
)
}),
"session.step.streamed": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
draft.time.streamed = created
})
},
"session.step.ended": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
draft.time.completed = created
+1
View File
@@ -660,6 +660,7 @@ const layer = Layer.effectDiscard(
yield* bus.project(SessionEvent.Shell.Started, (event) => run(db, event))
yield* bus.project(SessionEvent.Shell.Ended, (event) => run(db, event))
yield* bus.project(SessionEvent.Step.Started, (event) => run(db, event))
yield* bus.project(SessionEvent.Step.Streamed, (event) => run(db, event))
yield* bus.project(SessionEvent.Step.Ended, (event) =>
Effect.gen(function* () {
yield* run(db, event)
+1
View File
@@ -489,6 +489,7 @@ const layer = Layer.effect(
// Note: Exit.hasInterrupts is a type guard whose false branch unsoundly narrows
// away non-interrupt failures, so both interrupt checks stay Cause-based.
const streamInterrupted = stream._tag === "Failure" && Cause.hasInterrupts(stream.cause)
if (!overflowFailure && publisher.hasStarted()) yield* publisher.streamed()
// Join every owned tool run first: await all exits, not just the first failure.
// Afterwards no fiber is alive, settlement is the only writer, and the record
@@ -103,6 +103,7 @@ export const createLLMEventPublisher = (bus: Pick<Bus.Interface, "publish">, inp
let stepFailed = false
let providerFailed = false
let outputStarted = false
let stepStreamed = false
let stepFailure: SessionError.Error | undefined
let stepSettlement: StepRecord["finish"]
@@ -120,6 +121,14 @@ export const createLLMEventPublisher = (bus: Pick<Bus.Interface, "publish">, inp
})
const currentAssistantMessageID = () =>
stepStarted ? Effect.succeed(assistantMessageID) : Effect.die(new Error("Tool event before assistant step start"))
const streamed = Effect.fnUntraced(function* () {
if (stepStreamed) return
stepStreamed = true
yield* bus.publish(SessionEvent.Step.Streamed, {
sessionID: input.sessionID,
assistantMessageID: yield* startAssistant(),
})
})
const providerState = (metadata: ProviderMetadata | undefined) => metadata?.[input.providerMetadataKey]
const fragments = (
name: string,
@@ -583,6 +592,7 @@ export const createLLMEventPublisher = (bus: Pick<Bus.Interface, "publish">, inp
publishStepFailure,
failUnsettledTools,
hasProviderError: () => providerFailed,
hasStarted: () => stepStarted,
/** Immutable snapshot of everything recorded for this step so far. */
record: (): StepRecord => ({
outputStarted,
@@ -598,6 +608,7 @@ export const createLLMEventPublisher = (bus: Pick<Bus.Interface, "publish">, inp
})),
}),
startAssistant,
streamed,
assistantMessageID: assistantMessageIDForTool,
}
}
+5 -1
View File
@@ -784,6 +784,10 @@ describe("SessionProjector", () => {
.pipe(Effect.orDie)
const service = yield* Bus.Service
yield* service.publish(SessionEvent.Step.Streamed, {
sessionID,
assistantMessageID: endedID,
})
yield* service.publish(SessionEvent.Step.Ended, {
sessionID,
assistantMessageID: endedID,
@@ -824,7 +828,7 @@ describe("SessionProjector", () => {
cost: Money.USD.make(1),
tokens: { input: 2, output: 3, reasoning: 4, cache: { read: 5, write: 6 } },
snapshot: { end: "snap_ended", files: ["src/ended.ts"] },
time: { completed: created },
time: { streamed: created, completed: created },
})
expect(messages[1]).toMatchObject({
type: "assistant",
@@ -241,6 +241,7 @@ describe("SessionRunnerLLM recorded", () => {
"session.step.started.1",
"session.text.started.1",
"session.text.ended.1",
"session.step.streamed.1",
"session.step.ended.1",
])
}),
+29
View File
@@ -2926,6 +2926,35 @@ describe("SessionRunnerLLM", () => {
}),
)
it.effect("records the stream boundary before local tools complete", () =>
Effect.gen(function* () {
const session = yield* setup
const bus = yield* Bus.Service
yield* admit(session, "Echo this")
yield* TestLLM.push(TestLLM.tool("call-streamed", "echo", { text: "hello" }), TestLLM.stop())
const tools = yield* blockTools()
const streamed = yield* bus
.subscribe(SessionEvent.Step.Streamed)
.pipe(
Stream.filter((event) => event.data.sessionID === sessionID),
Stream.take(1),
Stream.runDrain,
Effect.forkScoped({ startImmediately: true }),
)
const run = yield* Effect.forkChild(session.resume(sessionID))
yield* tools.started
yield* Fiber.join(streamed)
const assistant = requireAssistant(yield* session.context(sessionID))
expect(assistant.time.streamed).toBeDefined()
expect(assistant.time.completed).toBeUndefined()
expect(assistant.content).toMatchObject([{ type: "tool", state: { status: "running" } }])
yield* tools.release
yield* Fiber.join(run)
}),
)
it.effect("restores durable reasoning provider metadata in the next request", () =>
Effect.gen(function* () {
const session = yield* setup
+12
View File
@@ -316,6 +316,17 @@ export namespace Step {
})
export type Started = typeof Started.Type
/** Records the provider response-body boundary independently of tool settlement. */
export const Streamed = Event.durable({
type: "session.step.streamed",
...options,
schema: {
...Base,
assistantMessageID: SessionMessage.ID,
},
})
export type Streamed = typeof Streamed.Type
export const Ended = Event.durable({
type: "session.step.ended",
...options,
@@ -628,6 +639,7 @@ export const Definitions = Event.inventory(
Shell.Started,
Shell.Ended,
Step.Started,
Step.Streamed,
Step.Ended,
Step.Failed,
Text.Started,
+2
View File
@@ -228,6 +228,8 @@ export const Assistant = Schema.Struct({
retry: AssistantRetry.pipe(optional),
time: Schema.Struct({
created: DateTimeUtcFromMillis,
/** When the provider response body ended, before tool settlement. */
streamed: DateTimeUtcFromMillis.pipe(optional),
completed: DateTimeUtcFromMillis.pipe(optional),
}),
}).annotate({ identifier: "Session.Message.Assistant" })
@@ -135,6 +135,7 @@ describe("public event manifest", () => {
"session.shell.started.1",
"session.shell.ended.1",
"session.step.started.1",
"session.step.streamed.1",
"session.step.ended.1",
"session.step.failed.1",
"session.text.started.1",
@@ -93,6 +93,15 @@ export const settings: Setting[] = [
labels: ["off", "on"],
keywords: ["attachments", "images", "tool output"],
},
{
title: "TPS",
category: "Session",
path: ["session", "tps"],
default: true,
values: [false, true],
labels: ["off", "on"],
keywords: ["tokens per second", "throughput"],
},
{
title: "New session location",
category: "Session",
+6 -1
View File
@@ -136,6 +136,9 @@ export const Info = Schema.Struct({
image_preview: Schema.optional(Schema.Boolean).annotate({
description: "Show user attachment and tool-result images in the session transcript",
}),
tps: Schema.optional(Schema.Boolean).annotate({
description: "Show output tokens per second in assistant footers",
}),
markdown: Schema.optional(Schema.Literals(["source", "rendered"])).annotate({
description: "Show Markdown syntax markers or conceal them in rendered transcript content",
}),
@@ -219,8 +222,9 @@ export type Resolved = Omit<Info, "attention" | "cursor" | "keybinds" | "leader"
style: "block" | "underline" | "line" | "default"
blinking: boolean
}
session: Omit<NonNullable<Info["session"]>, "new_location"> & {
session: Omit<NonNullable<Info["session"]>, "new_location" | "tps"> & {
new_location: "launch" | "inherit"
tps: boolean
}
tabs: {
enabled: boolean
@@ -265,6 +269,7 @@ export function resolve(input: Info, options: { terminalSuspend: boolean }): Res
session: {
...input.session,
new_location: input.session?.new_location ?? "launch",
tps: input.session?.tps ?? true,
},
tabs: {
...input.tabs,
+8 -1
View File
@@ -93,6 +93,7 @@ import {
messageBoundaryIDs,
resolvePart,
turnDuration,
turnTokensPerSecond,
type CacheUsage,
type PartRef,
type SessionRow,
@@ -1894,6 +1895,7 @@ function SessionGroupView(props: {
function AssistantFooter(props: { message: SessionMessageAssistant }) {
const ctx = use()
const config = useConfig()
const data = useData()
const local = useLocal()
const theme = useTheme("elevated")
@@ -1904,7 +1906,9 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) {
.find((model) => model.providerID === props.message.model.providerID && model.id === props.message.model.id)
?.name ?? `${props.message.model.providerID}/${props.message.model.id}`,
)
const duration = createMemo(() => turnDuration(props.message, data.session.message.list(ctx.sessionID)))
const messages = createMemo(() => data.session.message.list(ctx.sessionID))
const duration = createMemo(() => turnDuration(props.message, messages()))
const tokensPerSecond = createMemo(() => turnTokensPerSecond(props.message, messages()))
const interrupted = createMemo(() => props.message.error?.message === "Step interrupted")
return (
<>
@@ -1925,6 +1929,9 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) {
<Show when={duration() && (ctx.terminal.width < 28 || ctx.terminal.width >= 36)}>
<span style={{ fg: theme.text.subdued }}> · {Locale.duration(duration())}</span>
</Show>
<Show when={config.data.session.tps && tokensPerSecond()}>
{(value) => <span style={{ fg: theme.text.subdued }}> · {value().toFixed(1)} tok/s</span>}
</Show>
<Show when={interrupted()}>
<span style={{ fg: theme.text.subdued }}> · interrupted</span>
</Show>
+19
View File
@@ -353,6 +353,25 @@ export function turnDuration(message: SessionMessageAssistant, messages: Session
return Math.max(0, message.time.completed - (input?.time.created ?? message.time.created))
}
export function turnTokensPerSecond(message: SessionMessageAssistant, messages: SessionMessageInfo[]) {
const index = messages.findIndex((item) => item.id === message.id)
const end = index === -1 ? messages.length : index + 1
const start = messages
.slice(0, end)
.findLastIndex((item) => item.type === "user" || item.type === "synthetic")
const steps = messages
.slice(start + 1, end)
.filter((item): item is SessionMessageAssistant => item.type === "assistant")
const durations = steps.flatMap((step) =>
step.time.streamed === undefined ? [] : [Math.max(0, step.time.streamed - step.time.created)],
)
if (steps.length === 0 || durations.length !== steps.length) return
const output = steps.reduce((total, step) => total + (step.tokens?.output ?? 0), 0)
const duration = durations.reduce((total, value) => total + value, 0)
if (output <= 0 || duration <= 0) return
return output / (duration / 1_000)
}
function hasTokenUsage(
message: SessionMessageAssistant,
): message is SessionMessageAssistant & { tokens: NonNullable<SessionMessageAssistant["tokens"]> } {
+29 -1
View File
@@ -1,6 +1,12 @@
import { expect, test } from "bun:test"
import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client"
import { cacheReuseDrop, messageBoundaryIDs, reduceSessionRows, turnDuration } from "../../../src/routes/session/rows"
import {
cacheReuseDrop,
messageBoundaryIDs,
reduceSessionRows,
turnDuration,
turnTokensPerSecond,
} from "../../../src/routes/session/rows"
test("measures turn duration from the user prompt across assistant steps", () => {
const first = assistant("assistant-1", [])
@@ -16,6 +22,28 @@ test("measures turn duration from the user prompt across assistant steps", () =>
expect(turnDuration(final, messages)).toBe(29_000)
})
test("measures turn output throughput across model steps without tool time", () => {
const first = assistant("assistant-1", [])
first.time = { created: 8_000, streamed: 10_000, completed: 20_000 }
first.tokens = { input: 10, output: 20, reasoning: 5, cache: { read: 0, write: 0 } }
const final = assistant("assistant-2", [])
final.time = { created: 27_000, streamed: 30_000, completed: 31_000 }
final.tokens = { input: 20, output: 30, reasoning: 10, cache: { read: 0, write: 0 } }
const messages: SessionMessageInfo[] = [
{ type: "user", id: "user-1", text: "Question", time: { created: 1_000 } },
first,
final,
]
expect(turnTokensPerSecond(final, messages)).toBe(10)
})
test("omits turn throughput when a stream boundary is unavailable", () => {
const final = assistant("assistant-1", [])
final.tokens = { input: 10, output: 20, reasoning: 0, cache: { read: 0, write: 0 } }
expect(turnTokensPerSecond(final, [final])).toBeUndefined()
})
test("filters OpenAI cache quantization from cache reuse drops", () => {
const openai = { id: "gpt", providerID: "openai" }
expect(cacheReuseDrop(undefined, { read: 10_000, model: openai })).toBeUndefined()
+8
View File
@@ -27,6 +27,7 @@ test("validates the session tabs setting", () => {
expect(() => decode({ tabs: { enabled: "on" } })).toThrow()
expect(decode({ prompt: { image_preview: true } })).toEqual({ prompt: { image_preview: true } })
expect(decode({ session: { image_preview: true } })).toEqual({ session: { image_preview: true } })
expect(decode({ session: { tps: false } })).toEqual({ session: { tps: false } })
expect(decode({ session: { new_location: "inherit" } })).toEqual({ session: { new_location: "inherit" } })
expect(() => decode({ session: { new_location: "current" } })).toThrow()
})
@@ -50,6 +51,7 @@ test("resolves nested config and keybind defaults", () => {
expect(config.debug).toEqual({ devtools: true })
expect(config.tabs).toEqual({ enabled: true, scope: "cwd", layout: "horizontal" })
expect(config.session.new_location).toBe("launch")
expect(config.session.tps).toBe(true)
})
test("shows resolved tab defaults in settings", () => {
@@ -62,6 +64,12 @@ test("shows the new session location default in settings", () => {
expect(settings.find((setting) => setting.path.join(".") === "session.new_location")?.default).toBe("launch")
})
test("shows the TPS default in session settings", () => {
const setting = settings.find((setting) => setting.path.join(".") === "session.tps")
expect(setting?.category).toBe("Session")
expect(setting?.default).toBe(true)
})
test("validates terminal copy behavior", () => {
expect(decodeInfo({ terminal: { copy: "manual" } })).toEqual({ terminal: { copy: "manual" } })
expect(decodeInfo({ terminal: { copy: "select" } })).toEqual({ terminal: { copy: "select" } })