From d4303a9ca55e36517a3143f86e1ba7a1f3d6a466 Mon Sep 17 00:00:00 2001 From: Luke Parker <10430890+Hona@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:51:08 +1000 Subject: [PATCH] feat(app): show provider metrics in debug bar (#49674) --- packages/app/src/runtime/i18n/en.ts | 10 + packages/app/src/shell/debug/debug-bar.tsx | 286 ++++++++++++------ .../src/shell/debug/provider-metrics.test.ts | 153 ++++++++++ .../app/src/shell/debug/provider-metrics.ts | 135 +++++++++ packages/app/src/shell/shell.tsx | 31 +- 5 files changed, 516 insertions(+), 99 deletions(-) create mode 100644 packages/app/src/shell/debug/provider-metrics.test.ts create mode 100644 packages/app/src/shell/debug/provider-metrics.ts diff --git a/packages/app/src/runtime/i18n/en.ts b/packages/app/src/runtime/i18n/en.ts index 6ec50e7b13d..f3fa218fc1b 100644 --- a/packages/app/src/runtime/i18n/en.ts +++ b/packages/app/src/runtime/i18n/en.ts @@ -74,6 +74,7 @@ export const dict = { "command.category.workspace": "Worktree", "command.category.settings": "Settings", "command.logs.export": "Export logs", + "command.debugBar.toggle": "Toggle debug bar", "theme.scheme.system": "System", "theme.scheme.light": "Light", @@ -966,7 +967,16 @@ export const dict = { "sidebar.empty.description": "Open a project to get started", "debugBar.ariaLabel": "Development performance diagnostics", + "debugBar.providerAriaLabel": "Provider performance diagnostics", "debugBar.na": "n/a", + "debugBar.ttft.label": "TTFT", + "debugBar.ttft.tip": "Time from provider request dispatch to the first model output.", + "debugBar.ttfa.label": "TTFA", + "debugBar.ttfa.tip": "Time from provider request dispatch to the first answer text.", + "debugBar.tps.label": "TPS", + "debugBar.tps.tip": "Output tokens per second after the first model output.", + "debugBar.e2e.label": "E2E", + "debugBar.e2e.tip": "Time from provider request dispatch until its response stream ended.", "debugBar.nav.label": "NAV", "debugBar.nav.tip": "Last completed route transition touching a session page, measured from router start until the first paint after it settles.", diff --git a/packages/app/src/shell/debug/debug-bar.tsx b/packages/app/src/shell/debug/debug-bar.tsx index 3880c4349e9..9930998cf8d 100644 --- a/packages/app/src/shell/debug/debug-bar.tsx +++ b/packages/app/src/shell/debug/debug-bar.tsx @@ -1,10 +1,20 @@ -import { useIsRouting, useLocation } from "@solidjs/router" -import { batch, createEffect, onCleanup, onMount, Show } from "solid-js" +import { useIsRouting, useLocation, useParams } from "@solidjs/router" +import { batch, createEffect, createMemo, on, onCleanup, onMount, Show } from "solid-js" import { createStore } from "solid-js/store" import { makeEventListener } from "@solid-primitives/event-listener" import { Tooltip } from "@opencode/ui/tooltip" import { useLanguage } from "@/runtime/i18n/language" import { usePlatform } from "@/runtime/platform/platform" +import { useGlobal } from "@/runtime/server/runtime" +import { ServerConnection } from "@/runtime/server/registry" +import { base64Encode } from "@opencode/util/encode" +import { + applyProviderMetricEvent, + isProviderMetricEvent, + projectedProviderMetrics, + type ProviderMetrics, + type ProviderMetricState, +} from "./provider-metrics" type Mem = Performance & { memory?: { @@ -39,12 +49,23 @@ const time = (n?: number) => { return `${Math.round(n)}` } +const fixed = (n?: number, digits = 0) => { + if (n === undefined || Number.isNaN(n)) return + return n.toFixed(digits) +} + const mb = (n?: number) => { if (n === undefined || Number.isNaN(n)) return const v = n / 1024 / 1024 return `${v >= 1024 ? v.toFixed(0) : v.toFixed(1)}MB` } +const duration = (n?: number) => { + if (n === undefined || Number.isNaN(n)) return + if (n < 1_000) return `${Math.round(n)}ms` + return `${(n / 1_000).toFixed(n < 10_000 ? 1 : 0)}s` +} + const bad = (n: number | undefined, limit: number, low = false) => { if (n === undefined || Number.isNaN(n)) return false return low ? n < limit : n > limit @@ -80,6 +101,7 @@ function Cell(props: { }} >
- {props.label} - {props.value} + + {props.label} + + + {props.value} + ) @@ -149,9 +176,11 @@ function ToggleCell(props: { ) } -export function DebugBar(props: { inline?: boolean } = {}) { +export function DebugBar(props: { diagnostics?: boolean; inline?: boolean } = {}) { const language = useLanguage() const platform = usePlatform() + const global = useGlobal() + const params = useParams<{ serverKey?: string; id?: string }>() const location = useLocation() const routing = useIsRouting() const [state, setState] = createStore({ @@ -175,8 +204,55 @@ export function DebugBar(props: { inline?: boolean } = {}) { dur: undefined as number | undefined, pending: false, }, + live: undefined as ProviderMetrics | undefined, }) + const target = createMemo( + () => { + if (!params.serverKey || !params.id) return + const connection = global.servers + .list() + .find((item) => base64Encode(ServerConnection.key(item)) === params.serverKey) + if (!connection) return + return { ctx: global.ensureServerCtx(connection), id: params.id } + }, + undefined, + { equals: (a, b) => a?.ctx === b?.ctx && a?.id === b?.id }, + ) + // History comes from the already-loaded message projection; live requests refine it in place. + const projected = createMemo(() => { + const current = target() + if (!current) return + return projectedProviderMetrics(current.ctx.data.session.message.list(current.id)) + }) + const metrics = () => state.live ?? projected() + + // Missed events during an outage are never replayed; the refreshed projection must win. + createEffect( + on( + () => target()?.ctx.sdk.connection.status(), + (status) => { + if (status !== "connected") setState("live", undefined) + }, + { defer: true }, + ), + ) + + createEffect( + on(target, (current) => { + setState("live", undefined) + if (!current) return + const accumulator: ProviderMetricState = {} + onCleanup( + current.ctx.sdk.event.listen((event) => { + if (!isProviderMetricEvent(event) || event.data.sessionID !== current.id) return + applyProviderMetricEvent(accumulator, event) + if (accumulator.latest) setState("live", accumulator.latest) + }), + ) + }), + ) + const na = () => language.t("debugBar.na").toUpperCase() const heap = () => (state.heap.limit ? (state.heap.used ?? 0) / state.heap.limit : undefined) const heapv = () => { @@ -204,6 +280,7 @@ export function DebugBar(props: { inline?: boolean } = {}) { let two = 0 createEffect(() => { + if (!props.diagnostics) return const busy = routing() const next = `${location.pathname}${location.search}` @@ -248,6 +325,7 @@ export function DebugBar(props: { inline?: boolean } = {}) { }) onMount(() => { + if (!props.diagnostics) return const obs: PerformanceObserver[] = [] const fps: Array<{ at: number; dur: number }> = [] const long: Array<{ at: number; dur: number }> = [] @@ -448,7 +526,7 @@ export function DebugBar(props: { inline?: boolean } = {}) { return (
diff --git a/packages/app/src/shell/debug/provider-metrics.test.ts b/packages/app/src/shell/debug/provider-metrics.test.ts new file mode 100644 index 00000000000..c641018720b --- /dev/null +++ b/packages/app/src/shell/debug/provider-metrics.test.ts @@ -0,0 +1,153 @@ +import { expect, test } from "bun:test" +import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode/client/promise" +import type { ProviderMetricEvent } from "./provider-metrics" +import { foldProviderMetrics, projectedProviderMetrics } from "./provider-metrics" + +const durable = { aggregateID: "ses_test", seq: 0, version: 1 } as const + +const events: ProviderMetricEvent[] = [ + { + id: "evt_started", + created: 1_000, + type: "session.step.started", + durable, + data: { + sessionID: "ses_test", + assistantMessageID: "msg_assistant", + agent: "build", + model: { id: "model", providerID: "provider" }, + started: 1_000, + }, + }, + { + id: "evt_reasoning", + created: 1_300, + type: "session.reasoning.started", + durable: { ...durable, seq: 1 }, + data: { sessionID: "ses_test", assistantMessageID: "msg_assistant", ordinal: 0 }, + }, + { + id: "evt_text", + created: 1_800, + type: "session.text.started", + durable: { ...durable, seq: 2 }, + data: { sessionID: "ses_test", assistantMessageID: "msg_assistant", ordinal: 0 }, + }, + { + id: "evt_streamed", + created: 3_800, + type: "session.step.streamed", + durable: { ...durable, seq: 3 }, + data: { sessionID: "ses_test", assistantMessageID: "msg_assistant" }, + }, + { + id: "evt_ended", + created: 4_000, + type: "session.step.ended", + durable: { ...durable, seq: 4 }, + data: { + sessionID: "ses_test", + assistantMessageID: "msg_assistant", + finish: "stop", + cost: 0, + tokens: { input: 200, output: 100, reasoning: 20, cache: { read: 0, write: 0 } }, + }, + }, +] + +test("calculates provider response metrics from durable events", () => { + expect(foldProviderMetrics(events)).toEqual({ + tps: 50, + ttft: 300, + ttfa: 800, + e2e: 2_800, + }) +}) + +test("ignores failed attempts without usage", () => { + expect( + foldProviderMetrics([ + ...events.slice(0, 4), + { + id: "evt_failed", + created: 4_000, + type: "session.step.failed", + durable: { ...durable, seq: 4 }, + data: { + sessionID: "ses_test", + assistantMessageID: "msg_assistant", + error: { type: "aborted", message: "Step interrupted" }, + }, + }, + ]), + ).toBeUndefined() +}) + +test("keeps completed metrics while the next provider attempt runs", () => { + expect( + foldProviderMetrics([ + ...events, + { + id: "evt_retry", + created: 5_000, + type: "session.step.started", + durable: { ...durable, seq: 5 }, + data: { + sessionID: "ses_test", + assistantMessageID: "msg_assistant", + agent: "build", + model: { id: "model", providerID: "provider" }, + started: 5_000, + }, + }, + ]), + ).toEqual(foldProviderMetrics(events)) +}) + +const assistant: SessionMessageAssistant = { + id: "msg_assistant", + type: "assistant", + agent: "build", + model: { id: "model", providerID: "provider" }, + content: [ + { type: "reasoning", text: "Think", time: { created: 1_300, completed: 1_700 } }, + { type: "text", text: "Answer" }, + ], + tokens: { input: 200, output: 100, reasoning: 20, cache: { read: 0, write: 0 } }, + time: { created: 1_000, streamed: 3_800, completed: 4_000 }, +} + +test("derives a baseline from the latest completed projected request", () => { + const messages: SessionMessageInfo[] = [ + { id: "msg_user", type: "user", text: "Hi", time: { created: 1 } }, + assistant, + { ...assistant, id: "msg_running", tokens: undefined, time: { created: 5_000 } }, + ] + // Reasoning ended at 1_700, so TPS spans 1_700 → 3_800 = 100 / 2.1s. + expect(projectedProviderMetrics(messages)).toEqual({ tps: 100 / 2.1, ttft: 300, ttfa: 700, e2e: 2_800 }) +}) + +const tool = (created: number): SessionMessageAssistant["content"][number] => ({ + type: "tool", + id: "call_1", + name: "read", + state: { status: "running", input: {}, metadata: {} }, + time: { created, ran: created + 100 }, +}) + +test("leaves text-first history unavailable until a live request", () => { + const unavailable = { tps: undefined, ttft: undefined, ttfa: undefined, e2e: 2_800 } + expect(projectedProviderMetrics([{ ...assistant, content: [{ type: "text", text: "Answer" }] }])).toEqual(unavailable) + expect( + projectedProviderMetrics([{ ...assistant, content: [{ type: "text", text: "Answer" }, tool(2_500)] }]), + ).toEqual(unavailable) +}) + +test("uses the first tool call as first output for tool-first history", () => { + expect(projectedProviderMetrics([{ ...assistant, content: [tool(1_800)] }])).toEqual({ + tps: 50, + ttft: 800, + ttfa: undefined, + e2e: 2_800, + }) +}) diff --git a/packages/app/src/shell/debug/provider-metrics.ts b/packages/app/src/shell/debug/provider-metrics.ts new file mode 100644 index 00000000000..aec37b87dd4 --- /dev/null +++ b/packages/app/src/shell/debug/provider-metrics.ts @@ -0,0 +1,135 @@ +import type { + SessionLogItem, + SessionMessageAssistant, + SessionMessageInfo, + TokenUsageInfo, +} from "@opencode/client/promise" + +type ProviderMetricEventType = + | "session.step.started" + | "session.step.streamed" + | "session.step.ended" + | "session.step.failed" + | "session.text.started" + | "session.reasoning.started" + | "session.tool.input.started" + +const types: ReadonlySet = new Set([ + "session.step.started", + "session.step.streamed", + "session.step.ended", + "session.step.failed", + "session.text.started", + "session.reasoning.started", + "session.tool.input.started", +]) + +export type ProviderMetricEvent = Extract + +export type ProviderMetrics = { + tps?: number + ttft?: number + ttfa?: number + e2e?: number +} + +type Attempt = { + assistantMessageID: string + started: number + first?: number + answer?: number + streamed?: number + tokens?: TokenUsageInfo +} + +export type ProviderMetricState = { attempt?: Attempt; latest?: ProviderMetrics } + +export function isProviderMetricEvent(event: { type: string }): event is ProviderMetricEvent { + return types.has(event.type) +} + +export function applyProviderMetricEvent(state: ProviderMetricState, event: ProviderMetricEvent) { + if (event.type === "session.step.started") { + state.attempt = { + assistantMessageID: event.data.assistantMessageID, + started: event.data.started, + } + return + } + + if (!state.attempt || event.data.assistantMessageID !== state.attempt.assistantMessageID) return + + if ( + event.type === "session.text.started" || + event.type === "session.reasoning.started" || + event.type === "session.tool.input.started" + ) { + state.attempt.first ??= event.created + if (event.type === "session.text.started") state.attempt.answer ??= event.created + return + } + + if (event.type === "session.step.streamed") { + state.attempt.streamed = event.created + return + } + + // Interrupted or failed attempts without usage would publish misleading partial numbers. + if (!event.data.tokens || state.attempt.first === undefined || state.attempt.streamed === undefined) return + state.attempt.tokens = event.data.tokens + state.latest = attemptMetrics(state.attempt) +} + +export function foldProviderMetrics(events: readonly ProviderMetricEvent[]) { + const state: ProviderMetricState = {} + events.forEach((event) => applyProviderMetricEvent(state, event)) + return state.latest +} + +/** + * Baseline from already-loaded history. Text parts carry no start timestamp yet, so TTFT, TTFA, + * and TPS stay unavailable for text-first requests until a live request supplies them. + */ +export function projectedProviderMetrics(messages: readonly SessionMessageInfo[]): ProviderMetrics | undefined { + const message = messages.findLast( + (item): item is SessionMessageAssistant => + item.type === "assistant" && item.time.streamed !== undefined && item.tokens !== undefined, + ) + if (!message) return + // Content is chronological; only a non-text head carries the first-output time. + const head = message.content[0] + const first = head && head.type !== "text" ? head.time?.created : undefined + // Reasoning ends when the answer starts, so a reasoning part right before the first text + // approximates the live `session.text.started` timestamp. + const text = message.content.findIndex((item) => item.type === "text") + const before = text > 0 ? message.content[text - 1] : undefined + const answer = first !== undefined && before?.type === "reasoning" ? before.time?.completed : undefined + return attemptMetrics({ + assistantMessageID: message.id, + started: message.time.created, + first, + answer, + streamed: message.time.streamed, + tokens: message.tokens, + }) +} + +function attemptMetrics(attempt: Attempt): ProviderMetrics { + const ttft = elapsed(attempt.started, attempt.first) + const ttfa = elapsed(attempt.started, attempt.answer) + const e2e = elapsed(attempt.started, attempt.streamed) + // Output tokens exclude reasoning, so measure them from the answer start when one exists. + const generation = elapsed(attempt.answer ?? attempt.first, attempt.streamed) + const output = attempt.tokens?.output + return { + tps: generation && output && generation > 0 && output > 0 ? output / (generation / 1_000) : undefined, + ttft, + ttfa, + e2e, + } +} + +function elapsed(start: number | undefined, end: number | undefined) { + if (start === undefined || end === undefined) return + return Math.max(0, end - start) +} diff --git a/packages/app/src/shell/shell.tsx b/packages/app/src/shell/shell.tsx index 92510edb21d..df99ffa8ec0 100644 --- a/packages/app/src/shell/shell.tsx +++ b/packages/app/src/shell/shell.tsx @@ -11,6 +11,8 @@ import { useSettingsSurface } from "@/settings/surface" import { useSettings } from "@/settings/model" import { SshAuthentication } from "@/servers/ssh/authentication" import { useUpdaterInstall } from "@/shell/updates/download" +import { useCommand } from "@/shell/commands/command" +import { useLanguage } from "@/runtime/i18n/language" const DebugBar = lazy(() => import("@/shell/debug/debug-bar").then((module) => ({ default: module.DebugBar }))) @@ -19,6 +21,8 @@ export default function Layout(props: ParentProps) { const settings = useSettingsSurface() const preferences = useSettings() const installUpdate = useUpdaterInstall() + const command = useCommand() + const language = useLanguage() const mobile = createMediaQuery("(max-width: 767px)") const [state, setState] = createStore({ debugTools: false, @@ -35,14 +39,21 @@ export default function Layout(props: ParentProps) { install: installUpdate, } // A plain object avoids the compiler's conditional-prop memo, which leaks when read from event handlers. - const debugTools = import.meta.env.DEV - ? { - get visible() { - return state.debugTools - }, - toggle: () => setState("debugTools", (value) => !value), - } - : undefined + const debugTools = { + get visible() { + return state.debugTools + }, + toggle: () => setState("debugTools", (value) => !value), + } + + command.register("debug-bar", () => [ + { + id: "debugBar.toggle", + title: language.t("command.debugBar.toggle"), + category: language.t("command.category.view"), + onSelect: debugTools.toggle, + }, + ]) return ( @@ -106,9 +117,9 @@ export default function Layout(props: ParentProps) {
- + - +