mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 18:57:17 +03:00
feat(app): show provider metrics in debug bar (#49674)
This commit is contained in:
committed by
GitHub
Unverified
parent
735556eab8
commit
d4303a9ca5
@@ -74,6 +74,7 @@ export const dict = {
|
||||
"command.category.workspace": "Worktree",
|
||||
"command.category.settings": "Settings",
|
||||
"command.logs.export": "Export logs",
|
||||
"command.debugBar.toggle": "Toggle debug bar",
|
||||
|
||||
"theme.scheme.system": "System",
|
||||
"theme.scheme.light": "Light",
|
||||
@@ -966,7 +967,16 @@ export const dict = {
|
||||
"sidebar.empty.description": "Open a project to get started",
|
||||
|
||||
"debugBar.ariaLabel": "Development performance diagnostics",
|
||||
"debugBar.providerAriaLabel": "Provider performance diagnostics",
|
||||
"debugBar.na": "n/a",
|
||||
"debugBar.ttft.label": "TTFT",
|
||||
"debugBar.ttft.tip": "Time from provider request dispatch to the first model output.",
|
||||
"debugBar.ttfa.label": "TTFA",
|
||||
"debugBar.ttfa.tip": "Time from provider request dispatch to the first answer text.",
|
||||
"debugBar.tps.label": "TPS",
|
||||
"debugBar.tps.tip": "Output tokens per second after the first model output.",
|
||||
"debugBar.e2e.label": "E2E",
|
||||
"debugBar.e2e.tip": "Time from provider request dispatch until its response stream ended.",
|
||||
"debugBar.nav.label": "NAV",
|
||||
"debugBar.nav.tip":
|
||||
"Last completed route transition touching a session page, measured from router start until the first paint after it settles.",
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import { useIsRouting, useLocation } from "@solidjs/router"
|
||||
import { batch, createEffect, onCleanup, onMount, Show } from "solid-js"
|
||||
import { useIsRouting, useLocation, useParams } from "@solidjs/router"
|
||||
import { batch, createEffect, createMemo, on, onCleanup, onMount, Show } from "solid-js"
|
||||
import { createStore } from "solid-js/store"
|
||||
import { makeEventListener } from "@solid-primitives/event-listener"
|
||||
import { Tooltip } from "@opencode/ui/tooltip"
|
||||
import { useLanguage } from "@/runtime/i18n/language"
|
||||
import { usePlatform } from "@/runtime/platform/platform"
|
||||
import { useGlobal } from "@/runtime/server/runtime"
|
||||
import { ServerConnection } from "@/runtime/server/registry"
|
||||
import { base64Encode } from "@opencode/util/encode"
|
||||
import {
|
||||
applyProviderMetricEvent,
|
||||
isProviderMetricEvent,
|
||||
projectedProviderMetrics,
|
||||
type ProviderMetrics,
|
||||
type ProviderMetricState,
|
||||
} from "./provider-metrics"
|
||||
|
||||
type Mem = Performance & {
|
||||
memory?: {
|
||||
@@ -39,12 +49,23 @@ const time = (n?: number) => {
|
||||
return `${Math.round(n)}`
|
||||
}
|
||||
|
||||
const fixed = (n?: number, digits = 0) => {
|
||||
if (n === undefined || Number.isNaN(n)) return
|
||||
return n.toFixed(digits)
|
||||
}
|
||||
|
||||
const mb = (n?: number) => {
|
||||
if (n === undefined || Number.isNaN(n)) return
|
||||
const v = n / 1024 / 1024
|
||||
return `${v >= 1024 ? v.toFixed(0) : v.toFixed(1)}MB`
|
||||
}
|
||||
|
||||
const duration = (n?: number) => {
|
||||
if (n === undefined || Number.isNaN(n)) return
|
||||
if (n < 1_000) return `${Math.round(n)}ms`
|
||||
return `${(n / 1_000).toFixed(n < 10_000 ? 1 : 0)}s`
|
||||
}
|
||||
|
||||
const bad = (n: number | undefined, limit: number, low = false) => {
|
||||
if (n === undefined || Number.isNaN(n)) return false
|
||||
return low ? n < limit : n > limit
|
||||
@@ -80,6 +101,7 @@ function Cell(props: {
|
||||
}}
|
||||
>
|
||||
<div
|
||||
dir="ltr"
|
||||
classList={{
|
||||
"text-[10px] leading-none font-black uppercase tracking-[0.04em] opacity-70": true,
|
||||
}}
|
||||
@@ -87,6 +109,7 @@ function Cell(props: {
|
||||
{props.label}
|
||||
</div>
|
||||
<div
|
||||
dir="ltr"
|
||||
classList={{
|
||||
"uppercase font-bold tabular-nums": true,
|
||||
"text-[11px] leading-text-tight": !!props.inline,
|
||||
@@ -136,8 +159,12 @@ function ToggleCell(props: {
|
||||
"flex-col items-center": !props.inline,
|
||||
}}
|
||||
>
|
||||
<span class="text-[10px] leading-none font-black tracking-[0.04em] opacity-70">{props.label}</span>
|
||||
<span class="text-[11px] leading-none font-bold">{props.value}</span>
|
||||
<span dir="ltr" class="text-[10px] leading-none font-black tracking-[0.04em] opacity-70">
|
||||
{props.label}
|
||||
</span>
|
||||
<span dir="ltr" class="text-[11px] leading-none font-bold">
|
||||
{props.value}
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
)
|
||||
@@ -149,9 +176,11 @@ function ToggleCell(props: {
|
||||
)
|
||||
}
|
||||
|
||||
export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
export function DebugBar(props: { diagnostics?: boolean; inline?: boolean } = {}) {
|
||||
const language = useLanguage()
|
||||
const platform = usePlatform()
|
||||
const global = useGlobal()
|
||||
const params = useParams<{ serverKey?: string; id?: string }>()
|
||||
const location = useLocation()
|
||||
const routing = useIsRouting()
|
||||
const [state, setState] = createStore({
|
||||
@@ -175,8 +204,55 @@ export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
dur: undefined as number | undefined,
|
||||
pending: false,
|
||||
},
|
||||
live: undefined as ProviderMetrics | undefined,
|
||||
})
|
||||
|
||||
const target = createMemo(
|
||||
() => {
|
||||
if (!params.serverKey || !params.id) return
|
||||
const connection = global.servers
|
||||
.list()
|
||||
.find((item) => base64Encode(ServerConnection.key(item)) === params.serverKey)
|
||||
if (!connection) return
|
||||
return { ctx: global.ensureServerCtx(connection), id: params.id }
|
||||
},
|
||||
undefined,
|
||||
{ equals: (a, b) => a?.ctx === b?.ctx && a?.id === b?.id },
|
||||
)
|
||||
// History comes from the already-loaded message projection; live requests refine it in place.
|
||||
const projected = createMemo(() => {
|
||||
const current = target()
|
||||
if (!current) return
|
||||
return projectedProviderMetrics(current.ctx.data.session.message.list(current.id))
|
||||
})
|
||||
const metrics = () => state.live ?? projected()
|
||||
|
||||
// Missed events during an outage are never replayed; the refreshed projection must win.
|
||||
createEffect(
|
||||
on(
|
||||
() => target()?.ctx.sdk.connection.status(),
|
||||
(status) => {
|
||||
if (status !== "connected") setState("live", undefined)
|
||||
},
|
||||
{ defer: true },
|
||||
),
|
||||
)
|
||||
|
||||
createEffect(
|
||||
on(target, (current) => {
|
||||
setState("live", undefined)
|
||||
if (!current) return
|
||||
const accumulator: ProviderMetricState = {}
|
||||
onCleanup(
|
||||
current.ctx.sdk.event.listen((event) => {
|
||||
if (!isProviderMetricEvent(event) || event.data.sessionID !== current.id) return
|
||||
applyProviderMetricEvent(accumulator, event)
|
||||
if (accumulator.latest) setState("live", accumulator.latest)
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
const na = () => language.t("debugBar.na").toUpperCase()
|
||||
const heap = () => (state.heap.limit ? (state.heap.used ?? 0) / state.heap.limit : undefined)
|
||||
const heapv = () => {
|
||||
@@ -204,6 +280,7 @@ export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
let two = 0
|
||||
|
||||
createEffect(() => {
|
||||
if (!props.diagnostics) return
|
||||
const busy = routing()
|
||||
const next = `${location.pathname}${location.search}`
|
||||
|
||||
@@ -248,6 +325,7 @@ export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
})
|
||||
|
||||
onMount(() => {
|
||||
if (!props.diagnostics) return
|
||||
const obs: PerformanceObserver[] = []
|
||||
const fps: Array<{ at: number; dur: number }> = []
|
||||
const long: Array<{ at: number; dur: number }> = []
|
||||
@@ -448,7 +526,7 @@ export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
|
||||
return (
|
||||
<aside
|
||||
aria-label={language.t("debugBar.ariaLabel")}
|
||||
aria-label={language.t(props.diagnostics ? "debugBar.ariaLabel" : "debugBar.providerAriaLabel")}
|
||||
classList={{
|
||||
"pointer-events-auto hidden overflow-hidden text-text-strong md:block": true,
|
||||
"mt-[-6px] w-full shrink-0 px-3 py-1": !!props.inline,
|
||||
@@ -467,102 +545,132 @@ export function DebugBar(props: { inline?: boolean } = {}) {
|
||||
}}
|
||||
>
|
||||
<Cell
|
||||
label={language.t("debugBar.nav.label")}
|
||||
tip={language.t("debugBar.nav.tip")}
|
||||
value={navv()}
|
||||
bad={bad(state.nav.dur, 400)}
|
||||
dim={state.nav.dur === undefined && !state.nav.pending}
|
||||
label={language.t("debugBar.tps.label")}
|
||||
tip={language.t("debugBar.tps.tip")}
|
||||
value={fixed(metrics()?.tps, 1) ?? na()}
|
||||
dim={metrics()?.tps === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.fps.label")}
|
||||
tip={language.t("debugBar.fps.tip")}
|
||||
value={state.fps === undefined ? na() : `${Math.round(state.fps)}`}
|
||||
bad={bad(state.fps, 50, true)}
|
||||
dim={state.fps === undefined}
|
||||
label={language.t("debugBar.ttft.label")}
|
||||
tip={language.t("debugBar.ttft.tip")}
|
||||
value={duration(metrics()?.ttft) ?? na()}
|
||||
dim={metrics()?.ttft === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.frame.label")}
|
||||
tip={language.t("debugBar.frame.tip")}
|
||||
value={time(state.gap) ?? na()}
|
||||
bad={bad(state.gap, 50)}
|
||||
dim={state.gap === undefined}
|
||||
label={language.t("debugBar.ttfa.label")}
|
||||
tip={language.t("debugBar.ttfa.tip")}
|
||||
value={duration(metrics()?.ttfa) ?? na()}
|
||||
dim={metrics()?.ttfa === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.jank.label")}
|
||||
tip={language.t("debugBar.jank.tip")}
|
||||
value={state.jank === undefined ? na() : `${state.jank}`}
|
||||
bad={bad(state.jank, 8)}
|
||||
dim={state.jank === undefined}
|
||||
label={language.t("debugBar.e2e.label")}
|
||||
tip={language.t("debugBar.e2e.tip")}
|
||||
value={duration(metrics()?.e2e) ?? na()}
|
||||
dim={metrics()?.e2e === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.long.label")}
|
||||
tip={language.t("debugBar.long.tip", { max: ms(state.long.max) ?? na() })}
|
||||
value={longv()}
|
||||
bad={bad(state.long.block, 200)}
|
||||
dim={state.long.count === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.delay.label")}
|
||||
tip={language.t("debugBar.delay.tip")}
|
||||
value={time(state.delay) ?? na()}
|
||||
bad={bad(state.delay, 100)}
|
||||
dim={state.delay === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.inp.label")}
|
||||
tip={language.t("debugBar.inp.tip")}
|
||||
value={time(state.inp) ?? na()}
|
||||
bad={bad(state.inp, 200)}
|
||||
dim={state.inp === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.cls.label")}
|
||||
tip={language.t("debugBar.cls.tip")}
|
||||
value={state.cls === undefined ? na() : state.cls.toFixed(2)}
|
||||
bad={bad(state.cls, 0.1)}
|
||||
dim={state.cls === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.mem.label")}
|
||||
tip={
|
||||
state.heap.used === undefined
|
||||
? language.t("debugBar.mem.tipUnavailable")
|
||||
: language.t("debugBar.mem.tip", {
|
||||
used: mb(state.heap.used) ?? na(),
|
||||
limit: mb(state.heap.limit) ?? na(),
|
||||
})
|
||||
}
|
||||
value={heapv()}
|
||||
bad={bad(heap(), 0.8)}
|
||||
dim={state.heap.used === undefined}
|
||||
inline={props.inline}
|
||||
span={platform.setForceFocus ? 2 : 3}
|
||||
/>
|
||||
<ToggleCell
|
||||
active={language.direction() === "rtl"}
|
||||
inline={props.inline}
|
||||
label={language.t("debugBar.direction.label")}
|
||||
tip={language.t("debugBar.direction.tip")}
|
||||
value={language.t(`debugBar.direction.${language.direction()}`)}
|
||||
onClick={() => language.setDirection(language.direction() === "rtl" ? "ltr" : "rtl")}
|
||||
/>
|
||||
<Show when={platform.setForceFocus}>
|
||||
<ToggleCell
|
||||
active={state.focus}
|
||||
<Show when={props.diagnostics}>
|
||||
<Cell
|
||||
label={language.t("debugBar.nav.label")}
|
||||
tip={language.t("debugBar.nav.tip")}
|
||||
value={navv()}
|
||||
bad={bad(state.nav.dur, 400)}
|
||||
dim={state.nav.dur === undefined && !state.nav.pending}
|
||||
inline={props.inline}
|
||||
label={language.t("debugBar.focus.label")}
|
||||
tip={language.t("debugBar.focus.tip")}
|
||||
value={language.t(state.focus ? "debugBar.focus.on" : "debugBar.focus.off")}
|
||||
onClick={() => void toggleFocus()}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.fps.label")}
|
||||
tip={language.t("debugBar.fps.tip")}
|
||||
value={state.fps === undefined ? na() : `${Math.round(state.fps)}`}
|
||||
bad={bad(state.fps, 50, true)}
|
||||
dim={state.fps === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.frame.label")}
|
||||
tip={language.t("debugBar.frame.tip")}
|
||||
value={time(state.gap) ?? na()}
|
||||
bad={bad(state.gap, 50)}
|
||||
dim={state.gap === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.jank.label")}
|
||||
tip={language.t("debugBar.jank.tip")}
|
||||
value={state.jank === undefined ? na() : `${state.jank}`}
|
||||
bad={bad(state.jank, 8)}
|
||||
dim={state.jank === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.long.label")}
|
||||
tip={language.t("debugBar.long.tip", { max: ms(state.long.max) ?? na() })}
|
||||
value={longv()}
|
||||
bad={bad(state.long.block, 200)}
|
||||
dim={state.long.count === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.delay.label")}
|
||||
tip={language.t("debugBar.delay.tip")}
|
||||
value={time(state.delay) ?? na()}
|
||||
bad={bad(state.delay, 100)}
|
||||
dim={state.delay === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.inp.label")}
|
||||
tip={language.t("debugBar.inp.tip")}
|
||||
value={time(state.inp) ?? na()}
|
||||
bad={bad(state.inp, 200)}
|
||||
dim={state.inp === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.cls.label")}
|
||||
tip={language.t("debugBar.cls.tip")}
|
||||
value={state.cls === undefined ? na() : state.cls.toFixed(2)}
|
||||
bad={bad(state.cls, 0.1)}
|
||||
dim={state.cls === undefined}
|
||||
inline={props.inline}
|
||||
/>
|
||||
<Cell
|
||||
label={language.t("debugBar.mem.label")}
|
||||
tip={
|
||||
state.heap.used === undefined
|
||||
? language.t("debugBar.mem.tipUnavailable")
|
||||
: language.t("debugBar.mem.tip", {
|
||||
used: mb(state.heap.used) ?? na(),
|
||||
limit: mb(state.heap.limit) ?? na(),
|
||||
})
|
||||
}
|
||||
value={heapv()}
|
||||
bad={bad(heap(), 0.8)}
|
||||
dim={state.heap.used === undefined}
|
||||
inline={props.inline}
|
||||
span={platform.setForceFocus ? 2 : 3}
|
||||
/>
|
||||
<ToggleCell
|
||||
active={language.direction() === "rtl"}
|
||||
inline={props.inline}
|
||||
label={language.t("debugBar.direction.label")}
|
||||
tip={language.t("debugBar.direction.tip")}
|
||||
value={language.t(`debugBar.direction.${language.direction()}`)}
|
||||
onClick={() => language.setDirection(language.direction() === "rtl" ? "ltr" : "rtl")}
|
||||
/>
|
||||
<Show when={platform.setForceFocus}>
|
||||
<ToggleCell
|
||||
active={state.focus}
|
||||
inline={props.inline}
|
||||
label={language.t("debugBar.focus.label")}
|
||||
tip={language.t("debugBar.focus.tip")}
|
||||
value={language.t(state.focus ? "debugBar.focus.on" : "debugBar.focus.off")}
|
||||
onClick={() => void toggleFocus()}
|
||||
/>
|
||||
</Show>
|
||||
</Show>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode/client/promise"
|
||||
import type { ProviderMetricEvent } from "./provider-metrics"
|
||||
import { foldProviderMetrics, projectedProviderMetrics } from "./provider-metrics"
|
||||
|
||||
const durable = { aggregateID: "ses_test", seq: 0, version: 1 } as const
|
||||
|
||||
const events: ProviderMetricEvent[] = [
|
||||
{
|
||||
id: "evt_started",
|
||||
created: 1_000,
|
||||
type: "session.step.started",
|
||||
durable,
|
||||
data: {
|
||||
sessionID: "ses_test",
|
||||
assistantMessageID: "msg_assistant",
|
||||
agent: "build",
|
||||
model: { id: "model", providerID: "provider" },
|
||||
started: 1_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "evt_reasoning",
|
||||
created: 1_300,
|
||||
type: "session.reasoning.started",
|
||||
durable: { ...durable, seq: 1 },
|
||||
data: { sessionID: "ses_test", assistantMessageID: "msg_assistant", ordinal: 0 },
|
||||
},
|
||||
{
|
||||
id: "evt_text",
|
||||
created: 1_800,
|
||||
type: "session.text.started",
|
||||
durable: { ...durable, seq: 2 },
|
||||
data: { sessionID: "ses_test", assistantMessageID: "msg_assistant", ordinal: 0 },
|
||||
},
|
||||
{
|
||||
id: "evt_streamed",
|
||||
created: 3_800,
|
||||
type: "session.step.streamed",
|
||||
durable: { ...durable, seq: 3 },
|
||||
data: { sessionID: "ses_test", assistantMessageID: "msg_assistant" },
|
||||
},
|
||||
{
|
||||
id: "evt_ended",
|
||||
created: 4_000,
|
||||
type: "session.step.ended",
|
||||
durable: { ...durable, seq: 4 },
|
||||
data: {
|
||||
sessionID: "ses_test",
|
||||
assistantMessageID: "msg_assistant",
|
||||
finish: "stop",
|
||||
cost: 0,
|
||||
tokens: { input: 200, output: 100, reasoning: 20, cache: { read: 0, write: 0 } },
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
test("calculates provider response metrics from durable events", () => {
|
||||
expect(foldProviderMetrics(events)).toEqual({
|
||||
tps: 50,
|
||||
ttft: 300,
|
||||
ttfa: 800,
|
||||
e2e: 2_800,
|
||||
})
|
||||
})
|
||||
|
||||
test("ignores failed attempts without usage", () => {
|
||||
expect(
|
||||
foldProviderMetrics([
|
||||
...events.slice(0, 4),
|
||||
{
|
||||
id: "evt_failed",
|
||||
created: 4_000,
|
||||
type: "session.step.failed",
|
||||
durable: { ...durable, seq: 4 },
|
||||
data: {
|
||||
sessionID: "ses_test",
|
||||
assistantMessageID: "msg_assistant",
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
},
|
||||
},
|
||||
]),
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
test("keeps completed metrics while the next provider attempt runs", () => {
|
||||
expect(
|
||||
foldProviderMetrics([
|
||||
...events,
|
||||
{
|
||||
id: "evt_retry",
|
||||
created: 5_000,
|
||||
type: "session.step.started",
|
||||
durable: { ...durable, seq: 5 },
|
||||
data: {
|
||||
sessionID: "ses_test",
|
||||
assistantMessageID: "msg_assistant",
|
||||
agent: "build",
|
||||
model: { id: "model", providerID: "provider" },
|
||||
started: 5_000,
|
||||
},
|
||||
},
|
||||
]),
|
||||
).toEqual(foldProviderMetrics(events))
|
||||
})
|
||||
|
||||
const assistant: SessionMessageAssistant = {
|
||||
id: "msg_assistant",
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { id: "model", providerID: "provider" },
|
||||
content: [
|
||||
{ type: "reasoning", text: "Think", time: { created: 1_300, completed: 1_700 } },
|
||||
{ type: "text", text: "Answer" },
|
||||
],
|
||||
tokens: { input: 200, output: 100, reasoning: 20, cache: { read: 0, write: 0 } },
|
||||
time: { created: 1_000, streamed: 3_800, completed: 4_000 },
|
||||
}
|
||||
|
||||
test("derives a baseline from the latest completed projected request", () => {
|
||||
const messages: SessionMessageInfo[] = [
|
||||
{ id: "msg_user", type: "user", text: "Hi", time: { created: 1 } },
|
||||
assistant,
|
||||
{ ...assistant, id: "msg_running", tokens: undefined, time: { created: 5_000 } },
|
||||
]
|
||||
// Reasoning ended at 1_700, so TPS spans 1_700 → 3_800 = 100 / 2.1s.
|
||||
expect(projectedProviderMetrics(messages)).toEqual({ tps: 100 / 2.1, ttft: 300, ttfa: 700, e2e: 2_800 })
|
||||
})
|
||||
|
||||
const tool = (created: number): SessionMessageAssistant["content"][number] => ({
|
||||
type: "tool",
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
state: { status: "running", input: {}, metadata: {} },
|
||||
time: { created, ran: created + 100 },
|
||||
})
|
||||
|
||||
test("leaves text-first history unavailable until a live request", () => {
|
||||
const unavailable = { tps: undefined, ttft: undefined, ttfa: undefined, e2e: 2_800 }
|
||||
expect(projectedProviderMetrics([{ ...assistant, content: [{ type: "text", text: "Answer" }] }])).toEqual(unavailable)
|
||||
expect(
|
||||
projectedProviderMetrics([{ ...assistant, content: [{ type: "text", text: "Answer" }, tool(2_500)] }]),
|
||||
).toEqual(unavailable)
|
||||
})
|
||||
|
||||
test("uses the first tool call as first output for tool-first history", () => {
|
||||
expect(projectedProviderMetrics([{ ...assistant, content: [tool(1_800)] }])).toEqual({
|
||||
tps: 50,
|
||||
ttft: 800,
|
||||
ttfa: undefined,
|
||||
e2e: 2_800,
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
import type {
|
||||
SessionLogItem,
|
||||
SessionMessageAssistant,
|
||||
SessionMessageInfo,
|
||||
TokenUsageInfo,
|
||||
} from "@opencode/client/promise"
|
||||
|
||||
type ProviderMetricEventType =
|
||||
| "session.step.started"
|
||||
| "session.step.streamed"
|
||||
| "session.step.ended"
|
||||
| "session.step.failed"
|
||||
| "session.text.started"
|
||||
| "session.reasoning.started"
|
||||
| "session.tool.input.started"
|
||||
|
||||
const types: ReadonlySet<string> = new Set<ProviderMetricEventType>([
|
||||
"session.step.started",
|
||||
"session.step.streamed",
|
||||
"session.step.ended",
|
||||
"session.step.failed",
|
||||
"session.text.started",
|
||||
"session.reasoning.started",
|
||||
"session.tool.input.started",
|
||||
])
|
||||
|
||||
export type ProviderMetricEvent = Extract<SessionLogItem, { type: ProviderMetricEventType }>
|
||||
|
||||
export type ProviderMetrics = {
|
||||
tps?: number
|
||||
ttft?: number
|
||||
ttfa?: number
|
||||
e2e?: number
|
||||
}
|
||||
|
||||
type Attempt = {
|
||||
assistantMessageID: string
|
||||
started: number
|
||||
first?: number
|
||||
answer?: number
|
||||
streamed?: number
|
||||
tokens?: TokenUsageInfo
|
||||
}
|
||||
|
||||
export type ProviderMetricState = { attempt?: Attempt; latest?: ProviderMetrics }
|
||||
|
||||
export function isProviderMetricEvent(event: { type: string }): event is ProviderMetricEvent {
|
||||
return types.has(event.type)
|
||||
}
|
||||
|
||||
export function applyProviderMetricEvent(state: ProviderMetricState, event: ProviderMetricEvent) {
|
||||
if (event.type === "session.step.started") {
|
||||
state.attempt = {
|
||||
assistantMessageID: event.data.assistantMessageID,
|
||||
started: event.data.started,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (!state.attempt || event.data.assistantMessageID !== state.attempt.assistantMessageID) return
|
||||
|
||||
if (
|
||||
event.type === "session.text.started" ||
|
||||
event.type === "session.reasoning.started" ||
|
||||
event.type === "session.tool.input.started"
|
||||
) {
|
||||
state.attempt.first ??= event.created
|
||||
if (event.type === "session.text.started") state.attempt.answer ??= event.created
|
||||
return
|
||||
}
|
||||
|
||||
if (event.type === "session.step.streamed") {
|
||||
state.attempt.streamed = event.created
|
||||
return
|
||||
}
|
||||
|
||||
// Interrupted or failed attempts without usage would publish misleading partial numbers.
|
||||
if (!event.data.tokens || state.attempt.first === undefined || state.attempt.streamed === undefined) return
|
||||
state.attempt.tokens = event.data.tokens
|
||||
state.latest = attemptMetrics(state.attempt)
|
||||
}
|
||||
|
||||
export function foldProviderMetrics(events: readonly ProviderMetricEvent[]) {
|
||||
const state: ProviderMetricState = {}
|
||||
events.forEach((event) => applyProviderMetricEvent(state, event))
|
||||
return state.latest
|
||||
}
|
||||
|
||||
/**
|
||||
* Baseline from already-loaded history. Text parts carry no start timestamp yet, so TTFT, TTFA,
|
||||
* and TPS stay unavailable for text-first requests until a live request supplies them.
|
||||
*/
|
||||
export function projectedProviderMetrics(messages: readonly SessionMessageInfo[]): ProviderMetrics | undefined {
|
||||
const message = messages.findLast(
|
||||
(item): item is SessionMessageAssistant =>
|
||||
item.type === "assistant" && item.time.streamed !== undefined && item.tokens !== undefined,
|
||||
)
|
||||
if (!message) return
|
||||
// Content is chronological; only a non-text head carries the first-output time.
|
||||
const head = message.content[0]
|
||||
const first = head && head.type !== "text" ? head.time?.created : undefined
|
||||
// Reasoning ends when the answer starts, so a reasoning part right before the first text
|
||||
// approximates the live `session.text.started` timestamp.
|
||||
const text = message.content.findIndex((item) => item.type === "text")
|
||||
const before = text > 0 ? message.content[text - 1] : undefined
|
||||
const answer = first !== undefined && before?.type === "reasoning" ? before.time?.completed : undefined
|
||||
return attemptMetrics({
|
||||
assistantMessageID: message.id,
|
||||
started: message.time.created,
|
||||
first,
|
||||
answer,
|
||||
streamed: message.time.streamed,
|
||||
tokens: message.tokens,
|
||||
})
|
||||
}
|
||||
|
||||
function attemptMetrics(attempt: Attempt): ProviderMetrics {
|
||||
const ttft = elapsed(attempt.started, attempt.first)
|
||||
const ttfa = elapsed(attempt.started, attempt.answer)
|
||||
const e2e = elapsed(attempt.started, attempt.streamed)
|
||||
// Output tokens exclude reasoning, so measure them from the answer start when one exists.
|
||||
const generation = elapsed(attempt.answer ?? attempt.first, attempt.streamed)
|
||||
const output = attempt.tokens?.output
|
||||
return {
|
||||
tps: generation && output && generation > 0 && output > 0 ? output / (generation / 1_000) : undefined,
|
||||
ttft,
|
||||
ttfa,
|
||||
e2e,
|
||||
}
|
||||
}
|
||||
|
||||
function elapsed(start: number | undefined, end: number | undefined) {
|
||||
if (start === undefined || end === undefined) return
|
||||
return Math.max(0, end - start)
|
||||
}
|
||||
@@ -11,6 +11,8 @@ import { useSettingsSurface } from "@/settings/surface"
|
||||
import { useSettings } from "@/settings/model"
|
||||
import { SshAuthentication } from "@/servers/ssh/authentication"
|
||||
import { useUpdaterInstall } from "@/shell/updates/download"
|
||||
import { useCommand } from "@/shell/commands/command"
|
||||
import { useLanguage } from "@/runtime/i18n/language"
|
||||
|
||||
const DebugBar = lazy(() => import("@/shell/debug/debug-bar").then((module) => ({ default: module.DebugBar })))
|
||||
|
||||
@@ -19,6 +21,8 @@ export default function Layout(props: ParentProps) {
|
||||
const settings = useSettingsSurface()
|
||||
const preferences = useSettings()
|
||||
const installUpdate = useUpdaterInstall()
|
||||
const command = useCommand()
|
||||
const language = useLanguage()
|
||||
const mobile = createMediaQuery("(max-width: 767px)")
|
||||
const [state, setState] = createStore({
|
||||
debugTools: false,
|
||||
@@ -35,14 +39,21 @@ export default function Layout(props: ParentProps) {
|
||||
install: installUpdate,
|
||||
}
|
||||
// A plain object avoids the compiler's conditional-prop memo, which leaks when read from event handlers.
|
||||
const debugTools = import.meta.env.DEV
|
||||
? {
|
||||
get visible() {
|
||||
return state.debugTools
|
||||
},
|
||||
toggle: () => setState("debugTools", (value) => !value),
|
||||
}
|
||||
: undefined
|
||||
const debugTools = {
|
||||
get visible() {
|
||||
return state.debugTools
|
||||
},
|
||||
toggle: () => setState("debugTools", (value) => !value),
|
||||
}
|
||||
|
||||
command.register("debug-bar", () => [
|
||||
{
|
||||
id: "debugBar.toggle",
|
||||
title: language.t("command.debugBar.toggle"),
|
||||
category: language.t("command.category.view"),
|
||||
onSelect: debugTools.toggle,
|
||||
},
|
||||
])
|
||||
|
||||
return (
|
||||
<TitlebarRightProvider>
|
||||
@@ -106,9 +117,9 @@ export default function Layout(props: ParentProps) {
|
||||
</SshAuthentication>
|
||||
</main>
|
||||
</div>
|
||||
<Show when={import.meta.env.DEV && state.debugTools}>
|
||||
<Show when={state.debugTools}>
|
||||
<Suspense>
|
||||
<DebugBar inline />
|
||||
<DebugBar diagnostics={import.meta.env.DEV} inline />
|
||||
</Suspense>
|
||||
</Show>
|
||||
<ToastRegion />
|
||||
|
||||
Reference in New Issue
Block a user