mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 18:57:17 +03:00
fix(session): measure tps from the first streamed token (#63321)
This commit is contained in:
@@ -4,7 +4,7 @@ import { SessionTurn } from "./session-turn.js"
|
||||
type Message = {
|
||||
id: string
|
||||
type: string
|
||||
time: { created: number; streamed?: number; completed?: number }
|
||||
time: { created: number; firstToken?: number; streamed?: number; completed?: number }
|
||||
tokens?: { output: number; reasoning: number }
|
||||
}
|
||||
|
||||
@@ -15,12 +15,34 @@ test("measures turn duration from the user prompt across assistant steps", () =>
|
||||
expect(SessionTurn.duration(messages, final)).toBe(29_000)
|
||||
})
|
||||
|
||||
test("measures decode throughput from the first streamed token", () => {
|
||||
const first = step("assistant-1", { created: 6_000, firstToken: 8_000, streamed: 10_000, completed: 20_000 }, 20, 5)
|
||||
const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10)
|
||||
const messages = [input("user-1", 1_000), first, final]
|
||||
|
||||
// 65 tokens over the 5s the provider spent streaming them, not the 10s that include waiting for the first.
|
||||
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(13)
|
||||
expect(SessionTurn.timeToFirstToken(messages, final)).toBe(2_000)
|
||||
})
|
||||
|
||||
test("falls back to the request window for steps recorded without a first token", () => {
|
||||
const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5)
|
||||
const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10)
|
||||
const messages = [input("user-1", 1_000), first, final]
|
||||
|
||||
// The old step counts its 4s request window, the new one its 3s decode window.
|
||||
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(65 / 7)
|
||||
// Only the step that recorded a first token can report how long it waited for one.
|
||||
expect(SessionTurn.timeToFirstToken(messages, final)).toBe(1_000)
|
||||
})
|
||||
|
||||
test("measures request throughput including reasoning without tool time", () => {
|
||||
const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5)
|
||||
const final = step("assistant-2", { created: 24_000, streamed: 30_000, completed: 31_000 }, 30, 10)
|
||||
const messages = [input("user-1", 1_000), first, final]
|
||||
|
||||
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(6.5)
|
||||
expect(SessionTurn.timeToFirstToken(messages, final)).toBeUndefined()
|
||||
first.time.streamed = undefined
|
||||
expect(SessionTurn.tokensPerSecond(messages, final)).toBeUndefined()
|
||||
})
|
||||
|
||||
@@ -4,7 +4,13 @@ export * as SessionTurn from "./session-turn.js"
|
||||
export interface Message {
|
||||
readonly id: string
|
||||
readonly type: string
|
||||
readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number }
|
||||
readonly time: {
|
||||
readonly created: number
|
||||
/** When the provider streamed the first output token; steps recorded before this existed leave it unset. */
|
||||
readonly firstToken?: number
|
||||
readonly streamed?: number
|
||||
readonly completed?: number
|
||||
}
|
||||
readonly tokens?: { readonly output: number; readonly reasoning: number }
|
||||
}
|
||||
|
||||
@@ -27,14 +33,19 @@ export function duration(messages: readonly Message[], message: Message, options
|
||||
return Math.max(0, message.time.completed - (input?.time.created ?? message.time.created))
|
||||
}
|
||||
|
||||
/** Output and reasoning tokens per second across the turn's steps up to `message`, or undefined until every step has streamed. */
|
||||
/**
|
||||
* Output and reasoning tokens per second while the provider streamed them, or undefined until every step has streamed.
|
||||
*
|
||||
* Each step counts from its first streamed token to the end of the response body, so the wait for that first token
|
||||
* (time to first token) is not divided into the output rate. Steps recorded before `time.firstToken` existed count
|
||||
* from request dispatch instead, which is what this measured for them before the field existed.
|
||||
*/
|
||||
export function tokensPerSecond(messages: readonly Message[], message: Message, options: Options = {}) {
|
||||
const index = options.index ?? messages.findIndex((item) => item.id === message.id)
|
||||
const end = index === -1 ? messages.length : index + 1
|
||||
const start = inputIndex(messages, end, options.legacy ?? legacy(messages))
|
||||
const steps = messages.slice(start + 1, end).filter((item) => item.type === "assistant")
|
||||
const steps = turnSteps(messages, message, options)
|
||||
const durations = steps.flatMap((step) =>
|
||||
step.time.streamed === undefined ? [] : [Math.max(0, step.time.streamed - step.time.created)],
|
||||
step.time.streamed === undefined
|
||||
? []
|
||||
: [Math.max(0, step.time.streamed - (step.time.firstToken ?? step.time.created))],
|
||||
)
|
||||
if (steps.length === 0 || durations.length !== steps.length) return
|
||||
const output = steps.reduce((total, step) => total + (step.tokens?.output ?? 0) + (step.tokens?.reasoning ?? 0), 0)
|
||||
@@ -44,10 +55,27 @@ export function tokensPerSecond(messages: readonly Message[], message: Message,
|
||||
return output / (total / 1_000)
|
||||
}
|
||||
|
||||
/**
|
||||
* Milliseconds from the dispatch of the turn's first token-producing step to its first streamed token, or undefined
|
||||
* until a step records one.
|
||||
*/
|
||||
export function timeToFirstToken(messages: readonly Message[], message: Message, options: Options = {}) {
|
||||
const step = turnSteps(messages, message, options).find((item) => item.time.firstToken !== undefined)
|
||||
if (step?.time.firstToken === undefined) return
|
||||
return Math.max(0, step.time.firstToken - step.time.created)
|
||||
}
|
||||
|
||||
export function legacy(messages: readonly Message[]) {
|
||||
return !messages.some((message) => message.type === "idle")
|
||||
}
|
||||
|
||||
function turnSteps(messages: readonly Message[], message: Message, options: Options) {
|
||||
const index = options.index ?? messages.findIndex((item) => item.id === message.id)
|
||||
const end = index === -1 ? messages.length : index + 1
|
||||
const start = inputIndex(messages, end, options.legacy ?? legacy(messages))
|
||||
return messages.slice(start + 1, end).filter((item) => item.type === "assistant")
|
||||
}
|
||||
|
||||
function inputIndex(messages: readonly Message[], end: number, legacy: boolean) {
|
||||
// Reading a sliced prefix subscribes every caller to unrelated historical messages, so walk
|
||||
// back only as far as the turn boundary: the nearest input in legacy sessions, otherwise the
|
||||
|
||||
Reference in New Issue
Block a user