fix(session): measure tps from the first streamed token (#63321)

This commit is contained in:
Dax
2026-11-11 10:32:49 -05:00
committed by GitHub
Unverified
parent 2ef38b2172
commit 99f2bea6df
17 changed files with 150 additions and 18 deletions
+23 -1
View File
@@ -4,7 +4,7 @@ import { SessionTurn } from "./session-turn.js"
type Message = {
id: string
type: string
time: { created: number; streamed?: number; completed?: number }
time: { created: number; firstToken?: number; streamed?: number; completed?: number }
tokens?: { output: number; reasoning: number }
}
@@ -15,12 +15,34 @@ test("measures turn duration from the user prompt across assistant steps", () =>
expect(SessionTurn.duration(messages, final)).toBe(29_000)
})
test("measures decode throughput from the first streamed token", () => {
const first = step("assistant-1", { created: 6_000, firstToken: 8_000, streamed: 10_000, completed: 20_000 }, 20, 5)
const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10)
const messages = [input("user-1", 1_000), first, final]
// 65 tokens over the 5s the provider spent streaming them, not the 10s that include waiting for the first.
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(13)
expect(SessionTurn.timeToFirstToken(messages, final)).toBe(2_000)
})
test("falls back to the request window for steps recorded without a first token", () => {
const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5)
const final = step("assistant-2", { created: 24_000, firstToken: 25_000, streamed: 28_000, completed: 31_000 }, 30, 10)
const messages = [input("user-1", 1_000), first, final]
// The old step counts its 4s request window, the new one its 3s decode window.
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(65 / 7)
// Only the step that recorded a first token can report how long it waited for one.
expect(SessionTurn.timeToFirstToken(messages, final)).toBe(1_000)
})
test("measures request throughput including reasoning without tool time", () => {
const first = step("assistant-1", { created: 6_000, streamed: 10_000, completed: 20_000 }, 20, 5)
const final = step("assistant-2", { created: 24_000, streamed: 30_000, completed: 31_000 }, 30, 10)
const messages = [input("user-1", 1_000), first, final]
expect(SessionTurn.tokensPerSecond(messages, final)).toBe(6.5)
expect(SessionTurn.timeToFirstToken(messages, final)).toBeUndefined()
first.time.streamed = undefined
expect(SessionTurn.tokensPerSecond(messages, final)).toBeUndefined()
})
+35 -7
View File
@@ -4,7 +4,13 @@ export * as SessionTurn from "./session-turn.js"
export interface Message {
readonly id: string
readonly type: string
readonly time: { readonly created: number; readonly streamed?: number; readonly completed?: number }
readonly time: {
readonly created: number
/** When the provider streamed the first output token; steps recorded before this existed leave it unset. */
readonly firstToken?: number
readonly streamed?: number
readonly completed?: number
}
readonly tokens?: { readonly output: number; readonly reasoning: number }
}
@@ -27,14 +33,19 @@ export function duration(messages: readonly Message[], message: Message, options
return Math.max(0, message.time.completed - (input?.time.created ?? message.time.created))
}
/** Output and reasoning tokens per second across the turn's steps up to `message`, or undefined until every step has streamed. */
/**
* Output and reasoning tokens per second while the provider streamed them, or undefined until every step has streamed.
*
* Each step counts from its first streamed token to the end of the response body, so the wait for that first token
* (time to first token) is not divided into the output rate. Steps recorded before `time.firstToken` existed count
* from request dispatch instead, which is what this measured for them before the field existed.
*/
export function tokensPerSecond(messages: readonly Message[], message: Message, options: Options = {}) {
const index = options.index ?? messages.findIndex((item) => item.id === message.id)
const end = index === -1 ? messages.length : index + 1
const start = inputIndex(messages, end, options.legacy ?? legacy(messages))
const steps = messages.slice(start + 1, end).filter((item) => item.type === "assistant")
const steps = turnSteps(messages, message, options)
const durations = steps.flatMap((step) =>
step.time.streamed === undefined ? [] : [Math.max(0, step.time.streamed - step.time.created)],
step.time.streamed === undefined
? []
: [Math.max(0, step.time.streamed - (step.time.firstToken ?? step.time.created))],
)
if (steps.length === 0 || durations.length !== steps.length) return
const output = steps.reduce((total, step) => total + (step.tokens?.output ?? 0) + (step.tokens?.reasoning ?? 0), 0)
@@ -44,10 +55,27 @@ export function tokensPerSecond(messages: readonly Message[], message: Message,
return output / (total / 1_000)
}
/**
* Milliseconds from the dispatch of the turn's first token-producing step to its first streamed token, or undefined
* until a step records one.
*/
export function timeToFirstToken(messages: readonly Message[], message: Message, options: Options = {}) {
const step = turnSteps(messages, message, options).find((item) => item.time.firstToken !== undefined)
if (step?.time.firstToken === undefined) return
return Math.max(0, step.time.firstToken - step.time.created)
}
export function legacy(messages: readonly Message[]) {
return !messages.some((message) => message.type === "idle")
}
function turnSteps(messages: readonly Message[], message: Message, options: Options) {
const index = options.index ?? messages.findIndex((item) => item.id === message.id)
const end = index === -1 ? messages.length : index + 1
const start = inputIndex(messages, end, options.legacy ?? legacy(messages))
return messages.slice(start + 1, end).filter((item) => item.type === "assistant")
}
function inputIndex(messages: readonly Message[], end: number, legacy: boolean) {
// Reading a sliced prefix subscribes every caller to unrelated historical messages, so walk
// back only as far as the turn boundary: the nearest input in legacy sessions, otherwise the