perf(app): web benches without Playwright, so they also run in Firefox (#62297)

This commit is contained in:
Luke Parker
2026-11-06 09:36:24 -06:00
committed by GitHub
Unverified
parent 885df74024
commit c62357a58e
14 changed files with 190 additions and 179 deletions
+11 -6
View File
@@ -140,12 +140,17 @@ strobe bench packages/app/perf/timeline.bench.ts --filter "scroll*" --bisect <go
## Write a case
- A file's `prepare` runs in Playwright before the first page load and mocks the server under the variant's own
origin (`new URL(url).origin`); `ctx.case` lets one file give each case its own fixture. What it returns is
`prepared` in `setup`, `run` and `reset`.
- Mock on `cached(playwright.page, mock)` (`fixtures/mock.ts`), never on `playwright.page` itself: Playwright's routing
turns the browser's HTTP cache off for the whole page, so every reload would refetch every script. `cached` answers
the same routes through strobe's `mock`, which pauses only the API, and the e2e mock server runs on it unchanged.
- A file's `prepare` runs before the first page load and mocks the server under the variant's own origin
(`new URL(ctx.url).origin`); `ctx.case` lets one file give each case its own fixture. What it returns is `prepared`
in `setup`, `run` and `reset`.
- Mock on `cached(ctx)` (`fixtures/mock.ts`), never on `ctx.playwright.page` itself: Playwright's routing turns the
browser's HTTP cache off for the whole page, so every reload would refetch every script. `cached` answers the same
routes through strobe's `mock`, which pauses only the API, and the e2e mock server runs on it unchanged. Don't read
`ctx.playwright` in a bench file: it exists only in Chromium. Use `ctx.addInitScript` for init scripts, and seed
state in `setup` with `page.eval`.
- Firefox: `strobe bench <file> --engine firefox --headless --check --primary cpu.work`. `main.*` metrics are
Chromium's, so those cases are refused without `--primary cpu.work`. In Firefox the mock server publishes no events
after a mutation (it has no Playwright page to push them through); `eventSource` still delivers.
- `run` is measured; `reset` puts the state back between runs and is not. A case that should pay a first visit every
run reloads in `reset`; park the pointer with `page.hover(...)` afterwards, because the app prefetches a tab's
transcript when the pointer rests on it, and the pointer stays where the last click left it.
+7 -5
View File
@@ -17,9 +17,10 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1440, height: 900 },
async prepare({ case: name, playwright, mock, url }) {
async prepare(ctx) {
const { case: name, url } = ctx
const server = new URL(url).origin
const page = cached(playwright.page, mock)
const page = cached(ctx)
if (!name.includes("history")) {
await countDraftEncodes(page)
@@ -27,11 +28,11 @@ export default bench({
return mockLargeDraft(page, server)
}
await seedHistory(page, server)
await mockStressTimeline(page, { server })
await installProjects(page, [fixture.directory])
return { path: "/" }
// A static file first: setup seeds the history before the app opens its store.
return { path: "/favicon.svg" }
},
url: ({ prepared }) => prepared.path,
}),
@@ -39,7 +40,8 @@ export default bench({
async setup({ case: name, page }) {
if (name.includes("history")) {
await page.waitFor('[data-action="home-new-session"]', { timeout: 60_000 })
await page.eval(seedHistory)
await page.goto("/", { until: '[data-action="home-new-session"]', timeout: 60_000 })
await page.idle()
return
+3 -2
View File
@@ -25,8 +25,9 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1366, height: 768 },
async prepare({ case: name, playwright, mock, url }) {
const page = cached(playwright.page, mock)
async prepare(ctx) {
const { case: name, url } = ctx
const page = cached(ctx)
const session = await mockTimelineSession(page, { server: new URL(url).origin, historyTurns: 0 })
await serveDiffs(page, changes(name))
+3 -3
View File
@@ -23,9 +23,9 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1366, height: 768 },
prepare: ({ playwright, mock, url }) =>
mockTimelineSession(cached(playwright.page, mock), {
server: new URL(url).origin,
prepare: (ctx) =>
mockTimelineSession(cached(ctx), {
server: new URL(ctx.url).origin,
historyTurns: 20,
stream: true,
}),
+77 -80
View File
@@ -82,88 +82,85 @@ export const recalled = "Review the retry policy in src/network/request-0.ts."
/**
* A long prompt history in the browser's draft store: 100 prompts of about 460 characters with a code fence, every
* other one with a 1440×900 screenshot (five distinct images), and 100 shell commands. Seeded on the app's origin
* before the app first opens the store, from a static file so the app does not start.
* other one with a 1440×900 screenshot (five distinct images), and 100 shell commands. Runs in the page (`page.eval`)
* on the app's origin before the app first opens the store: from a static file, so the app has not started.
*/
export async function seedHistory(page: Page, origin: string) {
await page.goto(new URL("/favicon.svg", origin).href)
await page.evaluate(async () => {
const normal = Array.from({ length: 100 }, (_, index) => {
const content =
`Review the retry policy in src/network/request-${index}.ts. Preserve cancellation and the existing error messages.\n\n` +
`The request should stop after three attempts. Add coverage for a 429 response, a connection reset, and a successful retry. Verify that only idempotent requests are retried.\n\n` +
`Report ${index}:\n\`\`\`ts\nexport async function request(input: Request) {\n const response = await fetch(input)\n if (!response.ok) throw new Error(response.statusText)\n return response.json()\n}\n\`\`\``
export async function seedHistory() {
const normal = Array.from({ length: 100 }, (_, index) => {
const content =
`Review the retry policy in src/network/request-${index}.ts. Preserve cancellation and the existing error messages.\n\n` +
`The request should stop after three attempts. Add coverage for a 429 response, a connection reset, and a successful retry. Verify that only idempotent requests are retried.\n\n` +
`Report ${index}:\n\`\`\`ts\nexport async function request(input: Request) {\n const response = await fetch(input)\n if (!response.ok) throw new Error(response.statusText)\n return response.json()\n}\n\`\`\``
return {
prompt: [
{ type: "text", content, start: 0, end: content.length },
...(index % 2 === 0
? [
{
type: "image",
id: `attachment-${index}`,
filename: `request-${index}.png`,
mime: "image/png",
blob: { id: `screenshot-${index % 10}` },
},
]
: []),
],
comments: [],
}
})
const shell = Array.from({ length: 100 }, (_, index) => {
const content = `bun test src/network/request-${index}.test.ts --timeout 30000`
return { prompt: [{ type: "text", content, start: 0, end: content.length }], comments: [] }
})
const screenshots = await Promise.all(
[0, 2, 4, 6, 8].map(async (index) => {
const id = `screenshot-${index}`
const canvas = new OffscreenCanvas(1440, 900)
const context = canvas.getContext("2d")!
context.fillStyle = "#15191f"
context.fillRect(0, 0, 1440, 900)
context.font = "16px monospace"
context.fillStyle = "#b8c8d8"
context.fillText(`request.ts - ${id}`, 30, 35)
for (let line = 0; line < 38; line++) {
context.fillStyle = line % 3 ? "#a8c7ba" : "#d4a882"
context.fillText(
`${String(line + 1).padStart(3)} const response${line} = await fetch('/api/request/${id}/${line}', { signal, headers });`,
30,
70 + line * 20,
)
}
return { id, blob: await canvas.convertToBlob({ type: "image/png" }) }
}),
)
const open = indexedDB.open("opencode-drafts", 1)
open.onupgradeneeded = () => {
open.result.createObjectStore("documents")
open.result.createObjectStore("blobs")
return {
prompt: [
{ type: "text", content, start: 0, end: content.length },
...(index % 2 === 0
? [
{
type: "image",
id: `attachment-${index}`,
filename: `request-${index}.png`,
mime: "image/png",
blob: { id: `screenshot-${index % 10}` },
},
]
: []),
],
comments: [],
}
const db = await new Promise<IDBDatabase>((resolve, reject) => {
open.onsuccess = () => resolve(open.result)
open.onerror = () => reject(open.error)
})
const transaction = db.transaction(["documents", "blobs"], "readwrite")
transaction.objectStore("documents").put(JSON.stringify({ entries: normal }), "opencode.global.dat:prompt-history")
transaction
.objectStore("documents")
.put(JSON.stringify({ entries: shell }), "opencode.global.dat:prompt-history-shell")
screenshots.forEach((screenshot) => transaction.objectStore("blobs").put(screenshot.blob, screenshot.id))
await new Promise<void>((resolve, reject) => {
transaction.oncomplete = () => resolve()
transaction.onerror = () => reject(transaction.error)
})
db.close()
})
const shell = Array.from({ length: 100 }, (_, index) => {
const content = `bun test src/network/request-${index}.test.ts --timeout 30000`
return { prompt: [{ type: "text", content, start: 0, end: content.length }], comments: [] }
})
const screenshots = await Promise.all(
[0, 2, 4, 6, 8].map(async (index) => {
const id = `screenshot-${index}`
const canvas = new OffscreenCanvas(1440, 900)
const context = canvas.getContext("2d")!
context.fillStyle = "#15191f"
context.fillRect(0, 0, 1440, 900)
context.font = "16px monospace"
context.fillStyle = "#b8c8d8"
context.fillText(`request.ts - ${id}`, 30, 35)
for (let line = 0; line < 38; line++) {
context.fillStyle = line % 3 ? "#a8c7ba" : "#d4a882"
context.fillText(
`${String(line + 1).padStart(3)} const response${line} = await fetch('/api/request/${id}/${line}', { signal, headers });`,
30,
70 + line * 20,
)
}
return { id, blob: await canvas.convertToBlob({ type: "image/png" }) }
}),
)
const open = indexedDB.open("opencode-drafts", 1)
open.onupgradeneeded = () => {
open.result.createObjectStore("documents")
open.result.createObjectStore("blobs")
}
const db = await new Promise<IDBDatabase>((resolve, reject) => {
open.onsuccess = () => resolve(open.result)
open.onerror = () => reject(open.error)
})
const transaction = db.transaction(["documents", "blobs"], "readwrite")
transaction.objectStore("documents").put(JSON.stringify({ entries: normal }), "opencode.global.dat:prompt-history")
transaction
.objectStore("documents")
.put(JSON.stringify({ entries: shell }), "opencode.global.dat:prompt-history-shell")
screenshots.forEach((screenshot) => transaction.objectStore("blobs").put(screenshot.blob, screenshot.id))
await new Promise<void>((resolve, reject) => {
transaction.oncomplete = () => resolve()
transaction.onerror = () => reject(transaction.error)
})
db.close()
}
+17 -49
View File
@@ -1,7 +1,7 @@
import type { Page } from "@playwright/test"
import type { Page } from "@hona/strobe"
/** A divider to drag: its centre, how far it can travel each way (px), and the display's frame period (ms). */
export type Divider = { x: number; y: number; left: number; right: number; period: number }
/** A divider to drag: how far it can travel each way (px). */
export type Divider = { left: number; right: number }
type Leg = { seconds: number; at: (progress: number) => number }
@@ -36,13 +36,11 @@ function legs(divider: Divider) {
}
/**
* Presses the divider, moves the pointer once per display frame along the drag above without letting go, and releases
* it where it started. strobe's driver has no pointer drag (`page.move` does not hold a button, and its `until:
* "frame"` waits two animation frames and a round trip per move), so Playwright's mouse drives the drag, from the
* runner, paced by the display's frame period on the wall clock. A frame the page misses is skipped, as a hand keeps
* moving while the screen lags.
* Presses the divider (`selector`), moves the pointer along the drag above without letting go, and releases it where
* it started. strobe's `page.drag` sends one move per display frame on its own clock, without waiting for the page to
* handle each, as a hand keeps moving while the screen lags; `input.rate` says how evenly the moves went out.
*/
export async function dragDivider(mouse: Page["mouse"], divider: Divider) {
export function dragDivider(page: Page, selector: string, divider: Divider) {
const path = legs(divider)
const total = path.reduce((sum, leg) => sum + leg.seconds, 0)
@@ -57,49 +55,19 @@ export async function dragDivider(mouse: Page["mouse"], divider: Divider) {
return 0
}
await mouse.move(divider.x, divider.y)
await mouse.down()
const began = performance.now()
for (let frame = 1; ; frame++) {
const wait = began + frame * divider.period - performance.now()
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait))
frame = Math.max(frame, Math.floor((performance.now() - began) / divider.period))
const seconds = (performance.now() - began) / 1000
if (seconds >= total) break
await mouse.move(divider.x + offset(seconds), divider.y)
}
await mouse.move(divider.x, divider.y)
await mouse.up()
return page.drag(selector, (progress) => [offset(progress * total), 0], { duration: total * 1000 })
}
/**
* Reads, in the page, where the divider `selector` sits, the room its panel `panel` has to shrink to `min` and grow to
* the row `row` less `rest` (px), and the display's frame period over 30 frames. Only in setup: it reads layout.
* Reads, in the page, the room the panel `panel` beside the divider `selector` has to shrink to `min` and grow to the
* row `row` less `rest` (px). Only in setup: it reads layout.
*/
export function measureDivider(input: { selector: string; panel: string; row: string; min: number; rest: number }) {
return new Promise<Divider>((resolve) => {
const box = document.querySelector(input.selector)!.getBoundingClientRect()
const width = document.querySelector(input.panel)!.getBoundingClientRect().width
const row = document.querySelector(input.row)!.getBoundingClientRect().width
const times: number[] = []
export function measureDivider(input: { panel: string; row: string; min: number; rest: number }): Divider {
const width = document.querySelector(input.panel)!.getBoundingClientRect().width
const row = document.querySelector(input.row)!.getBoundingClientRect().width
const tick = (time: number) => {
times.push(time)
if (times.length < 31) return requestAnimationFrame(tick)
resolve({
x: box.left + box.width / 2,
y: box.top + box.height / 2,
left: Math.max(0, Math.min(360, width - input.min - 10)),
right: Math.max(0, Math.min(160, row - input.rest - width - 10)),
period: (times[30]! - times[0]!) / 30,
})
}
requestAnimationFrame(tick)
})
return {
left: Math.max(0, Math.min(360, width - input.min - 10)),
right: Math.max(0, Math.min(160, row - input.rest - width - 10)),
}
}
+40 -6
View File
@@ -12,13 +12,19 @@ type Fulfill = {
}
/**
* `page`, with its routes answered through strobe's `mock` (`prepare`'s `ctx.mock`). Playwright's routing turns the
* browser's HTTP cache off for the whole page, so a case that reloads in `reset` would refetch every script, which no
* person does; `mock` pauses only the mocked URLs. Everything else on the page is Playwright's, so the e2e mock server
* and its fixtures run unchanged: routes keep Playwright's order (the latest answers first, `fallback` passes on).
* Routes take URL globs and last for the whole case.
* The block's page for the e2e mock server and the fixtures, with its routes answered through strobe's `mock`
* (`prepare`'s `ctx.mock`). Playwright's routing turns the browser's HTTP cache off for the whole page, so a case that
* reloads in `reset` would refetch every script, which no person does; `mock` pauses only the mocked URLs. Routes keep
* Playwright's order (the latest answers first, `fallback` passes on), take URL globs and last for the whole case.
*
* In Chromium it is Playwright's page, so the mock server and its fixtures run as in the e2e suites. Other engines
* (`--engine firefox`) have no Playwright page: a stand-in takes init scripts through strobe's `addInitScript` and
* cannot evaluate in the page, so the mock server publishes no events there after a mutation.
*/
export function cached(page: Page, mock: PrepareContext<unknown>["mock"]) {
export function cached(ctx: Pick<PrepareContext<unknown>, "playwright" | "mock" | "addInitScript">) {
const page = playwrightPage(ctx) ?? standIn(ctx.addInitScript)
const { mock } = ctx
page.route = async (url, handler) => {
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- discriminates Playwright's own URL matcher union
if (typeof url !== "string") throw new Error("strobe mocks URL globs; no benchmark routes by RegExp or predicate")
@@ -34,6 +40,34 @@ export function cached(page: Page, mock: PrepareContext<unknown>["mock"]) {
return page
}
// `ctx.playwright` is Chromium's: reading it in another engine throws.
function playwrightPage(ctx: Pick<PrepareContext<unknown>, "playwright">): Page | undefined {
try {
// SAFETY: strobe hands Chromium blocks Playwright's own page.
return ctx.playwright.page as Page
} catch {
return undefined
}
}
// The members of a page the e2e mock server and the fixtures use, over strobe's calls. `route` is set by `cached`.
function standIn(addInitScript: PrepareContext<unknown>["addInitScript"]) {
const page = {
addInitScript,
// The block's browser closes with the block.
on: () => page,
isClosed: () => false,
// The mock server's event delivery treats a missing stream as retryable and drops the events.
evaluate: async () => {
throw new Error("No mock event stream: strobe drives this engine without Playwright")
},
}
// SAFETY: the e2e mock server and the benchmark fixtures call only these members (and `route`, `routeWebSocket`).
// oxlint-disable-next-line anti-slop/no-chained-type-assertions -- see SAFETY above
return page as unknown as Page
}
async function kept() {
throw new Error("a strobe mock cannot be removed")
}
+3 -2
View File
@@ -19,9 +19,10 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1440, height: 900 },
async prepare({ params, playwright, mock, url }) {
async prepare(ctx) {
const { params, url } = ctx
const server = new URL(url).origin
const page = cached(playwright.page, mock)
const page = cached(ctx)
const source = await eventSource(page, server)
const home = await mockHome(page, { server, count: params.sessions })
+3 -2
View File
@@ -104,10 +104,11 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1440, height: 900 },
async prepare({ case: name, playwright, mock, url }) {
async prepare(ctx) {
const { case: name, url } = ctx
const server = new URL(url).origin
const scene = scenes.get(name)
const page = cached(playwright.page, mock)
const page = cached(ctx)
if (!scene) throw new Error(`navigation.bench.ts: no scene for "${name}"`)
+3 -2
View File
@@ -17,9 +17,10 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1440, height: 900 },
async prepare({ playwright, mock, url }) {
async prepare(ctx) {
const { url } = ctx
const server = new URL(url).origin
const page = cached(playwright.page, mock)
const page = cached(ctx)
await mockOpenCodeServer(page, {
server,
+3 -2
View File
@@ -14,8 +14,9 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1366, height: 768 },
async prepare({ params, playwright, mock, url }) {
const page = cached(playwright.page, mock)
async prepare(ctx) {
const { params, url } = ctx
const page = cached(ctx)
const session = await mockTimelineSession(page, { server: new URL(url).origin, historyTurns: 0 })
// The app asks for the changes only when the pane opens.
await serveDiffs(page, scalingDiffs(params.files))
+6 -6
View File
@@ -50,14 +50,14 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1366, height: 768 },
prepare: ({ case: name, playwright, mock, url }) =>
mockTimelineSession(cached(playwright.page, mock), {
server: new URL(url).origin,
historyTurns: name === "stream into a tool-heavy history" ? 320 : 120,
toolHeavy: name === "stream into a tool-heavy history",
prepare: (ctx) =>
mockTimelineSession(cached(ctx), {
server: new URL(ctx.url).origin,
historyTurns: ctx.case === "stream into a tool-heavy history" ? 320 : 120,
toolHeavy: ctx.case === "stream into a tool-heavy history",
busy: true,
stream: true,
vcsDiff: name === "stream with review open" ? reviewDiffs() : undefined,
vcsDiff: ctx.case === "stream with review open" ? reviewDiffs() : undefined,
}),
url: ({ prepared }) => prepared.path,
}),
+3 -2
View File
@@ -99,9 +99,10 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1440, height: 900 },
async prepare({ case: name, playwright, mock, url }) {
async prepare(ctx) {
const { case: name, url } = ctx
const server = new URL(url).origin
const page = cached(playwright.page, mock)
const page = cached(ctx)
if (exported && name.endsWith("your session")) return mockExport(page, server, exported)
await mockOpenCodeServer(page, {
+11 -12
View File
@@ -19,6 +19,8 @@ const drag = "drag the review's edge"
// The divider between the session and the side panel, read once the review is open. The session panel keeps at least
// 400px, the side panel 320px and the 8px gap.
const handle = '[data-slot="session-chat-panel"] > div > [data-component="resize-handle"]'
let divider: Divider | undefined
export default bench({
@@ -26,17 +28,15 @@ export default bench({
web: web({
expect: { title: "OpenCode" },
viewport: { width: 1366, height: 768 },
prepare: async ({ case: name, playwright, mock, url }) => ({
...(await mockTimelineSession(cached(playwright.page, mock), {
server: new URL(url).origin,
prepare: async (ctx) =>
mockTimelineSession(cached(ctx), {
server: new URL(ctx.url).origin,
// The edits alone, so the case measures the diff, not the history around it.
historyTurns: name === "expand an edit" ? 0 : 120,
collapsedEdits: name === "expand an edit",
tools: name === "expand an edit" ? await editTools() : undefined,
vcsDiff: name === drag ? reviewDiffs() : undefined,
})),
mouse: playwright.page.mouse,
}),
historyTurns: ctx.case === "expand an edit" ? 0 : 120,
collapsedEdits: ctx.case === "expand an edit",
tools: ctx.case === "expand an edit" ? await editTools() : undefined,
vcsDiff: ctx.case === drag ? reviewDiffs() : undefined,
}),
url: ({ prepared }) => prepared.path,
}),
},
@@ -49,7 +49,6 @@ export default bench({
await openReview(page)
await page.idle()
divider = await page.eval(measureDivider, {
selector: '[data-slot="session-chat-panel"] > div > [data-component="resize-handle"]',
panel: '[data-slot="session-chat-panel"]',
row: "[data-pane-row]",
min: 400,
@@ -118,7 +117,7 @@ export default bench({
[drag]: {
primary: "frames.dropped.pct",
metrics: ["main.work", "main.layout", "main.style", "main.script", "frames.interval.p95"],
run: ({ prepared }) => dragDivider(prepared.mouse, divider!),
run: ({ page }) => dragDivider(page, handle, divider!),
reset: ({ page }) => page.idle(),
},
},