diff --git a/packages/app/perf/README.md b/packages/app/perf/README.md index a9295b22425..9246015d7db 100644 --- a/packages/app/perf/README.md +++ b/packages/app/perf/README.md @@ -140,12 +140,17 @@ strobe bench packages/app/perf/timeline.bench.ts --filter "scroll*" --bisect --engine firefox --headless --check --primary cpu.work`. `main.*` metrics are + Chromium's, so those cases are refused without `--primary cpu.work`. In Firefox the mock server publishes no events + after a mutation (it has no Playwright page to push them through); `eventSource` still delivers. - `run` is measured; `reset` puts the state back between runs and is not. A case that should pay a first visit every run reloads in `reset`; park the pointer with `page.hover(...)` afterwards, because the app prefetches a tab's transcript when the pointer rests on it, and the pointer stays where the last click left it. diff --git a/packages/app/perf/composer.bench.ts b/packages/app/perf/composer.bench.ts index ae9a93f5d31..66abc955e34 100644 --- a/packages/app/perf/composer.bench.ts +++ b/packages/app/perf/composer.bench.ts @@ -17,9 +17,10 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1440, height: 900 }, - async prepare({ case: name, playwright, mock, url }) { + async prepare(ctx) { + const { case: name, url } = ctx const server = new URL(url).origin - const page = cached(playwright.page, mock) + const page = cached(ctx) if (!name.includes("history")) { await countDraftEncodes(page) @@ -27,11 +28,11 @@ export default bench({ return mockLargeDraft(page, server) } - await seedHistory(page, server) await mockStressTimeline(page, { server }) await installProjects(page, [fixture.directory]) - return { path: "/" } + // A static file first: setup seeds the history before the app opens its store. + return { path: "/favicon.svg" } }, url: ({ prepared }) => prepared.path, }), @@ -39,7 +40,8 @@ export default bench({ async setup({ case: name, page }) { if (name.includes("history")) { - await page.waitFor('[data-action="home-new-session"]', { timeout: 60_000 }) + await page.eval(seedHistory) + await page.goto("/", { until: '[data-action="home-new-session"]', timeout: 60_000 }) await page.idle() return diff --git a/packages/app/perf/diff.bench.ts b/packages/app/perf/diff.bench.ts index b0452c4eeae..57df3f38f30 100644 --- a/packages/app/perf/diff.bench.ts +++ b/packages/app/perf/diff.bench.ts @@ -25,8 +25,9 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1366, height: 768 }, - async prepare({ case: name, playwright, mock, url }) { - const page = cached(playwright.page, mock) + async prepare(ctx) { + const { case: name, url } = ctx + const page = cached(ctx) const session = await mockTimelineSession(page, { server: new URL(url).origin, historyTurns: 0 }) await serveDiffs(page, changes(name)) diff --git a/packages/app/perf/docks.bench.ts b/packages/app/perf/docks.bench.ts index 1d853b8095b..428341b329c 100644 --- a/packages/app/perf/docks.bench.ts +++ b/packages/app/perf/docks.bench.ts @@ -23,9 +23,9 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1366, height: 768 }, - prepare: ({ playwright, mock, url }) => - mockTimelineSession(cached(playwright.page, mock), { - server: new URL(url).origin, + prepare: (ctx) => + mockTimelineSession(cached(ctx), { + server: new URL(ctx.url).origin, historyTurns: 20, stream: true, }), diff --git a/packages/app/perf/fixtures/composer.ts b/packages/app/perf/fixtures/composer.ts index ff49c977e97..670ddee1980 100644 --- a/packages/app/perf/fixtures/composer.ts +++ b/packages/app/perf/fixtures/composer.ts @@ -82,88 +82,85 @@ export const recalled = "Review the retry policy in src/network/request-0.ts." /** * A long prompt history in the browser's draft store: 100 prompts of about 460 characters with a code fence, every - * other one with a 1440×900 screenshot (five distinct images), and 100 shell commands. Seeded on the app's origin - * before the app first opens the store, from a static file so the app does not start. + * other one with a 1440×900 screenshot (five distinct images), and 100 shell commands. Runs in the page (`page.eval`) + * on the app's origin before the app first opens the store: from a static file, so the app has not started. */ -export async function seedHistory(page: Page, origin: string) { - await page.goto(new URL("/favicon.svg", origin).href) - await page.evaluate(async () => { - const normal = Array.from({ length: 100 }, (_, index) => { - const content = - `Review the retry policy in src/network/request-${index}.ts. Preserve cancellation and the existing error messages.\n\n` + - `The request should stop after three attempts. Add coverage for a 429 response, a connection reset, and a successful retry. Verify that only idempotent requests are retried.\n\n` + - `Report ${index}:\n\`\`\`ts\nexport async function request(input: Request) {\n const response = await fetch(input)\n if (!response.ok) throw new Error(response.statusText)\n return response.json()\n}\n\`\`\`` +export async function seedHistory() { + const normal = Array.from({ length: 100 }, (_, index) => { + const content = + `Review the retry policy in src/network/request-${index}.ts. Preserve cancellation and the existing error messages.\n\n` + + `The request should stop after three attempts. Add coverage for a 429 response, a connection reset, and a successful retry. Verify that only idempotent requests are retried.\n\n` + + `Report ${index}:\n\`\`\`ts\nexport async function request(input: Request) {\n const response = await fetch(input)\n if (!response.ok) throw new Error(response.statusText)\n return response.json()\n}\n\`\`\`` - return { - prompt: [ - { type: "text", content, start: 0, end: content.length }, - ...(index % 2 === 0 - ? [ - { - type: "image", - id: `attachment-${index}`, - filename: `request-${index}.png`, - mime: "image/png", - blob: { id: `screenshot-${index % 10}` }, - }, - ] - : []), - ], - comments: [], - } - }) - - const shell = Array.from({ length: 100 }, (_, index) => { - const content = `bun test src/network/request-${index}.test.ts --timeout 30000` - - return { prompt: [{ type: "text", content, start: 0, end: content.length }], comments: [] } - }) - - const screenshots = await Promise.all( - [0, 2, 4, 6, 8].map(async (index) => { - const id = `screenshot-${index}` - const canvas = new OffscreenCanvas(1440, 900) - const context = canvas.getContext("2d")! - context.fillStyle = "#15191f" - context.fillRect(0, 0, 1440, 900) - context.font = "16px monospace" - context.fillStyle = "#b8c8d8" - context.fillText(`request.ts - ${id}`, 30, 35) - - for (let line = 0; line < 38; line++) { - context.fillStyle = line % 3 ? "#a8c7ba" : "#d4a882" - context.fillText( - `${String(line + 1).padStart(3)} const response${line} = await fetch('/api/request/${id}/${line}', { signal, headers });`, - 30, - 70 + line * 20, - ) - } - - return { id, blob: await canvas.convertToBlob({ type: "image/png" }) } - }), - ) - - const open = indexedDB.open("opencode-drafts", 1) - open.onupgradeneeded = () => { - open.result.createObjectStore("documents") - open.result.createObjectStore("blobs") + return { + prompt: [ + { type: "text", content, start: 0, end: content.length }, + ...(index % 2 === 0 + ? [ + { + type: "image", + id: `attachment-${index}`, + filename: `request-${index}.png`, + mime: "image/png", + blob: { id: `screenshot-${index % 10}` }, + }, + ] + : []), + ], + comments: [], } - - const db = await new Promise((resolve, reject) => { - open.onsuccess = () => resolve(open.result) - open.onerror = () => reject(open.error) - }) - - const transaction = db.transaction(["documents", "blobs"], "readwrite") - transaction.objectStore("documents").put(JSON.stringify({ entries: normal }), "opencode.global.dat:prompt-history") - transaction - .objectStore("documents") - .put(JSON.stringify({ entries: shell }), "opencode.global.dat:prompt-history-shell") - screenshots.forEach((screenshot) => transaction.objectStore("blobs").put(screenshot.blob, screenshot.id)) - await new Promise((resolve, reject) => { - transaction.oncomplete = () => resolve() - transaction.onerror = () => reject(transaction.error) - }) - db.close() }) + + const shell = Array.from({ length: 100 }, (_, index) => { + const content = `bun test src/network/request-${index}.test.ts --timeout 30000` + + return { prompt: [{ type: "text", content, start: 0, end: content.length }], comments: [] } + }) + + const screenshots = await Promise.all( + [0, 2, 4, 6, 8].map(async (index) => { + const id = `screenshot-${index}` + const canvas = new OffscreenCanvas(1440, 900) + const context = canvas.getContext("2d")! + context.fillStyle = "#15191f" + context.fillRect(0, 0, 1440, 900) + context.font = "16px monospace" + context.fillStyle = "#b8c8d8" + context.fillText(`request.ts - ${id}`, 30, 35) + + for (let line = 0; line < 38; line++) { + context.fillStyle = line % 3 ? "#a8c7ba" : "#d4a882" + context.fillText( + `${String(line + 1).padStart(3)} const response${line} = await fetch('/api/request/${id}/${line}', { signal, headers });`, + 30, + 70 + line * 20, + ) + } + + return { id, blob: await canvas.convertToBlob({ type: "image/png" }) } + }), + ) + + const open = indexedDB.open("opencode-drafts", 1) + open.onupgradeneeded = () => { + open.result.createObjectStore("documents") + open.result.createObjectStore("blobs") + } + + const db = await new Promise((resolve, reject) => { + open.onsuccess = () => resolve(open.result) + open.onerror = () => reject(open.error) + }) + + const transaction = db.transaction(["documents", "blobs"], "readwrite") + transaction.objectStore("documents").put(JSON.stringify({ entries: normal }), "opencode.global.dat:prompt-history") + transaction + .objectStore("documents") + .put(JSON.stringify({ entries: shell }), "opencode.global.dat:prompt-history-shell") + screenshots.forEach((screenshot) => transaction.objectStore("blobs").put(screenshot.blob, screenshot.id)) + await new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve() + transaction.onerror = () => reject(transaction.error) + }) + db.close() } diff --git a/packages/app/perf/fixtures/drag.ts b/packages/app/perf/fixtures/drag.ts index 47837f2bb09..0947f28391d 100644 --- a/packages/app/perf/fixtures/drag.ts +++ b/packages/app/perf/fixtures/drag.ts @@ -1,7 +1,7 @@ -import type { Page } from "@playwright/test" +import type { Page } from "@hona/strobe" -/** A divider to drag: its centre, how far it can travel each way (px), and the display's frame period (ms). */ -export type Divider = { x: number; y: number; left: number; right: number; period: number } +/** A divider to drag: how far it can travel each way (px). */ +export type Divider = { left: number; right: number } type Leg = { seconds: number; at: (progress: number) => number } @@ -36,13 +36,11 @@ function legs(divider: Divider) { } /** - * Presses the divider, moves the pointer once per display frame along the drag above without letting go, and releases - * it where it started. strobe's driver has no pointer drag (`page.move` does not hold a button, and its `until: - * "frame"` waits two animation frames and a round trip per move), so Playwright's mouse drives the drag, from the - * runner, paced by the display's frame period on the wall clock. A frame the page misses is skipped, as a hand keeps - * moving while the screen lags. + * Presses the divider (`selector`), moves the pointer along the drag above without letting go, and releases it where + * it started. strobe's `page.drag` sends one move per display frame on its own clock, without waiting for the page to + * handle each, as a hand keeps moving while the screen lags; `input.rate` says how evenly the moves went out. */ -export async function dragDivider(mouse: Page["mouse"], divider: Divider) { +export function dragDivider(page: Page, selector: string, divider: Divider) { const path = legs(divider) const total = path.reduce((sum, leg) => sum + leg.seconds, 0) @@ -57,49 +55,19 @@ export async function dragDivider(mouse: Page["mouse"], divider: Divider) { return 0 } - await mouse.move(divider.x, divider.y) - await mouse.down() - const began = performance.now() - - for (let frame = 1; ; frame++) { - const wait = began + frame * divider.period - performance.now() - - if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)) - frame = Math.max(frame, Math.floor((performance.now() - began) / divider.period)) - const seconds = (performance.now() - began) / 1000 - - if (seconds >= total) break - await mouse.move(divider.x + offset(seconds), divider.y) - } - - await mouse.move(divider.x, divider.y) - await mouse.up() + return page.drag(selector, (progress) => [offset(progress * total), 0], { duration: total * 1000 }) } /** - * Reads, in the page, where the divider `selector` sits, the room its panel `panel` has to shrink to `min` and grow to - * the row `row` less `rest` (px), and the display's frame period over 30 frames. Only in setup: it reads layout. + * Reads, in the page, the room the panel `panel` beside the divider `selector` has to shrink to `min` and grow to the + * row `row` less `rest` (px). Only in setup: it reads layout. */ -export function measureDivider(input: { selector: string; panel: string; row: string; min: number; rest: number }) { - return new Promise((resolve) => { - const box = document.querySelector(input.selector)!.getBoundingClientRect() - const width = document.querySelector(input.panel)!.getBoundingClientRect().width - const row = document.querySelector(input.row)!.getBoundingClientRect().width - const times: number[] = [] +export function measureDivider(input: { panel: string; row: string; min: number; rest: number }): Divider { + const width = document.querySelector(input.panel)!.getBoundingClientRect().width + const row = document.querySelector(input.row)!.getBoundingClientRect().width - const tick = (time: number) => { - times.push(time) - - if (times.length < 31) return requestAnimationFrame(tick) - resolve({ - x: box.left + box.width / 2, - y: box.top + box.height / 2, - left: Math.max(0, Math.min(360, width - input.min - 10)), - right: Math.max(0, Math.min(160, row - input.rest - width - 10)), - period: (times[30]! - times[0]!) / 30, - }) - } - - requestAnimationFrame(tick) - }) + return { + left: Math.max(0, Math.min(360, width - input.min - 10)), + right: Math.max(0, Math.min(160, row - input.rest - width - 10)), + } } diff --git a/packages/app/perf/fixtures/mock.ts b/packages/app/perf/fixtures/mock.ts index 561c9b1e04a..d309b90a527 100644 --- a/packages/app/perf/fixtures/mock.ts +++ b/packages/app/perf/fixtures/mock.ts @@ -12,13 +12,19 @@ type Fulfill = { } /** - * `page`, with its routes answered through strobe's `mock` (`prepare`'s `ctx.mock`). Playwright's routing turns the - * browser's HTTP cache off for the whole page, so a case that reloads in `reset` would refetch every script, which no - * person does; `mock` pauses only the mocked URLs. Everything else on the page is Playwright's, so the e2e mock server - * and its fixtures run unchanged: routes keep Playwright's order (the latest answers first, `fallback` passes on). - * Routes take URL globs and last for the whole case. + * The block's page for the e2e mock server and the fixtures, with its routes answered through strobe's `mock` + * (`prepare`'s `ctx.mock`). Playwright's routing turns the browser's HTTP cache off for the whole page, so a case that + * reloads in `reset` would refetch every script, which no person does; `mock` pauses only the mocked URLs. Routes keep + * Playwright's order (the latest answers first, `fallback` passes on), take URL globs and last for the whole case. + * + * In Chromium it is Playwright's page, so the mock server and its fixtures run as in the e2e suites. Other engines + * (`--engine firefox`) have no Playwright page: a stand-in takes init scripts through strobe's `addInitScript` and + * cannot evaluate in the page, so the mock server publishes no events there after a mutation. */ -export function cached(page: Page, mock: PrepareContext["mock"]) { +export function cached(ctx: Pick, "playwright" | "mock" | "addInitScript">) { + const page = playwrightPage(ctx) ?? standIn(ctx.addInitScript) + const { mock } = ctx + page.route = async (url, handler) => { // oxlint-disable-next-line anti-slop/no-runtime-typeof -- discriminates Playwright's own URL matcher union if (typeof url !== "string") throw new Error("strobe mocks URL globs; no benchmark routes by RegExp or predicate") @@ -34,6 +40,34 @@ export function cached(page: Page, mock: PrepareContext["mock"]) { return page } +// `ctx.playwright` is Chromium's: reading it in another engine throws. +function playwrightPage(ctx: Pick, "playwright">): Page | undefined { + try { + // SAFETY: strobe hands Chromium blocks Playwright's own page. + return ctx.playwright.page as Page + } catch { + return undefined + } +} + +// The members of a page the e2e mock server and the fixtures use, over strobe's calls. `route` is set by `cached`. +function standIn(addInitScript: PrepareContext["addInitScript"]) { + const page = { + addInitScript, + // The block's browser closes with the block. + on: () => page, + isClosed: () => false, + // The mock server's event delivery treats a missing stream as retryable and drops the events. + evaluate: async () => { + throw new Error("No mock event stream: strobe drives this engine without Playwright") + }, + } + + // SAFETY: the e2e mock server and the benchmark fixtures call only these members (and `route`, `routeWebSocket`). + // oxlint-disable-next-line anti-slop/no-chained-type-assertions -- see SAFETY above + return page as unknown as Page +} + async function kept() { throw new Error("a strobe mock cannot be removed") } diff --git a/packages/app/perf/home.bench.ts b/packages/app/perf/home.bench.ts index fde36c8772a..91dec6add5c 100644 --- a/packages/app/perf/home.bench.ts +++ b/packages/app/perf/home.bench.ts @@ -19,9 +19,10 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1440, height: 900 }, - async prepare({ params, playwright, mock, url }) { + async prepare(ctx) { + const { params, url } = ctx const server = new URL(url).origin - const page = cached(playwright.page, mock) + const page = cached(ctx) const source = await eventSource(page, server) const home = await mockHome(page, { server, count: params.sessions }) diff --git a/packages/app/perf/navigation.bench.ts b/packages/app/perf/navigation.bench.ts index d59f1dce7b1..4c6ceec3e2d 100644 --- a/packages/app/perf/navigation.bench.ts +++ b/packages/app/perf/navigation.bench.ts @@ -104,10 +104,11 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1440, height: 900 }, - async prepare({ case: name, playwright, mock, url }) { + async prepare(ctx) { + const { case: name, url } = ctx const server = new URL(url).origin const scene = scenes.get(name) - const page = cached(playwright.page, mock) + const page = cached(ctx) if (!scene) throw new Error(`navigation.bench.ts: no scene for "${name}"`) diff --git a/packages/app/perf/question.bench.ts b/packages/app/perf/question.bench.ts index b95c8c2f9f8..4d357def885 100644 --- a/packages/app/perf/question.bench.ts +++ b/packages/app/perf/question.bench.ts @@ -17,9 +17,10 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1440, height: 900 }, - async prepare({ playwright, mock, url }) { + async prepare(ctx) { + const { url } = ctx const server = new URL(url).origin - const page = cached(playwright.page, mock) + const page = cached(ctx) await mockOpenCodeServer(page, { server, diff --git a/packages/app/perf/review.bench.ts b/packages/app/perf/review.bench.ts index b2d494c1995..6b500c4f3c4 100644 --- a/packages/app/perf/review.bench.ts +++ b/packages/app/perf/review.bench.ts @@ -14,8 +14,9 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1366, height: 768 }, - async prepare({ params, playwright, mock, url }) { - const page = cached(playwright.page, mock) + async prepare(ctx) { + const { params, url } = ctx + const page = cached(ctx) const session = await mockTimelineSession(page, { server: new URL(url).origin, historyTurns: 0 }) // The app asks for the changes only when the pane opens. await serveDiffs(page, scalingDiffs(params.files)) diff --git a/packages/app/perf/stream.bench.ts b/packages/app/perf/stream.bench.ts index b47564deca4..e2202d88b5c 100644 --- a/packages/app/perf/stream.bench.ts +++ b/packages/app/perf/stream.bench.ts @@ -50,14 +50,14 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1366, height: 768 }, - prepare: ({ case: name, playwright, mock, url }) => - mockTimelineSession(cached(playwright.page, mock), { - server: new URL(url).origin, - historyTurns: name === "stream into a tool-heavy history" ? 320 : 120, - toolHeavy: name === "stream into a tool-heavy history", + prepare: (ctx) => + mockTimelineSession(cached(ctx), { + server: new URL(ctx.url).origin, + historyTurns: ctx.case === "stream into a tool-heavy history" ? 320 : 120, + toolHeavy: ctx.case === "stream into a tool-heavy history", busy: true, stream: true, - vcsDiff: name === "stream with review open" ? reviewDiffs() : undefined, + vcsDiff: ctx.case === "stream with review open" ? reviewDiffs() : undefined, }), url: ({ prepared }) => prepared.path, }), diff --git a/packages/app/perf/tabs.bench.ts b/packages/app/perf/tabs.bench.ts index f2eab8299c2..ffb530543aa 100644 --- a/packages/app/perf/tabs.bench.ts +++ b/packages/app/perf/tabs.bench.ts @@ -99,9 +99,10 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1440, height: 900 }, - async prepare({ case: name, playwright, mock, url }) { + async prepare(ctx) { + const { case: name, url } = ctx const server = new URL(url).origin - const page = cached(playwright.page, mock) + const page = cached(ctx) if (exported && name.endsWith("your session")) return mockExport(page, server, exported) await mockOpenCodeServer(page, { diff --git a/packages/app/perf/timeline.bench.ts b/packages/app/perf/timeline.bench.ts index a5a2e968252..9e8ef6f0d0b 100644 --- a/packages/app/perf/timeline.bench.ts +++ b/packages/app/perf/timeline.bench.ts @@ -19,6 +19,8 @@ const drag = "drag the review's edge" // The divider between the session and the side panel, read once the review is open. The session panel keeps at least // 400px, the side panel 320px and the 8px gap. +const handle = '[data-slot="session-chat-panel"] > div > [data-component="resize-handle"]' + let divider: Divider | undefined export default bench({ @@ -26,17 +28,15 @@ export default bench({ web: web({ expect: { title: "OpenCode" }, viewport: { width: 1366, height: 768 }, - prepare: async ({ case: name, playwright, mock, url }) => ({ - ...(await mockTimelineSession(cached(playwright.page, mock), { - server: new URL(url).origin, + prepare: async (ctx) => + mockTimelineSession(cached(ctx), { + server: new URL(ctx.url).origin, // The edits alone, so the case measures the diff, not the history around it. - historyTurns: name === "expand an edit" ? 0 : 120, - collapsedEdits: name === "expand an edit", - tools: name === "expand an edit" ? await editTools() : undefined, - vcsDiff: name === drag ? reviewDiffs() : undefined, - })), - mouse: playwright.page.mouse, - }), + historyTurns: ctx.case === "expand an edit" ? 0 : 120, + collapsedEdits: ctx.case === "expand an edit", + tools: ctx.case === "expand an edit" ? await editTools() : undefined, + vcsDiff: ctx.case === drag ? reviewDiffs() : undefined, + }), url: ({ prepared }) => prepared.path, }), }, @@ -49,7 +49,6 @@ export default bench({ await openReview(page) await page.idle() divider = await page.eval(measureDivider, { - selector: '[data-slot="session-chat-panel"] > div > [data-component="resize-handle"]', panel: '[data-slot="session-chat-panel"]', row: "[data-pane-row]", min: 400, @@ -118,7 +117,7 @@ export default bench({ [drag]: { primary: "frames.dropped.pct", metrics: ["main.work", "main.layout", "main.style", "main.script", "frames.interval.p95"], - run: ({ prepared }) => dragDivider(prepared.mouse, divider!), + run: ({ page }) => dragDivider(page, handle, divider!), reset: ({ page }) => page.idle(), }, },