From 156bb610108c3da91c6963056f439e2acc1614f0 Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:01:07 -0600 Subject: [PATCH] feat(stats): run the stats sync on Cloudflare Workflows (#61461) --- bun.lock | 2 +- infra/stage.ts | 2 +- infra/stats.ts | 58 ++- packages/stats/README.md | 29 +- packages/stats/core/src/config.ts | 5 +- packages/stats/core/src/database.ts | 65 ++- .../stats/core/src/domain/catalog-identity.ts | 15 + packages/stats/core/src/domain/geo.ts | 18 +- packages/stats/core/src/domain/inference.ts | 67 +-- packages/stats/core/src/domain/model.ts | 38 +- packages/stats/core/src/domain/provider.ts | 19 +- packages/stats/core/src/domain/retention.ts | 31 +- packages/stats/core/src/domain/stat.ts | 5 +- packages/stats/core/src/honeycomb-backfill.ts | 2 +- packages/stats/core/src/migrate.ts | 37 +- packages/stats/core/src/r2-sql.test.ts | 35 +- packages/stats/core/src/r2-sql.ts | 52 ++- packages/stats/core/src/stat-sync.test.ts | 267 ++++++++++++ packages/stats/core/src/stat-sync.ts | 392 +++++++++++------- packages/stats/server/Dockerfile | 32 -- packages/stats/server/package.json | 9 +- packages/stats/server/src/cron.test.ts | 78 ++++ packages/stats/server/src/cron.ts | 53 +++ packages/stats/server/src/schedule.test.ts | 97 +++++ packages/stats/server/src/schedule.ts | 43 ++ packages/stats/server/src/stat-sync.ts | 74 ---- packages/stats/server/src/sync.ts | 10 + packages/stats/server/src/workflow.ts | 78 ++++ packages/stats/server/tsconfig.json | 5 +- sst.config.ts | 2 +- 30 files changed, 1175 insertions(+), 445 deletions(-) create mode 100644 packages/stats/core/src/stat-sync.test.ts delete mode 100644 packages/stats/server/Dockerfile create mode 100644 packages/stats/server/src/cron.test.ts create mode 100644 packages/stats/server/src/cron.ts create mode 100644 packages/stats/server/src/schedule.test.ts create mode 100644 packages/stats/server/src/schedule.ts delete mode 100644 packages/stats/server/src/stat-sync.ts create mode 100644 packages/stats/server/src/sync.ts create mode 100644 packages/stats/server/src/workflow.ts diff --git a/bun.lock b/bun.lock index 12da13a8166..eb540209e2b 100644 --- a/bun.lock +++ b/bun.lock @@ -769,12 +769,12 @@ "name": "@opencode/stats-server", "version": "2.2.5", "dependencies": { - "@effect/platform-node": "catalog:", "@opencode/stats-core": "workspace:*", "effect": "catalog:", "sst": "catalog:", }, "devDependencies": { + "@cloudflare/workers-types": "catalog:", "@tsconfig/node22": "catalog:", "@types/bun": "catalog:", "@types/node": "catalog:", diff --git a/infra/stage.ts b/infra/stage.ts index f0db7974485..f4ccb18d80b 100644 --- a/infra/stage.ts +++ b/infra/stage.ts @@ -6,7 +6,7 @@ export const domain = (() => { export const zoneID = "430ba34c138cfb5360826c4909f99be8" export const awsStage = $app.stage === "production" ? "production" : "dev" -export const deployAws = $app.stage === awsStage +export const deployStats = $app.stage === awsStage if ($app.stage === "production") { new cloudflare.DnsRecord("TrustCenter", { diff --git a/infra/stats.ts b/infra/stats.ts index 6bd9d55eea9..58e3c6165aa 100644 --- a/infra/stats.ts +++ b/infra/stats.ts @@ -1,11 +1,6 @@ import { EMAILOCTOPUS_API_KEY } from "./app" import { domain } from "./stage" -// Keep the deployed names and arguments: the R2 stats sync still uses this VPC -// and cluster after the legacy lake is removed. -const lakeVpc = new sst.aws.Vpc("LakeVpc") -const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc }) - //////////////// // DATABASE //////////////// @@ -101,25 +96,46 @@ const r2Sql = new sst.Linkable("R2Sql", { }, }) -export const statSync = new sst.aws.Service("StatsSyncService", { - cluster: lakeCluster, - architecture: "arm64", - cpu: "0.25 vCPU", - // 0.5 GB caused an OOM crash loop: every restart immediately re-ran the 4 Athena - // stats queries (~$5/pass) every ~5 minutes instead of hourly. - memory: "2 GB", - image: { - context: ".", - dockerfile: "packages/stats/server/Dockerfile", - }, - command: ["bun", "src/stat-sync.ts"], +// One instance per run: a plan step, then one step per stat period and retention +// cohort. See packages/stats/README.md for triggering a run by hand. +export const statsSyncWorkflow = new sst.cloudflare.Workflow("StatsSyncWorkflow", { + handler: "packages/stats/server/src/workflow.ts", + className: "StatsSyncWorkflow", link: [database, r2Sql, r2SqlAuthToken, statsHiddenModels, statsSyncConfig], - scaling: { - min: 1, - max: 1, + transform: { + worker: { + transform: { + worker: (args) => { + // Raise CPU time from the 30-second default to the 5-minute maximum for + // ranking a full week of geo rows. + args.limits = { cpuMs: 300_000 } + args.observability = { enabled: true } + }, + }, + }, }, +}) + +// Schedules must match packages/stats/server/src/schedule.ts, which maps the +// daily one to a history run. +if (!$dev) + new sst.cloudflare.Cron("StatsSyncCron", { + schedules: ["5 * * * *", "20 3 * * *"], + worker: { + handler: "packages/stats/server/src/cron.ts", + link: [statsSyncWorkflow], + transform: { + worker: (args) => { + args.observability = { enabled: true } + }, + }, + }, + }) + +new sst.x.DevCommand("StatsSync", { + link: [database, r2Sql, r2SqlAuthToken, statsHiddenModels, statsSyncConfig], dev: { - command: "bun src/stat-sync.ts", + command: "bun src/sync.ts", directory: "packages/stats/server", autostart: false, }, diff --git a/packages/stats/README.md b/packages/stats/README.md index 9d66cf69f88..7ed96ec4c28 100644 --- a/packages/stats/README.md +++ b/packages/stats/README.md @@ -1,19 +1,42 @@ # OpenCode Stats -Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; deployable Lambda entrypoints live in `function`. +Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; the Cloudflare Workers that sync stats live in `server`. ## Packages - `app`: SolidStart frontend/site. - `core`: Effect services, app config, Drizzle schema/migrations, and stats domains. -- `function`: Lambda handlers that call into `core` services. +- `server`: the `StatsSyncWorkflow` Workflow, the `StatsSyncCron` Worker that starts it, and a local sync script. ## Commands - `bun run dev:stats` from the repo root starts the SolidStart app. - `bun run --cwd packages/stats/app typecheck` typechecks the site. - `bun run --cwd packages/stats/core typecheck` typechecks the Effect/database package. -- `bun run --cwd packages/stats/function typecheck` typechecks Lambda entrypoints. +- `bun run --cwd packages/stats/server typecheck` typechecks the sync Workers. + +## Stats sync + +The sync reads aggregates from R2 SQL and writes them to the stats PlanetScale database. It runs on the `dev` and `production` stages as a Cloudflare Workflow, `StatsSyncWorkflow`, defined in `infra/stats.ts`. + +Each run has a `plan` step, then one step per stat period (`week 2026-W45`, `day 2026-11-03`) and one per retention cohort (`retention 2026-10-05`). The plan snapshots the model catalog once, so every step attributes models the same way. Steps run one at a time and retry twice with exponential backoff. A rejected query (HTTP 400, 401 or 403) or a result over the 10,000-row limit fails its step after one attempt, since a retry repeats the same R2 SQL spend. A failed step does not stop the others, but the run ends as errored and logs each failed step with its error. Because queries run per period, expect an hourly run to take about 15 to 25 minutes and a history run about 2 to 3.5 hours. These are estimates until measured on a deployed stage. + +The `StatsSyncCron` Worker starts runs on two schedules: + +- `5 * * * *`: an hourly run of the current ISO week and the most recent complete retention cohort, with instance ID `stats-hourly-YYYY-MM-DDTHH`. +- `20 3 * * *`: a daily history run of the rest of the 56-day display window, with instance ID `stats-history-YYYY-MM-DD`. + +Times and IDs are in UTC. Before starting a run, the Cron checks this slot's ID and the recent IDs of the same kind: the five previous hours for an hourly run, the two previous days for a history run. It skips the run if any of them is queued, running, waiting or paused, or if this slot's instance already exists, and logs the reason. Hourly and history runs can overlap, since they cover different periods and their writes are idempotent. + +To start a run by hand, find the Workflow's name with `bunx wrangler workflows list` (it is `opencode--statssyncworkflowworkflow-`), then trigger it with an ID from the Cron's convention so scheduled runs of the same kind see it and wait: + +```sh +bunx wrangler workflows trigger '{"history":true}' --id stats-history-2026-11-03 +``` + +Omit `history` for an hourly run. If the ID already exists, that slot has run; wait for the next one. You can also trigger a run from the Workflow's page in the Cloudflare dashboard; without params it runs as an hourly pass. + +To run one sync pass on your machine against a stage's resources, run `bun sst shell --stage= -- bun run --cwd packages/stats/server sync`, adding `--history` for a history pass. `sst dev` also offers it as the `StatsSync` command. ## Public page caching diff --git a/packages/stats/core/src/config.ts b/packages/stats/core/src/config.ts index f6aa6d63323..c30a01d50bd 100644 --- a/packages/stats/core/src/config.ts +++ b/packages/stats/core/src/config.ts @@ -9,10 +9,11 @@ export class AppConfigValue extends Schema.Class("AppConfigValue const decodeAppConfigValue = Schema.decodeUnknownSync(AppConfigValue) +// Read the linked stage lazily, so importing this module in a Worker does not +// touch bindings at module scope. const config = Config.all({ - stage: Config.succeed(Resource.App.stage), publicUrl: Config.String("PUBLIC_URL").pipe(Config.withDefault("http://localhost:3000")), -}).pipe(Config.map(decodeAppConfigValue)) +}).pipe(Config.map((value) => decodeAppConfigValue({ ...value, stage: Resource.App.stage }))) export class AppConfig extends Context.Service()("@opencode/stats/AppConfig") { static readonly config = config diff --git a/packages/stats/core/src/database.ts b/packages/stats/core/src/database.ts index c8df5bd402f..ae1f23183c6 100644 --- a/packages/stats/core/src/database.ts +++ b/packages/stats/core/src/database.ts @@ -1,7 +1,6 @@ import { Client } from "@planetscale/database" import { drizzle } from "drizzle-orm/planetscale-serverless" -import { migrate as drizzleMigrate } from "drizzle-orm/planetscale-serverless/migrator" -import { Config, ConfigProvider, Effect, Layer, Schema } from "effect" +import { Config, ConfigProvider, Effect, Layer, Option, Schema } from "effect" import * as Context from "effect/Context" import { Resource } from "sst/resource" @@ -10,14 +9,17 @@ export type DatabaseUrl = typeof DatabaseUrl.Type export class DatabaseSettings extends Schema.Class("DatabaseSettings")({ url: DatabaseUrl, - migrationsDir: Schema.NonEmptyString, }) {} const decodeDatabaseSettings = Schema.decodeUnknownSync(DatabaseSettings) const config = Config.all({ - url: Config.NonEmptyString("DATABASE_URL").pipe(Config.withDefault(Resource.StatsDatabase.url)), - migrationsDir: Config.NonEmptyString("DATABASE_MIGRATIONS_DIR").pipe(Config.withDefault("./migrations")), + // Read the linked database lazily, so importing this module in a Worker does not + // touch bindings at module scope. + url: Config.NonEmptyString("DATABASE_URL").pipe( + Config.option, + Config.map(Option.getOrElse(() => Resource.StatsDatabase.url)), + ), }).pipe(Config.map(decodeDatabaseSettings)) export class DatabaseConfig extends Context.Service()( @@ -30,8 +32,18 @@ export class DatabaseConfig extends Context.Service fetch(input, { ...init, signal: AbortSignal.timeout(DATABASE_REQUEST_TIMEOUT_MS) }), + }), + }) } export type Drizzle = ReturnType @@ -47,7 +59,10 @@ export class DatabaseError extends Error { readonly _tag = "DatabaseError" constructor(input: { cause: unknown }) { - super("Database operation failed", { cause: input.cause }) + const reason = rootCause(input.cause) + super(reason instanceof Error ? `Database operation failed: ${reason.toString()}` : "Database operation failed", { + cause: input.cause, + }) this.name = "DatabaseError" } @@ -56,38 +71,12 @@ export class DatabaseError extends Error { } } -export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause })) - -export class MigrationError extends Error { - readonly _tag = "MigrationError" - - constructor(input: { message: string; cause?: unknown }) { - super(input.message, { cause: input.cause }) - this.name = "MigrationError" - } +// Drizzle wraps the driver's error in one whose message holds the whole query and +// its parameters, which for an upsert chunk is hundreds of rows. Name the driver's error. +function rootCause(cause: unknown): unknown { + return cause instanceof Error && cause.cause !== undefined ? rootCause(cause.cause) : cause } -export const migrate = Effect.fn("Database.migrate")(function* () { - const settings = yield* DatabaseConfig - yield* Effect.logInfo("applying database migrations").pipe( - Effect.annotateLogs({ migrationsDir: settings.migrationsDir }), - ) - const result = yield* Effect.tryPromise({ - try: () => - drizzleMigrate(drizzle({ client: new Client({ url: settings.url }) }), { - migrationsFolder: settings.migrationsDir, - }), - catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }), - }) - if (result) - return yield* Effect.fail( - new MigrationError({ - message: `Failed to initialize database migrations: ${result.exitCode}`, - }), - ) - yield* Effect.logInfo("database migrations complete").pipe( - Effect.annotateLogs({ migrationsDir: settings.migrationsDir }), - ) -}) +export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause })) export const layer = Layer.mergeAll(DatabaseConfig.layer, DrizzleClient.layer.pipe(Layer.provide(DatabaseConfig.layer))) diff --git a/packages/stats/core/src/domain/catalog-identity.ts b/packages/stats/core/src/domain/catalog-identity.ts index 2fb6c55b1b2..cb82d9fbeb8 100644 --- a/packages/stats/core/src/domain/catalog-identity.ts +++ b/packages/stats/core/src/domain/catalog-identity.ts @@ -9,6 +9,21 @@ export type CatalogIdentity = { models: ReadonlyMap } +// A JSON-safe copy of a catalog identity, so a sync plan can carry one snapshot +// to every step of a run. +export type CatalogSnapshot = { + offerings: [string, string][] + models: [string, string][] +} + +export function toCatalogSnapshot(identity: CatalogIdentity): CatalogSnapshot { + return { offerings: [...identity.offerings], models: [...identity.models] } +} + +export function fromCatalogSnapshot(snapshot: CatalogSnapshot): CatalogIdentity { + return { offerings: new Map(snapshot.offerings), models: new Map(snapshot.models) } +} + export async function loadCatalogIdentity() { if (cache.value && (cache.expiresAt ?? 0) > Date.now()) return { catalog: cache.value, stale: false } return fetch(CATALOG_URL, { signal: AbortSignal.timeout(10_000) }) diff --git a/packages/stats/core/src/domain/geo.ts b/packages/stats/core/src/domain/geo.ts index 5c781300c41..7408c9410ba 100644 --- a/packages/stats/core/src/domain/geo.ts +++ b/packages/stats/core/src/domain/geo.ts @@ -18,6 +18,7 @@ import { UPSERT_CHUNK_SIZE, withUniqueUsersFallback, type StatBaseAggregate, + type StatPeriodScope, } from "./stat" export type GeoStatRow = typeof geoStat.$inferInsert @@ -56,7 +57,7 @@ export declare namespace GeoStatRepo { }) => Effect.Effect readonly upsert: (rows: GeoStatRow[]) => Effect.Effect readonly deleteRetiredDimensions: ( - rows: GeoStatRow[], + scope: StatPeriodScope, hiddenModels: readonly string[], ) => Effect.Effect readonly deleteUnknownDimensions: (rows: GeoStatRow[]) => Effect.Effect @@ -190,23 +191,20 @@ export class GeoStatRepo extends Context.Service db .delete(geoStat) .where( and( - inArray(geoStat.grain, scope.grains), - inArray(geoStat.period_key, scope.periodKeys), - inArray(geoStat.dataset, scope.datasets), - inArray(geoStat.client, scope.clients), - inArray(geoStat.source, scope.sources), + eq(geoStat.grain, scope.grain), + eq(geoStat.period_key, scope.periodKey), + eq(geoStat.dataset, scope.dataset), + eq(geoStat.client, "all"), + eq(geoStat.source, "all"), or( inArray(geoStat.provider, RETIRED_STAT_PROVIDERS), inArray(geoStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]), diff --git a/packages/stats/core/src/domain/inference.ts b/packages/stats/core/src/domain/inference.ts index 3a7d45a13e5..99d00027f33 100644 --- a/packages/stats/core/src/domain/inference.ts +++ b/packages/stats/core/src/domain/inference.ts @@ -23,6 +23,7 @@ import { startOfIsoWeek, startOfUtcDay, type StatBaseAggregate, + type StatGrain, } from "./stat" export type StatDimension = "model" | "provider" | "geo" | "geo_model" @@ -47,24 +48,24 @@ const GO_PRODUCTS = ["go", "go-plus"] // and carry no usage. Counting them would publish any requested model name as a stats page. const ROUTED_ONLY = "attempts > 0" -// R2 SQL results are cursor-paginated after aggregation. Separate usage and geo -// queries per day/week avoid combining costly distinct user/session aggregates -// with the high-cardinality geo dimensions. +export type StatPeriod = { grain: StatGrain; key: string; start: Date; end: Date } +export type RetentionPeriod = { start: Date; end: Date; returnStart: Date; returnEnd: Date } + export function buildStatsQueries( periodStart: Date, periodEnd: Date, input?: StatsQuerySource, catalog?: CatalogIdentity, ) { - const source = input ?? { - namespace: Resource.R2Sql.namespace, - table: Resource.R2Sql.table, - dataset: Resource.StatsSyncConfig.dataset, - hiddenModels: hiddenStatModels(), - } - return [...statPeriods("week", periodStart, periodEnd), ...statPeriods("day", periodStart, periodEnd)].flatMap( - (period) => [buildStatsQuery(period, source, "usage", catalog), buildStatsQuery(period, source, "geo", catalog)], - ) + return statPeriods(periodStart, periodEnd).flatMap((period) => buildPeriodStatsQueries(period, input, catalog)) +} + +// R2 SQL results are cursor-paginated after aggregation. Separate usage and geo +// queries per day/week avoid combining costly distinct user/session aggregates +// with the high-cardinality geo dimensions. +export function buildPeriodStatsQueries(period: StatPeriod, input?: StatsQuerySource, catalog?: CatalogIdentity) { + const source = input ?? defaultSource() + return [buildStatsQuery(period, source, "usage", catalog), buildStatsQuery(period, source, "geo", catalog)] } export function buildRetentionQueries( @@ -73,26 +74,32 @@ export function buildRetentionQueries( input?: StatsQuerySource, catalog?: CatalogIdentity, ): RetentionQuery[] { - const source = input ?? { + return retentionPeriods(periodStart, periodEnd).map((period) => buildPeriodRetentionQuery(period, input, catalog)) +} + +// Bound the user-level joins to one activity week and its return week. +// Combining the entire display window makes full syncs much more expensive. +export function buildPeriodRetentionQuery( + period: RetentionPeriod, + input?: StatsQuerySource, + catalog?: CatalogIdentity, +): RetentionQuery { + return { + cohortDates: [period.start.toISOString().slice(0, 10)], + query: buildRetentionQuery([period], input ?? defaultSource(), catalog), + } +} + +function defaultSource(): StatsQuerySource { + return { namespace: Resource.R2Sql.namespace, table: Resource.R2Sql.table, dataset: Resource.StatsSyncConfig.dataset, hiddenModels: hiddenStatModels(), } - const periods = retentionPeriods(periodStart, periodEnd) - // Bound the user-level joins to one activity week and its return week. - // Combining the entire display window makes full syncs much more expensive. - return periods.map((period) => ({ - cohortDates: [period.start.toISOString().slice(0, 10)], - query: buildRetentionQuery([period], source, catalog), - })) } -function buildRetentionQuery( - periods: { start: Date; end: Date; returnStart: Date; returnEnd: Date }[], - source: StatsQuerySource, - catalog?: CatalogIdentity, -) { +function buildRetentionQuery(periods: RetentionPeriod[], source: StatsQuerySource, catalog?: CatalogIdentity) { const first = periods[0] const last = periods.at(-1)! const scanStartValue = sqlString(first.start.toISOString()) @@ -209,7 +216,7 @@ LIMIT 10000 } function buildStatsQuery( - period: { grain: "day" | "week"; key: string; start: Date; end: Date }, + period: StatPeriod, source: StatsQuerySource, family: StatsQueryFamily, catalog?: CatalogIdentity, @@ -477,7 +484,11 @@ function sqlString(value: string) { return `'${value.replace(/'/g, "''")}'` } -function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) { +export function statPeriods(periodStart: Date, periodEnd: Date) { + return [...grainPeriods("week", periodStart, periodEnd), ...grainPeriods("day", periodStart, periodEnd)] +} + +function grainPeriods(grain: StatGrain, periodStart: Date, periodEnd: Date): StatPeriod[] { const interval = grain === "day" ? DAY_MS : WEEK_MS const first = grain === "day" ? startOfUtcDay(periodStart) : startOfIsoWeek(periodStart) const count = Math.max(0, Math.ceil((periodEnd.getTime() - first.getTime()) / interval)) @@ -492,7 +503,7 @@ function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) }) } -function retentionPeriods(periodStart: Date, periodEnd: Date) { +export function retentionPeriods(periodStart: Date, periodEnd: Date): RetentionPeriod[] { const first = startOfIsoWeek(periodStart) const completeEnd = startOfIsoWeek(periodEnd) const count = Math.max(0, Math.floor((completeEnd.getTime() - first.getTime()) / WEEK_MS) - 1) diff --git a/packages/stats/core/src/domain/model.ts b/packages/stats/core/src/domain/model.ts index 5b5f507d56e..fe67d2f5c31 100644 --- a/packages/stats/core/src/domain/model.ts +++ b/packages/stats/core/src/domain/model.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm" +import { and, asc, eq, inArray, or } from "drizzle-orm" import { Effect, Layer } from "effect" import * as Context from "effect/Context" import { DatabaseError, DrizzleClient } from "../database" @@ -10,7 +10,6 @@ import { DATA_SITE_TIERS, inserted, omitUniqueUsers, - periodKeyFor, rankBy, statPeriodKey, statRowScope, @@ -19,6 +18,7 @@ import { UPSERT_CHUNK_SIZE, withUniqueUsersFallback, type StatBaseAggregate, + type StatPeriodScope, } from "./stat" export type ModelStatRow = typeof modelStat.$inferInsert @@ -45,10 +45,9 @@ export type ModelStatMetric = { export declare namespace ModelStatRepo { export interface Service { readonly listDaily: () => Effect.Effect - readonly lastSyncedAt: (since: Date) => Effect.Effect readonly upsert: (rows: ModelStatRow[]) => Effect.Effect readonly deleteRetiredDimensions: ( - rows: ModelStatRow[], + scope: StatPeriodScope, hiddenModels: readonly string[], ) => Effect.Effect readonly deleteUnknownDimensions: (rows: ModelStatRow[]) => Effect.Effect @@ -117,18 +116,6 @@ export class ModelStatRepo extends Context.Service - db - .select({ value: max(modelStat.updated_at) }) - .from(modelStat) - .where(and(eq(modelStat.grain, "day"), gte(modelStat.period_key, periodKeyFor("day", since)))), - catch: (cause) => DatabaseError.make({ cause }), - }) - return result[0]?.value ?? null - }) - const upsert = Effect.fn("ModelStatRepo.upsert")(function* (rows: ModelStatRow[]) { yield* Effect.forEach( chunks(rows, UPSERT_CHUNK_SIZE), @@ -177,23 +164,20 @@ export class ModelStatRepo extends Context.Service db .delete(modelStat) .where( and( - inArray(modelStat.grain, scope.grains), - inArray(modelStat.period_key, scope.periodKeys), - inArray(modelStat.dataset, scope.datasets), - inArray(modelStat.client, scope.clients), - inArray(modelStat.source, scope.sources), + eq(modelStat.grain, scope.grain), + eq(modelStat.period_key, scope.periodKey), + eq(modelStat.dataset, scope.dataset), + eq(modelStat.client, "all"), + eq(modelStat.source, "all"), or( inArray(modelStat.provider, RETIRED_STAT_PROVIDERS), inArray(modelStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]), @@ -253,7 +237,7 @@ export class ModelStatRepo extends Context.Service>((result, row) => { const key = statPeriodKey(row) - result[key] = [...(result[key] ?? []), row] + ;(result[key] ??= []).push(row) return result }, {}), ).flatMap((group) => { diff --git a/packages/stats/core/src/domain/provider.ts b/packages/stats/core/src/domain/provider.ts index f2aaf6e0869..11a64e4a474 100644 --- a/packages/stats/core/src/domain/provider.ts +++ b/packages/stats/core/src/domain/provider.ts @@ -11,12 +11,12 @@ import { inserted, omitUniqueUsers, rankRowsWithMarketShare, - statRowScope, synthesizeAllTierRows, toStatBaseRow, UPSERT_CHUNK_SIZE, withUniqueUsersFallback, type StatBaseAggregate, + type StatPeriodScope, } from "./stat" export type ProviderStatRow = typeof providerStat.$inferInsert @@ -41,7 +41,7 @@ export declare namespace ProviderStatRepo { readonly source?: string }) => Effect.Effect readonly upsert: (rows: ProviderStatRow[]) => Effect.Effect - readonly deleteRetiredDimensions: (rows: ProviderStatRow[]) => Effect.Effect + readonly deleteRetiredDimensions: (scope: StatPeriodScope) => Effect.Effect } } @@ -157,22 +157,19 @@ export class ProviderStatRepo extends Context.Service db .delete(providerStat) .where( and( - inArray(providerStat.grain, scope.grains), - inArray(providerStat.period_key, scope.periodKeys), - inArray(providerStat.dataset, scope.datasets), - inArray(providerStat.client, scope.clients), - inArray(providerStat.source, scope.sources), + eq(providerStat.grain, scope.grain), + eq(providerStat.period_key, scope.periodKey), + eq(providerStat.dataset, scope.dataset), + eq(providerStat.client, "all"), + eq(providerStat.source, "all"), inArray(providerStat.provider, RETIRED_STAT_PROVIDERS), ), ), diff --git a/packages/stats/core/src/domain/retention.ts b/packages/stats/core/src/domain/retention.ts index 14b154aac7b..689c5b9312c 100644 --- a/packages/stats/core/src/domain/retention.ts +++ b/packages/stats/core/src/domain/retention.ts @@ -54,28 +54,23 @@ export class RetentionStatRepo extends Context.Service - db - .delete(modelRetention) - .where( - and( - inArray(modelRetention.cohort_date, scope.cohortDates), - eq(modelRetention.dataset, scope.dataset), - eq(modelRetention.tier, scope.tier), - ), - ), + db.transaction(async (tx) => { + await tx + .delete(modelRetention) + .where( + and( + inArray(modelRetention.cohort_date, scope.cohortDates), + eq(modelRetention.dataset, scope.dataset), + eq(modelRetention.tier, scope.tier), + ), + ) + for (const chunk of chunks(rows, UPSERT_CHUNK_SIZE)) await tx.insert(modelRetention).values(chunk) + }), catch: (cause) => DatabaseError.make({ cause }), }) - yield* Effect.forEach( - chunks(rows, UPSERT_CHUNK_SIZE), - (chunk) => - Effect.tryPromise({ - try: () => db.insert(modelRetention).values(chunk), - catch: (cause) => DatabaseError.make({ cause }), - }), - { discard: true }, - ) }) return RetentionStatRepo.of({ available, replace }) diff --git a/packages/stats/core/src/domain/stat.ts b/packages/stats/core/src/domain/stat.ts index b22d44eaf01..478cb9c77e1 100644 --- a/packages/stats/core/src/domain/stat.ts +++ b/packages/stats/core/src/domain/stat.ts @@ -173,6 +173,9 @@ export function statPeriodKey(row: StatBaseRow) { return [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source].join("\u0000") } +// One synced period. The sync writes only the all-client, all-source rows. +export type StatPeriodScope = { grain: StatGrain; periodKey: string; dataset: string } + export function statRowScope(rows: StatBaseRow[]) { if (rows.length === 0) return return { @@ -221,7 +224,7 @@ export function rankRowsWithMarketShare( return Object.values( rows.reduce>((result, row) => { const key = groupKey(row) - result[key] = [...(result[key] ?? []), row] + ;(result[key] ??= []).push(row) return result }, {}), ).flatMap((group) => { diff --git a/packages/stats/core/src/honeycomb-backfill.ts b/packages/stats/core/src/honeycomb-backfill.ts index a22362dcda9..641c57be610 100644 --- a/packages/stats/core/src/honeycomb-backfill.ts +++ b/packages/stats/core/src/honeycomb-backfill.ts @@ -569,7 +569,7 @@ function rankModelRows(rows: ModelStatRow[]) { return Object.values( rows.reduce>((result, row) => { const key = statPeriodKey(row) - result[key] = [...(result[key] ?? []), row] + ;(result[key] ??= []).push(row) return result }, {}), ).flatMap((group) => { diff --git a/packages/stats/core/src/migrate.ts b/packages/stats/core/src/migrate.ts index f2a7c52bca3..e616cd3013a 100644 --- a/packages/stats/core/src/migrate.ts +++ b/packages/stats/core/src/migrate.ts @@ -1,4 +1,35 @@ -import { Effect } from "effect" -import { layer, migrate } from "./database" +import { migrate } from "drizzle-orm/planetscale-serverless/migrator" +import { Config, ConfigProvider, Effect } from "effect" +import { DrizzleClient, layer } from "./database" -await Effect.runPromise(migrate().pipe(Effect.provide(layer))) +// The migrator reads migration files from disk, so it stays out of the +// Worker-safe database module. +class MigrationError extends Error { + readonly _tag = "MigrationError" + + constructor(input: { message: string; cause?: unknown }) { + super(input.message, { cause: input.cause }) + this.name = "MigrationError" + } +} + +const applyMigrations = Effect.fn("Database.migrate")(function* () { + const migrationsDir = yield* Config.NonEmptyString("DATABASE_MIGRATIONS_DIR") + .pipe(Config.withDefault("./migrations")) + .parse(ConfigProvider.fromEnv()) + const db = yield* DrizzleClient + yield* Effect.logInfo("applying database migrations").pipe(Effect.annotateLogs({ migrationsDir })) + const result = yield* Effect.tryPromise({ + try: () => migrate(db, { migrationsFolder: migrationsDir }), + catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }), + }) + if (result) + return yield* Effect.fail( + new MigrationError({ + message: `Failed to initialize database migrations: ${result.exitCode}`, + }), + ) + yield* Effect.logInfo("database migrations complete").pipe(Effect.annotateLogs({ migrationsDir })) +}) + +await Effect.runPromise(applyMigrations().pipe(Effect.provide(layer))) diff --git a/packages/stats/core/src/r2-sql.test.ts b/packages/stats/core/src/r2-sql.test.ts index a014f1be2b5..efa94babe02 100644 --- a/packages/stats/core/src/r2-sql.test.ts +++ b/packages/stats/core/src/r2-sql.test.ts @@ -2,7 +2,7 @@ import { Database } from "bun:sqlite" import { Effect, Fiber } from "effect" import { TestClock } from "effect/testing" import { expect, test } from "bun:test" -import { queryR2SqlPages, R2SqlQueryError, type R2SqlData } from "./r2-sql" +import { isDeterministicR2SqlError, queryR2SqlPages, R2SqlQueryError, type R2SqlData } from "./r2-sql" test("includes the transport failure in the message used by sync logging", () => { const cause = new DOMException("The operation timed out.", "TimeoutError") @@ -158,3 +158,36 @@ test("bounds transient retries and preserves the final error", async () => { expect(calls).toHaveLength(3) expect(result).toBe(error) }, 20000) + +test("treats rejected queries, rejected credentials and row-limit results as deterministic", async () => { + const rowLimit = await Effect.runPromise( + queryR2SqlPages("SELECT * FROM aggregates LIMIT 10000", undefined, () => + Effect.succeed(Array.from({ length: 10000 }, () => ({}))), + ).pipe(Effect.flip), + ) + + expect( + [ + rowLimit, + new R2SqlQueryError({ message: "syntax error", status: 400 }), + new R2SqlQueryError({ message: "unauthorized", status: 401 }), + new R2SqlQueryError({ message: "forbidden", status: 403 }), + ].map(isDeterministicR2SqlError), + ).toEqual([true, true, true, true]) +}) + +test("leaves transient and unexplained failures retryable", () => { + expect( + [ + new R2SqlQueryError({ message: "query timeout", status: 400, code: 40005 }), + new R2SqlQueryError({ message: "Unprocessable Entity", status: 422, code: 60104 }), + new R2SqlQueryError({ message: "rate limited", status: 429 }), + new R2SqlQueryError({ message: "unavailable", status: 503 }), + new R2SqlQueryError({ message: "Failed to run R2 SQL stats query", cause: new TypeError("connection closed") }), + new R2SqlQueryError({ + message: "Failed to run R2 SQL stats query", + cause: new DOMException("The operation timed out.", "TimeoutError"), + }), + ].map(isDeterministicR2SqlError), + ).toEqual([false, false, false, false, false, false]) +}) diff --git a/packages/stats/core/src/r2-sql.ts b/packages/stats/core/src/r2-sql.ts index 73b92c54491..0afd7c15614 100644 --- a/packages/stats/core/src/r2-sql.ts +++ b/packages/stats/core/src/r2-sql.ts @@ -2,7 +2,7 @@ import { Context, Effect, Layer, Schedule, Schema } from "effect" import { Resource } from "sst/resource" const R2_SQL_MAX_ROWS = 10_000 -const R2_SQL_TIMEOUT_MS = 15 * 60_000 +const R2_SQL_TIMEOUT_MS = 10 * 60_000 const R2SqlValue = Schema.Union([Schema.String, Schema.Number, Schema.Boolean, Schema.Null]) const R2SqlResponse = Schema.Struct({ success: Schema.Boolean, @@ -26,8 +26,16 @@ export class R2SqlQueryError extends Error { readonly requestId?: string readonly status?: number readonly code?: number + readonly rowLimit: boolean - constructor(input: { message: string; requestId?: string; status?: number; code?: number; cause?: unknown }) { + constructor(input: { + message: string + requestId?: string + status?: number + code?: number + rowLimit?: boolean + cause?: unknown + }) { super(input.cause instanceof Error ? `${input.message}: ${input.cause.toString()}` : input.message, { cause: input.cause, }) @@ -35,9 +43,29 @@ export class R2SqlQueryError extends Error { this.requestId = input.requestId this.status = input.status this.code = input.code + this.rowLimit = input.rowLimit ?? false } } +// Retrying these repeats the same R2 SQL spend for the same failure: a rejected +// query, rejected credentials, or a result that will hit the row limit again. +export function isDeterministicR2SqlError(error: R2SqlQueryError) { + if (isTransientR2SqlError(error)) return false + return error.rowLimit || error.status === 400 || error.status === 401 || error.status === 403 +} + +// 40005 is a query timeout and 60104 an intermittent "Unprocessable Entity", both +// sent with 4xx statuses. fetch rejects with a TypeError when the connection drops. +function isTransientR2SqlError(error: R2SqlQueryError) { + return ( + error.code === 40005 || + error.code === 60104 || + error.status === 429 || + (error.status !== undefined && error.status >= 500) || + error.cause instanceof TypeError + ) +} + export declare namespace R2Sql { export interface Service { readonly query: (query: string, columns?: readonly string[]) => Effect.Effect @@ -67,23 +95,16 @@ export const queryR2SqlPages = Effect.fn("R2Sql.query")(function* ( fetchRows(columns?.length ? pageQuery(query, columns, cursor) : query), ).pipe( // Retry only this page; a transient R2 failure must not discard the whole - // pass. 40005 is a query timeout, 60104 an intermittent "Unprocessable - // Entity", and fetch rejects with a TypeError when the connection drops. - // Syntax, auth, row-limit, and 15-minute request timeout errors still fail. + // period. Syntax, auth, row-limit, and request timeout errors still fail. Effect.retry({ times: 2, schedule: Schedule.exponential("5 seconds"), - while: (error) => - error.code === 40005 || - error.code === 60104 || - error.status === 429 || - (error.status !== undefined && error.status >= 500) || - error.cause instanceof TypeError, + while: isTransientR2SqlError, }), ) if (page.length >= R2_SQL_MAX_ROWS && !columns?.length) return yield* Effect.fail( - new R2SqlQueryError({ message: `R2 SQL stats query reached the ${R2_SQL_MAX_ROWS} row limit` }), + new R2SqlQueryError({ message: `R2 SQL stats query reached the ${R2_SQL_MAX_ROWS} row limit`, rowLimit: true }), ) rows.push(...page) if (page.length < R2_SQL_MAX_ROWS) return rows @@ -112,8 +133,9 @@ const fetchRows = Effect.fn("R2Sql.fetchRows")(function* (query: string) { try: (signal) => { const options = { method: "POST", - // Analytical queries can exceed Bun's default five-minute idle timer. - // Bound the whole request and cancel it when the sync is interrupted. + // Bun's fetch, used by the local CLI, otherwise stops at a five-minute + // idle timer; Workers ignore the option. Bound the whole request and + // cancel it when the sync is interrupted. timeout: false, signal: AbortSignal.any([signal, AbortSignal.timeout(R2_SQL_TIMEOUT_MS)]), headers: { @@ -122,7 +144,7 @@ const fetchRows = Effect.fn("R2Sql.fetchRows")(function* (query: string) { }, body: JSON.stringify({ query }), } - return Bun.fetch( + return fetch( `https://api.sql.cloudflarestorage.com/api/v1/accounts/${Resource.R2Sql.accountId}/r2-sql/query/${Resource.R2Sql.bucket}`, options, ) diff --git a/packages/stats/core/src/stat-sync.test.ts b/packages/stats/core/src/stat-sync.test.ts new file mode 100644 index 00000000000..ed48809d6fe --- /dev/null +++ b/packages/stats/core/src/stat-sync.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, test } from "bun:test" +import { Effect, Layer } from "effect" +import { catalogIdentity, type CatalogSnapshot } from "./domain/catalog-identity" +import type { StatPeriodScope } from "./domain/stat" +import type { R2SqlData } from "./r2-sql" + +process.env.SST_RESOURCE_App = JSON.stringify({ name: "opencode", stage: "test" }) +process.env.SST_RESOURCE_R2Sql = JSON.stringify({ + accountId: "account", + bucket: "bucket", + namespace: "inference", + table: "generation", +}) +process.env.SST_RESOURCE_StatsSyncConfig = JSON.stringify({ dataset: "zen" }) +process.env.SST_RESOURCE_StatsHiddenModels = JSON.stringify({ value: "secret-model" }) + +const { GeoStatRepo } = await import("./domain/geo") +const { ModelStatRepo } = await import("./domain/model") +const { ProviderStatRepo } = await import("./domain/provider") +const { RetentionStatRepo } = await import("./domain/retention") +const { R2Sql, R2SqlQueryError } = await import("./r2-sql") +const { CatalogUnavailableError, deterministicFailureAsResult, planStatsSync, syncRetentionCohort, syncStatPeriod } = + await import("./stat-sync") +const { DatabaseError } = await import("./database") + +const catalog: CatalogSnapshot = { offerings: [["opencode/model-a", "lab"]], models: [["model-a", "lab"]] } +const loadCatalog = () => + Promise.resolve({ + catalog: catalogIdentity({ + models: { "lab/model-a": {} }, + providers: { opencode: { models: { "model-a": { canonical_model_id: "lab/model-a" } } } }, + }), + stale: false, + }) + +describe("stats sync plan", () => { + test("fails instead of planning a run without the model catalog", async () => { + const error = await Effect.runPromise( + planStatsSync(new Date("2026-10-23T19:05:00.000Z"), { + history: false, + loadCatalog: () => Promise.resolve({ catalog: undefined, stale: true }), + }).pipe(Effect.flip, Effect.provide(services({ events: [] }))), + ) + + expect(error).toBeInstanceOf(CatalogUnavailableError) + }) + + test("plans the current week for hourly runs and the rest of the window for history runs", async () => { + const startedAt = new Date("2026-10-23T19:05:00.000Z") + const plan = (history: boolean) => + Effect.runPromise( + planStatsSync(startedAt, { history, loadCatalog }).pipe(Effect.provide(services({ events: [] }))), + ) + + const hourly = await plan(false) + expect(hourly.catalog).toEqual({ offerings: [["opencode/model-a", "lab"]], models: [["model-a", "lab"]] }) + expect(hourly.periods.map((period) => `${period.grain} ${period.key}`)).toEqual([ + "week 2026-W43", + "day 2026-10-19", + "day 2026-10-20", + "day 2026-10-21", + "day 2026-10-22", + "day 2026-10-23", + ]) + expect(hourly.periods.at(-1)).toEqual({ + grain: "day", + key: "2026-10-23", + start: "2026-10-23T00:00:00.000Z", + end: "2026-10-23T19:00:00.000Z", + }) + expect(hourly.retentionPeriods).toEqual([ + { + start: "2026-10-05T00:00:00.000Z", + end: "2026-10-12T00:00:00.000Z", + returnStart: "2026-10-12T00:00:00.000Z", + returnEnd: "2026-10-19T00:00:00.000Z", + }, + ]) + + const history = await plan(true) + expect(history.periods.filter((period) => period.grain === "week").map((period) => period.key)).toEqual([ + "2026-W35", + "2026-W36", + "2026-W37", + "2026-W38", + "2026-W39", + "2026-W40", + "2026-W41", + "2026-W42", + ]) + expect(history.periods.filter((period) => period.grain === "day")).toHaveLength(52) + expect(history.periods.at(-1)?.key).toBe("2026-10-18") + expect(history.retentionPeriods.map((period) => period.start.slice(0, 10))).toEqual([ + "2026-08-24", + "2026-08-31", + "2026-09-07", + "2026-09-14", + "2026-09-21", + "2026-09-28", + ]) + }) +}) + +describe("stats sync steps", () => { + test("drops retired and hidden dimensions from a period with no remaining rows", async () => { + const events: string[] = [] + + const result = await Effect.runPromise( + syncStatPeriod( + { grain: "day", key: "2026-10-22", start: "2026-10-22T00:00:00.000Z", end: "2026-10-23T00:00:00.000Z" }, + catalog, + ).pipe(Effect.provide(services({ events }))), + ) + + expect(result).toEqual({ grain: "day", periodKey: "2026-10-22", modelRows: 0, providerRows: 0, geoRows: 0 }) + expect(events.filter((event) => event.startsWith("deleteRetired")).toSorted()).toEqual([ + 'deleteRetired geo {"grain":"day","periodKey":"2026-10-22","dataset":"zen"} secret-model', + 'deleteRetired model {"grain":"day","periodKey":"2026-10-22","dataset":"zen"} secret-model', + 'deleteRetired provider {"grain":"day","periodKey":"2026-10-22","dataset":"zen"}', + ]) + }) + + test("replaces a retention cohort only after its query returns", async () => { + const events: string[] = [] + + const result = await Effect.runPromise( + syncRetentionCohort(retentionPeriod, catalog).pipe( + Effect.provide( + services({ + events, + rows: [ + { + cohort_date: "2026-10-05", + dataset: "zen", + tier: "Go", + provider: "lab", + model: "model-a", + eligible_users: "10", + retained_users: "4", + }, + ], + }), + ), + ), + ) + + expect(result).toEqual({ cohortDate: "2026-10-05", rows: 1 }) + expect(events).toEqual(["query", 'replace ["2026-10-05"] zen Go model-a:10:4']) + }) + + test("keeps the existing cohort when its query fails", async () => { + const events: string[] = [] + const failure = new R2SqlQueryError({ message: "unavailable", status: 400 }) + + const error = await Effect.runPromise( + syncRetentionCohort(retentionPeriod, catalog).pipe(Effect.flip, Effect.provide(services({ events, failure }))), + ) + + expect(error).toBe(failure) + expect(events).toEqual(["query"]) + }) +}) + +describe("stats sync failures", () => { + test("returns a deterministic R2 SQL failure as the result", async () => { + const result = await Effect.runPromise( + deterministicFailureAsResult( + Effect.fail(new R2SqlQueryError({ message: "R2 SQL stats query failed: syntax error", status: 400 })), + ), + ) + + expect(result).toEqual({ failed: "R2 SQL stats query failed: syntax error" }) + }) + + test("keeps transient R2 SQL and database failures as failures so the step retries", async () => { + const transient = new R2SqlQueryError({ message: "query timeout", status: 400, code: 40005 }) + const database = new DatabaseError({ cause: new Error("connection reset") }) + + expect(await Effect.runPromise(deterministicFailureAsResult(Effect.fail(transient)).pipe(Effect.flip))).toBe( + transient, + ) + expect(await Effect.runPromise(deterministicFailureAsResult(Effect.fail(database)).pipe(Effect.flip))).toBe( + database, + ) + }) + + test("names the driver's error in a database failure without the wrapped query", () => { + const driver = new Error("target: stats.-.primary: vttablet: rpc error: code = Unavailable") + const wrapped = Object.assign(new Error("Failed query: insert into model_stat ...\nparams: 1,2,3"), { + cause: driver, + }) + + expect(new DatabaseError({ cause: wrapped }).message).toBe( + "Database operation failed: Error: target: stats.-.primary: vttablet: rpc error: code = Unavailable", + ) + expect(new DatabaseError({ cause: "timeout" }).message).toBe("Database operation failed") + }) +}) + +const retentionPeriod = { + start: "2026-10-05T00:00:00.000Z", + end: "2026-10-12T00:00:00.000Z", + returnStart: "2026-10-12T00:00:00.000Z", + returnEnd: "2026-10-19T00:00:00.000Z", +} + +function services(input: { events: string[]; rows?: R2SqlData[]; failure?: InstanceType }) { + const unused = () => Effect.die("unused") + const retired = (table: string) => (scope: StatPeriodScope, hidden?: readonly string[]) => + Effect.sync(() => { + input.events.push(`deleteRetired ${table} ${JSON.stringify(scope)}${hidden ? ` ${hidden.join(",")}` : ""}`) + }) + return Layer.mergeAll( + Layer.succeed( + R2Sql, + R2Sql.of({ + query: () => + Effect.suspend(() => { + input.events.push("query") + return input.failure ? Effect.fail(input.failure) : Effect.succeed(input.rows ?? []) + }), + }), + ), + Layer.succeed( + ModelStatRepo, + ModelStatRepo.of({ + listDaily: unused, + upsert: () => Effect.void, + deleteRetiredDimensions: retired("model"), + deleteUnknownDimensions: () => Effect.void, + }), + ), + Layer.succeed( + ProviderStatRepo, + ProviderStatRepo.of({ + listDaily: unused, + listByPeriod: unused, + upsert: () => Effect.void, + deleteRetiredDimensions: retired("provider"), + }), + ), + Layer.succeed( + GeoStatRepo, + GeoStatRepo.of({ + listDaily: unused, + listByPeriod: unused, + upsert: () => Effect.void, + deleteRetiredDimensions: retired("geo"), + deleteUnknownDimensions: () => Effect.void, + }), + ), + Layer.succeed( + RetentionStatRepo, + RetentionStatRepo.of({ + available: () => Effect.succeed(true), + replace: (rows, scope) => + Effect.sync(() => { + input.events.push( + `replace ${JSON.stringify(scope.cohortDates)} ${scope.dataset} ${scope.tier} ${rows + .map((row) => `${row.model}:${row.eligible_users}:${row.retained_users}`) + .join(",")}`, + ) + }), + }), + ), + ) +} diff --git a/packages/stats/core/src/stat-sync.ts b/packages/stats/core/src/stat-sync.ts index 138f20beaa8..cde8a5955e3 100644 --- a/packages/stats/core/src/stat-sync.ts +++ b/packages/stats/core/src/stat-sync.ts @@ -1,12 +1,19 @@ import { DateTime, Effect } from "effect" import { Resource } from "sst/resource" import { DatabaseError } from "./database" -import { loadCatalogIdentity } from "./domain/catalog-identity" +import { + fromCatalogSnapshot, + loadCatalogIdentity, + toCatalogSnapshot, + type CatalogSnapshot, +} from "./domain/catalog-identity" import { GeoStatRepo, rowsFromAggregates as geoRowsFromAggregates } from "./domain/geo" import { - buildRetentionQueries, - buildStatsQueries, + buildPeriodRetentionQuery, + buildPeriodStatsQueries, hiddenStatModels, + retentionPeriods, + statPeriods, toGeoAggregate, toModelAggregate, toProviderAggregate, @@ -16,169 +23,256 @@ import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./ import { statProvider } from "./domain/model-normalization" import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider" import { RetentionStatRepo, rowsFromAggregates as retentionRowsFromAggregates } from "./domain/retention" -import { R2Sql, R2SqlQueryError } from "./r2-sql" +import type { StatGrain } from "./domain/stat" +import { isDeterministicR2SqlError, R2Sql, R2SqlQueryError } from "./r2-sql" import { syncRange } from "./sync-range" +// Plans cross a Workflow step boundary, so every value is plain JSON. +export type StatsSyncPeriod = { grain: StatGrain; key: string; start: string; end: string } +export type StatsSyncRetentionPeriod = { start: string; end: string; returnStart: string; returnEnd: string } +export type StatsSyncPlan = { + startedAt: string + history: boolean + periodStart: string + periodEnd: string + catalog: CatalogSnapshot + periods: StatsSyncPeriod[] + retentionPeriods: StatsSyncRetentionPeriod[] +} export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string } -export type SyncStatsError = R2SqlQueryError | DatabaseError -type SyncStatsServices = R2Sql | ModelStatRepo | ProviderStatRepo | GeoStatRepo | RetentionStatRepo +export type SyncStatsError = R2SqlQueryError | DatabaseError | CatalogUnavailableError -export const syncStats: (options?: { - history?: boolean -}) => Effect.Effect = Effect.fn("StatSync.sync")( - function* (options?: { history?: boolean }) { - const startedAt = yield* DateTime.nowAsDate - const range = syncRange(startedAt, options?.history) - const r2Sql = yield* R2Sql - const modelStats = yield* ModelStatRepo - const providerStats = yield* ProviderStatRepo - const geoStats = yield* GeoStatRepo - const retentionStats = yield* RetentionStatRepo - const identity = yield* Effect.promise(loadCatalogIdentity) - const catalog = identity.catalog - if (identity.stale) - yield* Effect.logWarning( - catalog - ? "model catalog unavailable; using cached identity" - : "model catalog unavailable; using legacy stats attribution", - ) +export class CatalogUnavailableError extends Error { + readonly _tag = "CatalogUnavailableError" - yield* logRuntimeCheck() + constructor() { + super("Model catalog unavailable") + this.name = "CatalogUnavailableError" + } +} - const queries = buildStatsQueries(range.periodStart, range.periodEnd, undefined, catalog) - yield* Effect.logInfo( - `stats sync started ${JSON.stringify({ history: options?.history ?? false, periodStart: range.periodStart, periodEnd: range.periodEnd, queries: queries.length })}`, - ) - const rows = yield* Effect.forEach( - queries, - (query, index) => - r2Sql.query(query, ["dimension", "tier", "provider", "model", "country"]).pipe( - Effect.tap((rows) => - Effect.logInfo( - `stats query complete ${JSON.stringify({ index, total: queries.length, rows: rows.length })}`, - ), - ), +const STATS_CURSOR = ["dimension", "tier", "provider", "model", "country"] + +// Each stat period and retention cohort is independent: ranks, upserts and +// deletes stay inside one (grain, period_key) or one cohort. A run is one plan +// and one write per period and cohort, so each can be retried on its own. +export const syncStats = Effect.fn("StatSync.sync")(function* (options?: { history?: boolean }) { + const plan = yield* planStatsSync(yield* DateTime.nowAsDate, { history: options?.history ?? false }) + const periods = yield* Effect.forEach(plan.periods, (period) => syncStatPeriod(period, plan.catalog), { + concurrency: 2, + }) + const cohorts = yield* Effect.forEach(plan.retentionPeriods, (period) => syncRetentionCohort(period, plan.catalog), { + concurrency: 4, + }) + yield* Effect.logInfo( + `stats sync complete ${JSON.stringify({ + history: plan.history, + startedAt: plan.startedAt, + periodStart: plan.periodStart, + periodEnd: plan.periodEnd, + rows: periods.reduce((sum, item) => sum + item.modelRows, 0), + providerRows: periods.reduce((sum, item) => sum + item.providerRows, 0), + geoRows: periods.reduce((sum, item) => sum + item.geoRows, 0), + retentionRows: cohorts.reduce((sum, item) => sum + item.rows, 0), + stage: Resource.App.stage, + })}`, + ) + return { + ok: true, + rows: periods.reduce((sum, item) => sum + item.modelRows, 0), + startedAt: plan.startedAt, + periodStart: plan.periodStart, + periodEnd: plan.periodEnd, + } satisfies SyncStatsResult +}) + +export const planStatsSync = Effect.fn("StatSync.plan")(function* ( + startedAt: Date, + options: { history: boolean; loadCatalog?: typeof loadCatalogIdentity }, +) { + const retentionStats = yield* RetentionStatRepo + const range = syncRange(startedAt, options.history) + const identity = yield* Effect.promise(options.loadCatalog ?? loadCatalogIdentity) + // Without the catalog every period written would attribute models to + // "unknown"; fail so the run retries instead. + if (!identity.catalog) return yield* Effect.fail(new CatalogUnavailableError()) + if (identity.stale) yield* Effect.logWarning("model catalog unavailable; using cached identity") + const retentionAvailable = yield* retentionStats.available() + yield* logRuntimeCheck() + + const plan: StatsSyncPlan = { + startedAt: startedAt.toISOString(), + history: options.history, + periodStart: range.periodStart.toISOString(), + periodEnd: range.periodEnd.toISOString(), + catalog: toCatalogSnapshot(identity.catalog), + periods: statPeriods(range.periodStart, range.periodEnd).map((period) => ({ + grain: period.grain, + key: period.key, + start: period.start.toISOString(), + end: period.end.toISOString(), + })), + retentionPeriods: retentionAvailable + ? retentionPeriods(range.retentionStart, range.retentionEnd).map((period) => ({ + start: period.start.toISOString(), + end: period.end.toISOString(), + returnStart: period.returnStart.toISOString(), + returnEnd: period.returnEnd.toISOString(), + })) + : [], + } + yield* Effect.logInfo( + `stats sync planned ${JSON.stringify({ + history: plan.history, + periodStart: plan.periodStart, + periodEnd: plan.periodEnd, + periods: plan.periods.length, + retentionPeriods: plan.retentionPeriods.length, + retentionAvailable, + })}`, + ) + return plan +}) + +export const syncStatPeriod = Effect.fn("StatSync.period")(function* ( + period: StatsSyncPeriod, + snapshot: CatalogSnapshot, +) { + const r2Sql = yield* R2Sql + const modelStats = yield* ModelStatRepo + const providerStats = yield* ProviderStatRepo + const geoStats = yield* GeoStatRepo + const catalog = fromCatalogSnapshot(snapshot) + const queries = buildPeriodStatsQueries( + { grain: period.grain, key: period.key, start: new Date(period.start), end: new Date(period.end) }, + undefined, + catalog, + ) + const rows = yield* Effect.forEach( + queries, + (query) => + r2Sql + .query(query, STATS_CURSOR) + .pipe( Effect.tapError((error) => Effect.logError( - `stats query failed ${JSON.stringify({ index, total: queries.length, error: error.message })}`, + `stats query failed ${JSON.stringify({ grain: period.grain, periodKey: period.key, error: error.message })}`, ), ), ), - { - concurrency: 4, - }, - ).pipe(Effect.map((batches) => batches.flat())) - const modelRows = modelRowsFromAggregates( - rows.filter((row) => row.dimension === "model").flatMap((row) => toModelAggregate(row, catalog)), - ) - const providerRows = providerRowsFromAggregates( - rows.filter((row) => row.dimension === "provider").flatMap((row) => toProviderAggregate(row, catalog)), - ) - const geoRows = geoRowsFromAggregates( - rows - .filter((row) => row.dimension === "geo" || row.dimension === "geo_model") - .flatMap((row) => toGeoAggregate(row, catalog)), - ) - const retentionAvailable = yield* retentionStats.available() - const retentionQueries = retentionAvailable - ? buildRetentionQueries(range.retentionStart, range.retentionEnd, undefined, catalog) - : [] - yield* Effect.logInfo(`stats sync querying retention ${JSON.stringify({ queries: retentionQueries.length })}`) - const retentionRows = retentionRowsFromAggregates( - yield* Effect.forEach( - retentionQueries, - (item) => - r2Sql.query(item.query).pipe( - Effect.tap((rows) => - Effect.logInfo( - `retention query complete ${JSON.stringify({ cohortDates: item.cohortDates, rows: rows.length })}`, - ), - ), - Effect.tapError((error) => - Effect.logError( - `retention query failed ${JSON.stringify({ cohortDates: item.cohortDates, error: error.message })}`, - ), - ), - ), - { concurrency: 4 }, - ).pipe( - Effect.map((batches) => batches.flatMap((batch) => batch.flatMap((row) => toRetentionAggregate(row, catalog)))), + { concurrency: "unbounded" }, + ).pipe(Effect.map((batches) => batches.flat())) + const modelRows = modelRowsFromAggregates( + rows.filter((row) => row.dimension === "model").flatMap((row) => toModelAggregate(row, catalog)), + ) + const providerRows = providerRowsFromAggregates( + rows.filter((row) => row.dimension === "provider").flatMap((row) => toProviderAggregate(row, catalog)), + ) + const geoRows = geoRowsFromAggregates( + rows + .filter((row) => row.dimension === "geo" || row.dimension === "geo_model") + .flatMap((row) => toGeoAggregate(row, catalog)), + ) + + yield* Effect.all([modelStats.upsert(modelRows), providerStats.upsert(providerRows), geoStats.upsert(geoRows)], { + concurrency: "unbounded", + discard: true, + }) + // Scope by the period rather than the rows written, so a period whose only + // rows were just hidden still drops them. + const scope = { grain: period.grain, periodKey: period.key, dataset: Resource.StatsSyncConfig.dataset } + yield* Effect.all( + [ + modelStats.deleteRetiredDimensions(scope, hiddenStatModels()), + providerStats.deleteRetiredDimensions(scope), + geoStats.deleteRetiredDimensions(scope, hiddenStatModels()), + ], + { concurrency: "unbounded", discard: true }, + ) + yield* Effect.all( + [ + modelStats.deleteUnknownDimensions( + modelRows.filter( + (row) => + row.provider !== "unknown" && + (catalog.models.get(row.model) ?? statProvider(row.model, "", "unknown")) === row.provider, + ), ), - ) + geoStats.deleteUnknownDimensions( + geoRows.filter( + (row) => + row.model !== "all" && + row.provider !== "unknown" && + (catalog.models.get(row.model ?? "all") ?? statProvider(row.model ?? "all", "", "unknown")) === + row.provider, + ), + ), + ], + { concurrency: "unbounded", discard: true }, + ) - yield* Effect.logInfo( - `stats sync writing aggregates ${JSON.stringify({ modelRows: modelRows.length, providerRows: providerRows.length, geoRows: geoRows.length, retentionRows: retentionRows.length })}`, - ) - yield* Effect.all( - [ - modelStats.upsert(modelRows), - providerStats.upsert(providerRows), - geoStats.upsert(geoRows), - retentionStats.replace(retentionRows, { - cohortDates: retentionQueries.flatMap((item) => item.cohortDates), - dataset: Resource.StatsSyncConfig.dataset, - tier: "Go", - }), - ], - { - concurrency: "unbounded", - discard: true, - }, - ) - yield* Effect.all( - [ - modelStats.deleteRetiredDimensions(modelRows, hiddenStatModels()), - providerStats.deleteRetiredDimensions(providerRows), - geoStats.deleteRetiredDimensions(geoRows, hiddenStatModels()), - ], - { concurrency: "unbounded", discard: true }, - ) - yield* Effect.all( - [ - modelStats.deleteUnknownDimensions( - modelRows.filter( - (row) => - row.provider !== "unknown" && - (catalog?.models.get(row.model) ?? statProvider(row.model, "", "unknown")) === row.provider, + const result = { + grain: period.grain, + periodKey: period.key, + modelRows: modelRows.length, + providerRows: providerRows.length, + geoRows: geoRows.length, + } + yield* Effect.logInfo(`stats period synced ${JSON.stringify(result)}`) + return result +}) + +export const syncRetentionCohort = Effect.fn("StatSync.retentionCohort")(function* ( + period: StatsSyncRetentionPeriod, + snapshot: CatalogSnapshot, +) { + const r2Sql = yield* R2Sql + const retentionStats = yield* RetentionStatRepo + const catalog = fromCatalogSnapshot(snapshot) + const item = buildPeriodRetentionQuery( + { + start: new Date(period.start), + end: new Date(period.end), + returnStart: new Date(period.returnStart), + returnEnd: new Date(period.returnEnd), + }, + undefined, + catalog, + ) + const rows = retentionRowsFromAggregates( + (yield* r2Sql + .query(item.query) + .pipe( + Effect.tapError((error) => + Effect.logError( + `retention query failed ${JSON.stringify({ cohortDates: item.cohortDates, error: error.message })}`, ), ), - geoStats.deleteUnknownDimensions( - geoRows.filter( - (row) => - row.model !== "all" && - row.provider !== "unknown" && - (catalog?.models.get(row.model ?? "all") ?? statProvider(row.model ?? "all", "", "unknown")) === - row.provider, - ), - ), - ], - { concurrency: "unbounded", discard: true }, - ) + )).flatMap((row) => toRetentionAggregate(row, catalog)), + ) + // Replace only after the query succeeds, so a failed query keeps the last cohort. + yield* retentionStats.replace(rows, { + cohortDates: item.cohortDates, + dataset: Resource.StatsSyncConfig.dataset, + tier: "Go", + }) - yield* Effect.logInfo( - `stats sync complete ${JSON.stringify({ - history: options?.history ?? false, - startedAt: startedAt.toISOString(), - periodStart: range.periodStart.toISOString(), - periodEnd: range.periodEnd.toISOString(), - rows: modelRows.length, - providerRows: providerRows.length, - geoRows: geoRows.length, - retentionRows: retentionRows.length, - retentionAvailable, - stage: Resource.App.stage, - })}`, - ) + const result = { cohortDate: item.cohortDates[0]!, rows: rows.length } + yield* Effect.logInfo(`retention cohort synced ${JSON.stringify(result)}`) + return result +}) - return { - ok: true, - rows: modelRows.length, - startedAt: startedAt.toISOString(), - periodStart: range.periodStart.toISOString(), - periodEnd: range.periodEnd.toISOString(), - } - }, -) +// A rejected R2 SQL query fails the same way on every attempt, and each attempt +// repeats its spend. Return the failure as a result so a Workflow step stores it +// instead of retrying, and a replay reads it instead of running the query again. +export function deterministicFailureAsResult(effect: Effect.Effect) { + return effect.pipe( + Effect.catchIf( + (error): error is E & R2SqlQueryError => error instanceof R2SqlQueryError, + (error) => (isDeterministicR2SqlError(error) ? Effect.succeed({ failed: error.message }) : Effect.fail(error)), + ), + ) +} function logRuntimeCheck() { return Effect.logInfo( diff --git a/packages/stats/server/Dockerfile b/packages/stats/server/Dockerfile deleted file mode 100644 index 1ee301af99a..00000000000 --- a/packages/stats/server/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM oven/bun:1.4.4-alpine AS base - -WORKDIR /app - -ENV NODE_ENV=production -ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=0 - -FROM base AS pruner - -COPY . . - -RUN bunx turbo@2.8.13 prune @opencode/stats-server --docker --no-update-notifier --no-color - -FROM base AS installer - -COPY --from=pruner /app/out/json/ ./ -COPY --from=pruner /app/bun.lock ./bun.lock - -# Bun 1.3.x needs the pruned workspace globs and lockfile metadata refreshed before the frozen production install. -RUN bun -e 'const packageJson = await Bun.file("package.json").json(); packageJson.workspaces.packages = Array.from(new Bun.Glob("packages/**/package.json").scanSync(".")).map((file) => file.slice(0, -"/package.json".length)).sort(); await Bun.write("package.json", JSON.stringify(packageJson, null, 2) + "\n")' -# Preserve locked transitive versions while refreshing the pruned workspace metadata. -RUN bun install --filter @opencode/stats-server --lockfile-only --ignore-scripts -RUN bun install --filter @opencode/stats-server --frozen-lockfile --production --ignore-scripts - -FROM base AS runner - -COPY --from=installer /app ./ -COPY --from=pruner /app/out/full/ ./ - -WORKDIR /app/packages/stats/server - -CMD ["bun", "src/stat-sync.ts"] diff --git a/packages/stats/server/package.json b/packages/stats/server/package.json index 775616e095c..014c66f87f2 100644 --- a/packages/stats/server/package.json +++ b/packages/stats/server/package.json @@ -5,22 +5,19 @@ "private": true, "type": "module", "license": "MIT", - "main": "./src/stat-sync.ts", - "exports": { - ".": "./src/stat-sync.ts" - }, "scripts": { - "start": "bun src/stat-sync.ts", + "sync": "bun src/sync.ts", + "test": "bun test --only-failures", "typecheck": "bun check --no-script", "typecheck:tsc": "tsc --noEmit" }, "dependencies": { - "@effect/platform-node": "catalog:", "@opencode/stats-core": "workspace:*", "effect": "catalog:", "sst": "catalog:" }, "devDependencies": { + "@cloudflare/workers-types": "catalog:", "@tsconfig/node22": "catalog:", "@types/bun": "catalog:", "@types/node": "catalog:", diff --git a/packages/stats/server/src/cron.test.ts b/packages/stats/server/src/cron.test.ts new file mode 100644 index 00000000000..af50422ffdf --- /dev/null +++ b/packages/stats/server/src/cron.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "bun:test" +import cron from "./cron" + +const controller = { cron: "5 * * * *", scheduledTime: Date.parse("2026-11-03T04:05:00.000Z") } + +describe("stats sync cron", () => { + test("starts this hour's run when no recent run exists", async () => { + const workflow = fakeWorkflow({}) + + await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never }) + + expect(workflow.created).toEqual([ + { id: "stats-hourly-2026-11-03T04", params: { history: false, startedAt: "2026-11-03T04:05:00.000Z" } }, + ]) + }) + + test("fails the tick instead of starting a run when a lookup fails for another reason", async () => { + const workflow = fakeWorkflow({ get: { "stats-hourly-2026-11-03T02": new Error("internal error") } }) + + await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + "internal error", + ) + expect(workflow.created).toEqual([]) + }) + + test("fails the tick when an instance's status cannot be read", async () => { + const workflow = fakeWorkflow({ status: { "stats-hourly-2026-11-03T03": new Error("status unavailable") } }) + + await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + "status unavailable", + ) + expect(workflow.created).toEqual([]) + }) + + test("treats a concurrent create of the same slot as a skip", async () => { + const workflow = fakeWorkflow({ create: new Error("(instance.already_exists) Instance already exists") }) + + await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never }) + + expect(workflow.created).toEqual([]) + }) + + test("still fails on other create errors", async () => { + const workflow = fakeWorkflow({ create: new Error("internal error") }) + + await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + "internal error", + ) + }) +}) + +// Instances not listed in `status` do not exist, and `get` rejects for them the +// way the Workflow binding does. +function fakeWorkflow(input: { get?: Record; status?: Record; create?: Error }) { + const created: { id?: string; params?: unknown }[] = [] + return { + created, + binding: { + get: async (id: string) => { + const failure = input.get?.[id] + if (failure) throw failure + const status = input.status?.[id] + if (status === undefined) throw new Error("instance.not_found") + return { + status: async () => { + if (status instanceof Error) throw status + return { status } + }, + } + }, + create: async (options: { id?: string; params?: unknown }) => { + if (input.create) throw input.create + created.push(options) + return { id: options.id } + }, + }, + } +} diff --git a/packages/stats/server/src/cron.ts b/packages/stats/server/src/cron.ts new file mode 100644 index 00000000000..524c1ffac97 --- /dev/null +++ b/packages/stats/server/src/cron.ts @@ -0,0 +1,53 @@ +import { + HISTORY_SCHEDULE, + instanceId, + isWorkflowError, + recentInstanceIds, + skipReason, + type StatsSyncParams, +} from "./schedule" + +type Env = { StatsSyncWorkflow: Workflow } + +export default { + async scheduled(controller: ScheduledController, env: Env) { + const scheduledAt = new Date(controller.scheduledTime) + const history = controller.cron === HISTORY_SCHEDULE + const id = instanceId(scheduledAt, history) + // Any lookup failure other than a missing instance fails the tick, so an + // unreadable status never starts a duplicate run. + const instances = await Promise.all( + recentInstanceIds(scheduledAt, history).map(async (item) => ({ + id: item, + status: await env.StatsSyncWorkflow.get(item) + .then((instance) => instance.status()) + .then((value) => value.status) + .catch((error) => { + if (isWorkflowError(error, "instance.not_found")) return undefined + throw error + }), + })), + ) + const skip = skipReason(id, instances) + if (skip) { + console.log(`stats sync skipped ${JSON.stringify({ id, ...skip })}`) + return + } + // Cron delivery is at least once, so a second tick for this slot can get here too. + const created = await env.StatsSyncWorkflow.create({ + id, + params: { history, startedAt: scheduledAt.toISOString() }, + }).then( + () => true, + (error) => { + if (isWorkflowError(error, "instance.already_exists")) return false + throw error + }, + ) + if (!created) { + console.log(`stats sync skipped ${JSON.stringify({ id, reason: "already created" })}`) + return + } + console.log(`stats sync started ${JSON.stringify({ id, history })}`) + }, +} diff --git a/packages/stats/server/src/schedule.test.ts b/packages/stats/server/src/schedule.test.ts new file mode 100644 index 00000000000..154c4ee4b00 --- /dev/null +++ b/packages/stats/server/src/schedule.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test" +import { instanceId, isWorkflowError, recentInstanceIds, skipReason } from "./schedule" + +describe("stats sync schedule", () => { + test("names one instance per hourly slot and per history day", () => { + const at = new Date("2026-11-03T03:20:00.000Z") + + expect(instanceId(at, false)).toBe("stats-hourly-2026-11-03T03") + expect(instanceId(at, true)).toBe("stats-history-2026-11-03") + expect(instanceId(new Date("2026-11-03T03:05:59.999Z"), false)).toBe("stats-hourly-2026-11-03T03") + }) + + test("probes this hour and the previous five for an hourly run", () => { + expect(recentInstanceIds(new Date("2026-11-01T02:05:00.000Z"), false)).toEqual([ + "stats-hourly-2026-11-01T02", + "stats-hourly-2026-11-01T01", + "stats-hourly-2026-11-01T00", + "stats-hourly-2026-10-31T23", + "stats-hourly-2026-10-31T22", + "stats-hourly-2026-10-31T21", + ]) + }) + + test("probes today and the previous two days for a history run", () => { + expect(recentInstanceIds(new Date("2026-11-01T03:20:00.000Z"), true)).toEqual([ + "stats-history-2026-11-01", + "stats-history-2026-10-31", + "stats-history-2026-10-30", + ]) + }) + + test("lets hourly and history runs overlap", () => { + const at = new Date("2026-11-03T04:05:00.000Z") + const running = new Set(["stats-history-2026-11-03", "stats-hourly-2026-11-03T03"]) + const decide = (history: boolean) => + skipReason( + instanceId(at, history), + recentInstanceIds(at, history).map((id) => ({ id, status: running.has(id) ? "running" : undefined })), + ) + + expect(decide(false)).toEqual({ + reason: "previous run active", + instance: "stats-hourly-2026-11-03T03", + status: "running", + }) + running.delete("stats-hourly-2026-11-03T03") + expect(decide(false)).toBeUndefined() + expect(decide(true)).toEqual({ reason: "already created", instance: "stats-history-2026-11-03", status: "running" }) + running.delete("stats-history-2026-11-03") + running.add("stats-hourly-2026-11-03T04") + expect(decide(true)).toBeUndefined() + }) + + test("starts when no recent instance is active", () => { + expect( + skipReason("stats-hourly-2026-11-03T04", [ + { id: "stats-hourly-2026-11-03T04", status: undefined }, + { id: "stats-hourly-2026-11-03T03", status: "complete" }, + { id: "stats-hourly-2026-11-03T02", status: "errored" }, + { id: "stats-hourly-2026-11-03T01", status: "terminated" }, + ]), + ).toBeUndefined() + }) + + test.each(["queued", "running", "waiting", "paused", "waitingForPause"])( + "skips while a previous run is %s", + (status) => { + expect( + skipReason("stats-hourly-2026-11-03T04", [ + { id: "stats-hourly-2026-11-03T04", status: undefined }, + { id: "stats-hourly-2026-11-03T03", status }, + ]), + ).toEqual({ reason: "previous run active", instance: "stats-hourly-2026-11-03T03", status }) + }, + ) + + test("skips a slot that already has an instance in any state", () => { + expect(skipReason("stats-history-2026-11-03", [{ id: "stats-history-2026-11-03", status: "complete" }])).toEqual({ + reason: "already created", + instance: "stats-history-2026-11-03", + status: "complete", + }) + }) + + test("recognizes Workflow binding errors by their code only", () => { + expect(isWorkflowError(new Error("instance.not_found"), "instance.not_found")).toBe(true) + expect(isWorkflowError(new Error("(instance.not_found) Instance does not exist"), "instance.not_found")).toBe(true) + expect( + isWorkflowError(new Error("(instance.already_exists) Instance already exists"), "instance.already_exists"), + ).toBe(true) + expect(isWorkflowError(new Error("internal error"), "instance.not_found")).toBe(false) + expect(isWorkflowError(new Error("(instance.already_exists) Instance already exists"), "instance.not_found")).toBe( + false, + ) + expect(isWorkflowError("instance.not_found", "instance.not_found")).toBe(false) + }) +}) diff --git a/packages/stats/server/src/schedule.ts b/packages/stats/server/src/schedule.ts new file mode 100644 index 00000000000..8304efc67c2 --- /dev/null +++ b/packages/stats/server/src/schedule.ts @@ -0,0 +1,43 @@ +// Keep in sync with the StatsSyncCron schedules in infra/stats.ts. +export const HOURLY_SCHEDULE = "5 * * * *" +export const HISTORY_SCHEDULE = "20 3 * * *" + +const HOUR_MS = 3_600_000 +const DAY_MS = 24 * HOUR_MS +// How far back a run can still be active: hourly runs finish within hours, history +// runs within days. +const HOURLY_LOOKBACK = 6 +const HISTORY_LOOKBACK = 3 +const ACTIVE = new Set(["queued", "running", "waiting", "paused", "waitingForPause"]) + +export type StatsSyncParams = { history?: boolean; startedAt?: string } + +// One ID per schedule slot, so a repeated trigger cannot start a second run. +export function instanceId(scheduledAt: Date, history: boolean) { + if (history) return `stats-history-${scheduledAt.toISOString().slice(0, 10)}` + return `stats-hourly-${scheduledAt.toISOString().slice(0, 13)}` +} + +// The newest IDs of the same kind a run may still be using, starting with this +// slot's own. Hourly and history runs cover different periods and their writes +// are idempotent, so one kind never waits for the other. +export function recentInstanceIds(scheduledAt: Date, history: boolean) { + return Array.from({ length: history ? HISTORY_LOOKBACK : HOURLY_LOOKBACK }, (_, index) => + instanceId(new Date(scheduledAt.getTime() - index * (history ? DAY_MS : HOUR_MS)), history), + ) +} + +// Runs of one kind never overlap: two would repeat R2 SQL spend for the same +// periods. `undefined` means the instance does not exist. +export function skipReason(id: string, instances: { id: string; status: string | undefined }[]) { + const own = instances.find((instance) => instance.id === id && instance.status !== undefined) + if (own) return { reason: "already created", instance: own.id, status: own.status } + const active = instances.find((instance) => instance.status !== undefined && ACTIVE.has(instance.status)) + if (active) return { reason: "previous run active", instance: active.id, status: active.status } +} + +// Workflow bindings reject with plain Errors whose message carries the code, as +// "instance.not_found" or "(instance.already_exists) Instance already exists". +export function isWorkflowError(error: unknown, code: "instance.not_found" | "instance.already_exists") { + return error instanceof Error && error.message.includes(code) +} diff --git a/packages/stats/server/src/stat-sync.ts b/packages/stats/server/src/stat-sync.ts deleted file mode 100644 index 6cf47e0c27d..00000000000 --- a/packages/stats/server/src/stat-sync.ts +++ /dev/null @@ -1,74 +0,0 @@ -import * as NodeRuntime from "@effect/platform-node/NodeRuntime" -import { ModelStatRepo } from "@opencode/stats-core/domain/model" -import { R2Sql } from "@opencode/stats-core/r2-sql" -import { layer as statsLayer } from "@opencode/stats-core/runtime" -import { syncStats } from "@opencode/stats-core/stat-sync" -import { syncRange } from "@opencode/stats-core/sync-range" -import { Cause, Duration, Effect, Layer, Schedule } from "effect" - -const SYNC_INTERVAL = "1 hour" -const SYNC_INTERVAL_MS = 3_600_000 - -const runtimeLayer = Layer.mergeAll(statsLayer, R2Sql.layer) - -const daemon = Effect.gen(function* () { - yield* Effect.logInfo("stats sync daemon started") - yield* initialDelay() - - // Every pass recomputes the current ISO week. Once per UTC day, including after - // boot, it also starts a history pass for the rest of the display window. That - // pass takes over an hour, so it runs in the background instead of holding back - // the next hourly pass. - let lastHistoryDay = "" - let historyRunning = false - const history = syncStats({ history: true }).pipe( - Effect.catchCause((cause) => - Effect.logWarning(`stats history sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`), - ), - Effect.ensuring( - Effect.sync(() => { - historyRunning = false - }), - ), - ) - const pass = Effect.gen(function* () { - yield* syncStats() - const today = new Date().toISOString().slice(0, 10) - // A history pass still running past midnight defers the next day's until it ends. - if (lastHistoryDay === today || historyRunning) return - lastHistoryDay = today - historyRunning = true - yield* Effect.forkChild(history) - }).pipe( - Effect.tapCause((cause) => - Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`), - ), - // Without a retry, one transient R2 SQL failure leaves the site stale until the - // next hourly pass. Two retries cap the extra query spend per interval. - Effect.retry({ times: 2, schedule: Schedule.spaced("10 minutes") }), - Effect.ignoreCause, - ) - yield* pass.pipe(Effect.repeat(Schedule.fixed(SYNC_INTERVAL))) -}).pipe(Effect.forkScoped) - -// A restarted daemon must not immediately re-run the R2 SQL pass; resume the -// hourly cadence from the last completed sync instead. This caps the query spend -// of a crash loop at one pass per interval. Only hourly passes write the current -// week, so a history pass that rewrote older periods cannot push this back. -const initialDelay = Effect.fnUntraced(function* () { - const modelStats = yield* ModelStatRepo - const lastSynced = yield* modelStats - .lastSyncedAt(syncRange(new Date()).periodStart) - .pipe(Effect.catchCause(() => Effect.succeed(null))) - if (!lastSynced) return - const delayMs = Math.min(SYNC_INTERVAL_MS - (Date.now() - lastSynced.getTime()), SYNC_INTERVAL_MS) - if (delayMs <= 0) return - yield* Effect.logInfo( - `stats sync delaying first pass ${JSON.stringify({ lastSyncedAt: lastSynced.toISOString(), delayMs })}`, - ) - yield* Effect.sleep(Duration.millis(delayMs)) -}) - -NodeRuntime.runMain(Layer.launch(Layer.effectDiscard(daemon).pipe(Layer.provide(runtimeLayer))), { - disableErrorReporting: true, -}) diff --git a/packages/stats/server/src/sync.ts b/packages/stats/server/src/sync.ts new file mode 100644 index 00000000000..c1f67d9d099 --- /dev/null +++ b/packages/stats/server/src/sync.ts @@ -0,0 +1,10 @@ +import { R2Sql } from "@opencode/stats-core/r2-sql" +import { layer } from "@opencode/stats-core/runtime" +import { syncStats } from "@opencode/stats-core/stat-sync" +import { Effect, Layer } from "effect" + +// Runs one sync pass in this process, for local use. Deployed stages run the +// same steps in the StatsSyncWorkflow. +await Effect.runPromise( + syncStats({ history: process.argv.includes("--history") }).pipe(Effect.provide(Layer.mergeAll(layer, R2Sql.layer))), +) diff --git a/packages/stats/server/src/workflow.ts b/packages/stats/server/src/workflow.ts new file mode 100644 index 00000000000..4038aa08dba --- /dev/null +++ b/packages/stats/server/src/workflow.ts @@ -0,0 +1,78 @@ +import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep, type WorkflowStepConfig } from "cloudflare:workers" +import { R2Sql } from "@opencode/stats-core/r2-sql" +import { layer } from "@opencode/stats-core/runtime" +import { + deterministicFailureAsResult, + planStatsSync, + syncRetentionCohort, + syncStatPeriod, + type SyncStatsError, +} from "@opencode/stats-core/stat-sync" +import { Effect, Layer } from "effect" +import type { StatsSyncParams } from "./schedule" + +const runtimeLayer = Layer.mergeAll(layer, R2Sql.layer) +// Each step runs one period's R2 SQL queries (10-minute request timeout, with +// page retries) and its PlanetScale writes. +const STEP: WorkflowStepConfig = { + timeout: "30 minutes", + retries: { limit: 2, delay: "1 minute", backoff: "exponential" }, +} + +export class StatsSyncWorkflow extends WorkflowEntrypoint { + async run(event: WorkflowEvent, step: WorkflowStep) { + // A dashboard trigger can start an instance without params. + const plan = await step.do("plan", STEP, () => + run( + planStatsSync(new Date(event.payload?.startedAt ?? event.timestamp), { + history: event.payload?.history ?? false, + }), + ), + ) + + // Steps run one at a time to keep one period's rows in memory. A failed + // period does not stop the others; the run still ends as errored. + const failed: string[] = [] + const fail = (name: string, error: unknown) => { + failed.push(name) + console.error( + `stats sync step failed ${JSON.stringify({ + step: name, + error: error instanceof Error ? error.message : String(error), + cause: error instanceof Error && error.cause !== undefined ? String(error.cause) : undefined, + })}`, + ) + } + const settle = (name: string, attempt: Promise) => + attempt.then( + (result) => { + if ("failed" in result) fail(name, result.failed) + }, + (error) => fail(name, error), + ) + for (const period of plan.periods) { + const name = `${period.grain} ${period.key}` + await settle( + name, + step.do(name, STEP, () => run(deterministicFailureAsResult(syncStatPeriod(period, plan.catalog)))), + ) + } + for (const period of plan.retentionPeriods) { + const name = `retention ${period.start.slice(0, 10)}` + await settle( + name, + step.do(name, STEP, () => run(deterministicFailureAsResult(syncRetentionCohort(period, plan.catalog)))), + ) + } + if (failed.length > 0) throw new Error(`stats sync failed for ${failed.join(", ")}`) + return { periods: plan.periods.length, retentionPeriods: plan.retentionPeriods.length } + } +} + +// SST's generated entry wraps the default export; workerd rejects a module whose +// default export is undefined. +export default {} + +function run(effect: Effect.Effect>) { + return Effect.runPromise(effect.pipe(Effect.provide(runtimeLayer))) +} diff --git a/packages/stats/server/tsconfig.json b/packages/stats/server/tsconfig.json index ec63580497c..b7dd6615d54 100644 --- a/packages/stats/server/tsconfig.json +++ b/packages/stats/server/tsconfig.json @@ -7,7 +7,8 @@ "moduleResolution": "bundler", "strict": true, "noEmit": true, - "types": ["bun", "node"] + "types": ["@cloudflare/workers-types", "node"] }, - "include": ["src", "../core/src/resource.d.ts"] + "include": ["src", "../core/src/resource.d.ts"], + "exclude": ["src/**/*.test.ts"] } diff --git a/sst.config.ts b/sst.config.ts index 09b0c712165..19624722cb9 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -38,7 +38,7 @@ export default $config({ const stage = await import("./infra/stage.js") await import("./infra/app.js") - const stats = stage.deployAws ? await import("./infra/stats.js") : undefined + const stats = stage.deployStats ? await import("./infra/stats.js") : undefined await import("./infra/console.js") await import("./infra/share.js")