feat(stats): run the stats sync on Cloudflare Workflows (#61461)

This commit is contained in:
Adam
2026-11-03 13:01:07 -06:00
committed by GitHub
Unverified
parent 826381e44d
commit 156bb61010
30 changed files with 1175 additions and 445 deletions
+1 -1
View File
@@ -769,12 +769,12 @@
"name": "@opencode/stats-server",
"version": "2.2.5",
"dependencies": {
"@effect/platform-node": "catalog:",
"@opencode/stats-core": "workspace:*",
"effect": "catalog:",
"sst": "catalog:",
},
"devDependencies": {
"@cloudflare/workers-types": "catalog:",
"@tsconfig/node22": "catalog:",
"@types/bun": "catalog:",
"@types/node": "catalog:",
+1 -1
View File
@@ -6,7 +6,7 @@ export const domain = (() => {
export const zoneID = "430ba34c138cfb5360826c4909f99be8"
export const awsStage = $app.stage === "production" ? "production" : "dev"
export const deployAws = $app.stage === awsStage
export const deployStats = $app.stage === awsStage
if ($app.stage === "production") {
new cloudflare.DnsRecord("TrustCenter", {
+37 -21
View File
@@ -1,11 +1,6 @@
import { EMAILOCTOPUS_API_KEY } from "./app"
import { domain } from "./stage"
// Keep the deployed names and arguments: the R2 stats sync still uses this VPC
// and cluster after the legacy lake is removed.
const lakeVpc = new sst.aws.Vpc("LakeVpc")
const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
////////////////
// DATABASE
////////////////
@@ -101,25 +96,46 @@ const r2Sql = new sst.Linkable("R2Sql", {
},
})
export const statSync = new sst.aws.Service("StatsSyncService", {
cluster: lakeCluster,
architecture: "arm64",
cpu: "0.25 vCPU",
// 0.5 GB caused an OOM crash loop: every restart immediately re-ran the 4 Athena
// stats queries (~$5/pass) every ~5 minutes instead of hourly.
memory: "2 GB",
image: {
context: ".",
dockerfile: "packages/stats/server/Dockerfile",
},
command: ["bun", "src/stat-sync.ts"],
// One instance per run: a plan step, then one step per stat period and retention
// cohort. See packages/stats/README.md for triggering a run by hand.
export const statsSyncWorkflow = new sst.cloudflare.Workflow("StatsSyncWorkflow", {
handler: "packages/stats/server/src/workflow.ts",
className: "StatsSyncWorkflow",
link: [database, r2Sql, r2SqlAuthToken, statsHiddenModels, statsSyncConfig],
scaling: {
min: 1,
max: 1,
transform: {
worker: {
transform: {
worker: (args) => {
// Raise CPU time from the 30-second default to the 5-minute maximum for
// ranking a full week of geo rows.
args.limits = { cpuMs: 300_000 }
args.observability = { enabled: true }
},
},
},
},
})
// Schedules must match packages/stats/server/src/schedule.ts, which maps the
// daily one to a history run.
if (!$dev)
new sst.cloudflare.Cron("StatsSyncCron", {
schedules: ["5 * * * *", "20 3 * * *"],
worker: {
handler: "packages/stats/server/src/cron.ts",
link: [statsSyncWorkflow],
transform: {
worker: (args) => {
args.observability = { enabled: true }
},
},
},
})
new sst.x.DevCommand("StatsSync", {
link: [database, r2Sql, r2SqlAuthToken, statsHiddenModels, statsSyncConfig],
dev: {
command: "bun src/stat-sync.ts",
command: "bun src/sync.ts",
directory: "packages/stats/server",
autostart: false,
},
+26 -3
View File
@@ -1,19 +1,42 @@
# OpenCode Stats
Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; deployable Lambda entrypoints live in `function`.
Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; the Cloudflare Workers that sync stats live in `server`.
## Packages
- `app`: SolidStart frontend/site.
- `core`: Effect services, app config, Drizzle schema/migrations, and stats domains.
- `function`: Lambda handlers that call into `core` services.
- `server`: the `StatsSyncWorkflow` Workflow, the `StatsSyncCron` Worker that starts it, and a local sync script.
## Commands
- `bun run dev:stats` from the repo root starts the SolidStart app.
- `bun run --cwd packages/stats/app typecheck` typechecks the site.
- `bun run --cwd packages/stats/core typecheck` typechecks the Effect/database package.
- `bun run --cwd packages/stats/function typecheck` typechecks Lambda entrypoints.
- `bun run --cwd packages/stats/server typecheck` typechecks the sync Workers.
## Stats sync
The sync reads aggregates from R2 SQL and writes them to the stats PlanetScale database. It runs on the `dev` and `production` stages as a Cloudflare Workflow, `StatsSyncWorkflow`, defined in `infra/stats.ts`.
Each run has a `plan` step, then one step per stat period (`week 2026-W45`, `day 2026-11-03`) and one per retention cohort (`retention 2026-10-05`). The plan snapshots the model catalog once, so every step attributes models the same way. Steps run one at a time and retry twice with exponential backoff. A rejected query (HTTP 400, 401 or 403) or a result over the 10,000-row limit fails its step after one attempt, since a retry repeats the same R2 SQL spend. A failed step does not stop the others, but the run ends as errored and logs each failed step with its error. Because queries run per period, expect an hourly run to take about 15 to 25 minutes and a history run about 2 to 3.5 hours. These are estimates until measured on a deployed stage.
The `StatsSyncCron` Worker starts runs on two schedules:
- `5 * * * *`: an hourly run of the current ISO week and the most recent complete retention cohort, with instance ID `stats-hourly-YYYY-MM-DDTHH`.
- `20 3 * * *`: a daily history run of the rest of the 56-day display window, with instance ID `stats-history-YYYY-MM-DD`.
Times and IDs are in UTC. Before starting a run, the Cron checks this slot's ID and the recent IDs of the same kind: the five previous hours for an hourly run, the two previous days for a history run. It skips the run if any of them is queued, running, waiting or paused, or if this slot's instance already exists, and logs the reason. Hourly and history runs can overlap, since they cover different periods and their writes are idempotent.
To start a run by hand, find the Workflow's name with `bunx wrangler workflows list` (it is `opencode-<stage>-statssyncworkflowworkflow-<suffix>`), then trigger it with an ID from the Cron's convention so scheduled runs of the same kind see it and wait:
```sh
bunx wrangler workflows trigger <workflow-name> '{"history":true}' --id stats-history-2026-11-03
```
Omit `history` for an hourly run. If the ID already exists, that slot has run; wait for the next one. You can also trigger a run from the Workflow's page in the Cloudflare dashboard; without params it runs as an hourly pass.
To run one sync pass on your machine against a stage's resources, run `bun sst shell --stage=<stage> -- bun run --cwd packages/stats/server sync`, adding `--history` for a history pass. `sst dev` also offers it as the `StatsSync` command.
## Public page caching
+3 -2
View File
@@ -9,10 +9,11 @@ export class AppConfigValue extends Schema.Class<AppConfigValue>("AppConfigValue
const decodeAppConfigValue = Schema.decodeUnknownSync(AppConfigValue)
// Read the linked stage lazily, so importing this module in a Worker does not
// touch bindings at module scope.
const config = Config.all({
stage: Config.succeed(Resource.App.stage),
publicUrl: Config.String("PUBLIC_URL").pipe(Config.withDefault("http://localhost:3000")),
}).pipe(Config.map(decodeAppConfigValue))
}).pipe(Config.map((value) => decodeAppConfigValue({ ...value, stage: Resource.App.stage })))
export class AppConfig extends Context.Service<AppConfig, AppConfigValue>()("@opencode/stats/AppConfig") {
static readonly config = config
+27 -38
View File
@@ -1,7 +1,6 @@
import { Client } from "@planetscale/database"
import { drizzle } from "drizzle-orm/planetscale-serverless"
import { migrate as drizzleMigrate } from "drizzle-orm/planetscale-serverless/migrator"
import { Config, ConfigProvider, Effect, Layer, Schema } from "effect"
import { Config, ConfigProvider, Effect, Layer, Option, Schema } from "effect"
import * as Context from "effect/Context"
import { Resource } from "sst/resource"
@@ -10,14 +9,17 @@ export type DatabaseUrl = typeof DatabaseUrl.Type
export class DatabaseSettings extends Schema.Class<DatabaseSettings>("DatabaseSettings")({
url: DatabaseUrl,
migrationsDir: Schema.NonEmptyString,
}) {}
const decodeDatabaseSettings = Schema.decodeUnknownSync(DatabaseSettings)
const config = Config.all({
url: Config.NonEmptyString("DATABASE_URL").pipe(Config.withDefault(Resource.StatsDatabase.url)),
migrationsDir: Config.NonEmptyString("DATABASE_MIGRATIONS_DIR").pipe(Config.withDefault("./migrations")),
// Read the linked database lazily, so importing this module in a Worker does not
// touch bindings at module scope.
url: Config.NonEmptyString("DATABASE_URL").pipe(
Config.option,
Config.map(Option.getOrElse(() => Resource.StatsDatabase.url)),
),
}).pipe(Config.map(decodeDatabaseSettings))
export class DatabaseConfig extends Context.Service<DatabaseConfig, DatabaseSettings>()(
@@ -30,8 +32,18 @@ export class DatabaseConfig extends Context.Service<DatabaseConfig, DatabaseSett
)
}
// Each request is one statement (a transaction is several), so this bounds a
// single upsert chunk, not a whole write. A 500-row upsert normally finishes in
// seconds; without a bound, a hung request holds the step until its 30-minute timeout.
const DATABASE_REQUEST_TIMEOUT_MS = 60_000
function makeDrizzle(settings: DatabaseSettings) {
return drizzle({ client: new Client({ url: settings.url }) })
return drizzle({
client: new Client({
url: settings.url,
fetch: (input, init) => fetch(input, { ...init, signal: AbortSignal.timeout(DATABASE_REQUEST_TIMEOUT_MS) }),
}),
})
}
export type Drizzle = ReturnType<typeof makeDrizzle>
@@ -47,7 +59,10 @@ export class DatabaseError extends Error {
readonly _tag = "DatabaseError"
constructor(input: { cause: unknown }) {
super("Database operation failed", { cause: input.cause })
const reason = rootCause(input.cause)
super(reason instanceof Error ? `Database operation failed: ${reason.toString()}` : "Database operation failed", {
cause: input.cause,
})
this.name = "DatabaseError"
}
@@ -56,38 +71,12 @@ export class DatabaseError extends Error {
}
}
export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause }))
export class MigrationError extends Error {
readonly _tag = "MigrationError"
constructor(input: { message: string; cause?: unknown }) {
super(input.message, { cause: input.cause })
this.name = "MigrationError"
}
// Drizzle wraps the driver's error in one whose message holds the whole query and
// its parameters, which for an upsert chunk is hundreds of rows. Name the driver's error.
function rootCause(cause: unknown): unknown {
return cause instanceof Error && cause.cause !== undefined ? rootCause(cause.cause) : cause
}
export const migrate = Effect.fn("Database.migrate")(function* () {
const settings = yield* DatabaseConfig
yield* Effect.logInfo("applying database migrations").pipe(
Effect.annotateLogs({ migrationsDir: settings.migrationsDir }),
)
const result = yield* Effect.tryPromise({
try: () =>
drizzleMigrate(drizzle({ client: new Client({ url: settings.url }) }), {
migrationsFolder: settings.migrationsDir,
}),
catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }),
})
if (result)
return yield* Effect.fail(
new MigrationError({
message: `Failed to initialize database migrations: ${result.exitCode}`,
}),
)
yield* Effect.logInfo("database migrations complete").pipe(
Effect.annotateLogs({ migrationsDir: settings.migrationsDir }),
)
})
export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause }))
export const layer = Layer.mergeAll(DatabaseConfig.layer, DrizzleClient.layer.pipe(Layer.provide(DatabaseConfig.layer)))
@@ -9,6 +9,21 @@ export type CatalogIdentity = {
models: ReadonlyMap<string, string>
}
// A JSON-safe copy of a catalog identity, so a sync plan can carry one snapshot
// to every step of a run.
export type CatalogSnapshot = {
offerings: [string, string][]
models: [string, string][]
}
export function toCatalogSnapshot(identity: CatalogIdentity): CatalogSnapshot {
return { offerings: [...identity.offerings], models: [...identity.models] }
}
export function fromCatalogSnapshot(snapshot: CatalogSnapshot): CatalogIdentity {
return { offerings: new Map(snapshot.offerings), models: new Map(snapshot.models) }
}
export async function loadCatalogIdentity() {
if (cache.value && (cache.expiresAt ?? 0) > Date.now()) return { catalog: cache.value, stale: false }
return fetch(CATALOG_URL, { signal: AbortSignal.timeout(10_000) })
+8 -10
View File
@@ -18,6 +18,7 @@ import {
UPSERT_CHUNK_SIZE,
withUniqueUsersFallback,
type StatBaseAggregate,
type StatPeriodScope,
} from "./stat"
export type GeoStatRow = typeof geoStat.$inferInsert
@@ -56,7 +57,7 @@ export declare namespace GeoStatRepo {
}) => Effect.Effect<GeoStatRow[], DatabaseError>
readonly upsert: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
readonly deleteRetiredDimensions: (
rows: GeoStatRow[],
scope: StatPeriodScope,
hiddenModels: readonly string[],
) => Effect.Effect<void, DatabaseError>
readonly deleteUnknownDimensions: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
@@ -190,23 +191,20 @@ export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Servic
}
const deleteRetiredDimensions = Effect.fn("GeoStatRepo.deleteRetiredDimensions")(function* (
rows: GeoStatRow[],
scope: StatPeriodScope,
hiddenModels: readonly string[],
) {
const scope = statRowScope(rows)
if (!scope) return
yield* Effect.tryPromise({
try: () =>
db
.delete(geoStat)
.where(
and(
inArray(geoStat.grain, scope.grains),
inArray(geoStat.period_key, scope.periodKeys),
inArray(geoStat.dataset, scope.datasets),
inArray(geoStat.client, scope.clients),
inArray(geoStat.source, scope.sources),
eq(geoStat.grain, scope.grain),
eq(geoStat.period_key, scope.periodKey),
eq(geoStat.dataset, scope.dataset),
eq(geoStat.client, "all"),
eq(geoStat.source, "all"),
or(
inArray(geoStat.provider, RETIRED_STAT_PROVIDERS),
inArray(geoStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]),
+39 -28
View File
@@ -23,6 +23,7 @@ import {
startOfIsoWeek,
startOfUtcDay,
type StatBaseAggregate,
type StatGrain,
} from "./stat"
export type StatDimension = "model" | "provider" | "geo" | "geo_model"
@@ -47,24 +48,24 @@ const GO_PRODUCTS = ["go", "go-plus"]
// and carry no usage. Counting them would publish any requested model name as a stats page.
const ROUTED_ONLY = "attempts > 0"
// R2 SQL results are cursor-paginated after aggregation. Separate usage and geo
// queries per day/week avoid combining costly distinct user/session aggregates
// with the high-cardinality geo dimensions.
export type StatPeriod = { grain: StatGrain; key: string; start: Date; end: Date }
export type RetentionPeriod = { start: Date; end: Date; returnStart: Date; returnEnd: Date }
export function buildStatsQueries(
periodStart: Date,
periodEnd: Date,
input?: StatsQuerySource,
catalog?: CatalogIdentity,
) {
const source = input ?? {
namespace: Resource.R2Sql.namespace,
table: Resource.R2Sql.table,
dataset: Resource.StatsSyncConfig.dataset,
hiddenModels: hiddenStatModels(),
}
return [...statPeriods("week", periodStart, periodEnd), ...statPeriods("day", periodStart, periodEnd)].flatMap(
(period) => [buildStatsQuery(period, source, "usage", catalog), buildStatsQuery(period, source, "geo", catalog)],
)
return statPeriods(periodStart, periodEnd).flatMap((period) => buildPeriodStatsQueries(period, input, catalog))
}
// R2 SQL results are cursor-paginated after aggregation. Separate usage and geo
// queries per day/week avoid combining costly distinct user/session aggregates
// with the high-cardinality geo dimensions.
export function buildPeriodStatsQueries(period: StatPeriod, input?: StatsQuerySource, catalog?: CatalogIdentity) {
const source = input ?? defaultSource()
return [buildStatsQuery(period, source, "usage", catalog), buildStatsQuery(period, source, "geo", catalog)]
}
export function buildRetentionQueries(
@@ -73,26 +74,32 @@ export function buildRetentionQueries(
input?: StatsQuerySource,
catalog?: CatalogIdentity,
): RetentionQuery[] {
const source = input ?? {
return retentionPeriods(periodStart, periodEnd).map((period) => buildPeriodRetentionQuery(period, input, catalog))
}
// Bound the user-level joins to one activity week and its return week.
// Combining the entire display window makes full syncs much more expensive.
export function buildPeriodRetentionQuery(
period: RetentionPeriod,
input?: StatsQuerySource,
catalog?: CatalogIdentity,
): RetentionQuery {
return {
cohortDates: [period.start.toISOString().slice(0, 10)],
query: buildRetentionQuery([period], input ?? defaultSource(), catalog),
}
}
function defaultSource(): StatsQuerySource {
return {
namespace: Resource.R2Sql.namespace,
table: Resource.R2Sql.table,
dataset: Resource.StatsSyncConfig.dataset,
hiddenModels: hiddenStatModels(),
}
const periods = retentionPeriods(periodStart, periodEnd)
// Bound the user-level joins to one activity week and its return week.
// Combining the entire display window makes full syncs much more expensive.
return periods.map((period) => ({
cohortDates: [period.start.toISOString().slice(0, 10)],
query: buildRetentionQuery([period], source, catalog),
}))
}
function buildRetentionQuery(
periods: { start: Date; end: Date; returnStart: Date; returnEnd: Date }[],
source: StatsQuerySource,
catalog?: CatalogIdentity,
) {
function buildRetentionQuery(periods: RetentionPeriod[], source: StatsQuerySource, catalog?: CatalogIdentity) {
const first = periods[0]
const last = periods.at(-1)!
const scanStartValue = sqlString(first.start.toISOString())
@@ -209,7 +216,7 @@ LIMIT 10000
}
function buildStatsQuery(
period: { grain: "day" | "week"; key: string; start: Date; end: Date },
period: StatPeriod,
source: StatsQuerySource,
family: StatsQueryFamily,
catalog?: CatalogIdentity,
@@ -477,7 +484,11 @@ function sqlString(value: string) {
return `'${value.replace(/'/g, "''")}'`
}
function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) {
export function statPeriods(periodStart: Date, periodEnd: Date) {
return [...grainPeriods("week", periodStart, periodEnd), ...grainPeriods("day", periodStart, periodEnd)]
}
function grainPeriods(grain: StatGrain, periodStart: Date, periodEnd: Date): StatPeriod[] {
const interval = grain === "day" ? DAY_MS : WEEK_MS
const first = grain === "day" ? startOfUtcDay(periodStart) : startOfIsoWeek(periodStart)
const count = Math.max(0, Math.ceil((periodEnd.getTime() - first.getTime()) / interval))
@@ -492,7 +503,7 @@ function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date)
})
}
function retentionPeriods(periodStart: Date, periodEnd: Date) {
export function retentionPeriods(periodStart: Date, periodEnd: Date): RetentionPeriod[] {
const first = startOfIsoWeek(periodStart)
const completeEnd = startOfIsoWeek(periodEnd)
const count = Math.max(0, Math.floor((completeEnd.getTime() - first.getTime()) / WEEK_MS) - 1)
+11 -27
View File
@@ -1,4 +1,4 @@
import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm"
import { and, asc, eq, inArray, or } from "drizzle-orm"
import { Effect, Layer } from "effect"
import * as Context from "effect/Context"
import { DatabaseError, DrizzleClient } from "../database"
@@ -10,7 +10,6 @@ import {
DATA_SITE_TIERS,
inserted,
omitUniqueUsers,
periodKeyFor,
rankBy,
statPeriodKey,
statRowScope,
@@ -19,6 +18,7 @@ import {
UPSERT_CHUNK_SIZE,
withUniqueUsersFallback,
type StatBaseAggregate,
type StatPeriodScope,
} from "./stat"
export type ModelStatRow = typeof modelStat.$inferInsert
@@ -45,10 +45,9 @@ export type ModelStatMetric = {
export declare namespace ModelStatRepo {
export interface Service {
readonly listDaily: () => Effect.Effect<ModelStatMetric[], DatabaseError>
readonly lastSyncedAt: (since: Date) => Effect.Effect<Date | null, DatabaseError>
readonly upsert: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
readonly deleteRetiredDimensions: (
rows: ModelStatRow[],
scope: StatPeriodScope,
hiddenModels: readonly string[],
) => Effect.Effect<void, DatabaseError>
readonly deleteUnknownDimensions: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
@@ -117,18 +116,6 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
})
})
const lastSyncedAt = Effect.fn("ModelStatRepo.lastSyncedAt")(function* (since: Date) {
const result = yield* Effect.tryPromise({
try: () =>
db
.select({ value: max(modelStat.updated_at) })
.from(modelStat)
.where(and(eq(modelStat.grain, "day"), gte(modelStat.period_key, periodKeyFor("day", since)))),
catch: (cause) => DatabaseError.make({ cause }),
})
return result[0]?.value ?? null
})
const upsert = Effect.fn("ModelStatRepo.upsert")(function* (rows: ModelStatRow[]) {
yield* Effect.forEach(
chunks(rows, UPSERT_CHUNK_SIZE),
@@ -177,23 +164,20 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
}
const deleteRetiredDimensions = Effect.fn("ModelStatRepo.deleteRetiredDimensions")(function* (
rows: ModelStatRow[],
scope: StatPeriodScope,
hiddenModels: readonly string[],
) {
const scope = statRowScope(rows)
if (!scope) return
yield* Effect.tryPromise({
try: () =>
db
.delete(modelStat)
.where(
and(
inArray(modelStat.grain, scope.grains),
inArray(modelStat.period_key, scope.periodKeys),
inArray(modelStat.dataset, scope.datasets),
inArray(modelStat.client, scope.clients),
inArray(modelStat.source, scope.sources),
eq(modelStat.grain, scope.grain),
eq(modelStat.period_key, scope.periodKey),
eq(modelStat.dataset, scope.dataset),
eq(modelStat.client, "all"),
eq(modelStat.source, "all"),
or(
inArray(modelStat.provider, RETIRED_STAT_PROVIDERS),
inArray(modelStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]),
@@ -253,7 +237,7 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
)
})
return ModelStatRepo.of({ listDaily, lastSyncedAt, upsert, deleteRetiredDimensions, deleteUnknownDimensions })
return ModelStatRepo.of({ listDaily, upsert, deleteRetiredDimensions, deleteUnknownDimensions })
}),
)
}
@@ -293,7 +277,7 @@ function rankRows(rows: ModelStatRow[]) {
return Object.values(
rows.reduce<Record<string, ModelStatRow[]>>((result, row) => {
const key = statPeriodKey(row)
result[key] = [...(result[key] ?? []), row]
;(result[key] ??= []).push(row)
return result
}, {}),
).flatMap((group) => {
+8 -11
View File
@@ -11,12 +11,12 @@ import {
inserted,
omitUniqueUsers,
rankRowsWithMarketShare,
statRowScope,
synthesizeAllTierRows,
toStatBaseRow,
UPSERT_CHUNK_SIZE,
withUniqueUsersFallback,
type StatBaseAggregate,
type StatPeriodScope,
} from "./stat"
export type ProviderStatRow = typeof providerStat.$inferInsert
@@ -41,7 +41,7 @@ export declare namespace ProviderStatRepo {
readonly source?: string
}) => Effect.Effect<ProviderStatRow[], DatabaseError>
readonly upsert: (rows: ProviderStatRow[]) => Effect.Effect<void, DatabaseError>
readonly deleteRetiredDimensions: (rows: ProviderStatRow[]) => Effect.Effect<void, DatabaseError>
readonly deleteRetiredDimensions: (scope: StatPeriodScope) => Effect.Effect<void, DatabaseError>
}
}
@@ -157,22 +157,19 @@ export class ProviderStatRepo extends Context.Service<ProviderStatRepo, Provider
}
const deleteRetiredDimensions = Effect.fn("ProviderStatRepo.deleteRetiredDimensions")(function* (
rows: ProviderStatRow[],
scope: StatPeriodScope,
) {
const scope = statRowScope(rows)
if (!scope) return
yield* Effect.tryPromise({
try: () =>
db
.delete(providerStat)
.where(
and(
inArray(providerStat.grain, scope.grains),
inArray(providerStat.period_key, scope.periodKeys),
inArray(providerStat.dataset, scope.datasets),
inArray(providerStat.client, scope.clients),
inArray(providerStat.source, scope.sources),
eq(providerStat.grain, scope.grain),
eq(providerStat.period_key, scope.periodKey),
eq(providerStat.dataset, scope.dataset),
eq(providerStat.client, "all"),
eq(providerStat.source, "all"),
inArray(providerStat.provider, RETIRED_STAT_PROVIDERS),
),
),
+13 -18
View File
@@ -54,28 +54,23 @@ export class RetentionStatRepo extends Context.Service<RetentionStatRepo, Retent
) {
if (scope.cohortDates.length === 0) return
// Readers never see a cohort emptied by an interrupted replace.
yield* Effect.tryPromise({
try: () =>
db
.delete(modelRetention)
.where(
and(
inArray(modelRetention.cohort_date, scope.cohortDates),
eq(modelRetention.dataset, scope.dataset),
eq(modelRetention.tier, scope.tier),
),
),
db.transaction(async (tx) => {
await tx
.delete(modelRetention)
.where(
and(
inArray(modelRetention.cohort_date, scope.cohortDates),
eq(modelRetention.dataset, scope.dataset),
eq(modelRetention.tier, scope.tier),
),
)
for (const chunk of chunks(rows, UPSERT_CHUNK_SIZE)) await tx.insert(modelRetention).values(chunk)
}),
catch: (cause) => DatabaseError.make({ cause }),
})
yield* Effect.forEach(
chunks(rows, UPSERT_CHUNK_SIZE),
(chunk) =>
Effect.tryPromise({
try: () => db.insert(modelRetention).values(chunk),
catch: (cause) => DatabaseError.make({ cause }),
}),
{ discard: true },
)
})
return RetentionStatRepo.of({ available, replace })
+4 -1
View File
@@ -173,6 +173,9 @@ export function statPeriodKey(row: StatBaseRow) {
return [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source].join("\u0000")
}
// One synced period. The sync writes only the all-client, all-source rows.
export type StatPeriodScope = { grain: StatGrain; periodKey: string; dataset: string }
export function statRowScope(rows: StatBaseRow[]) {
if (rows.length === 0) return
return {
@@ -221,7 +224,7 @@ export function rankRowsWithMarketShare<T extends StatBaseRow>(
return Object.values(
rows.reduce<Record<string, T[]>>((result, row) => {
const key = groupKey(row)
result[key] = [...(result[key] ?? []), row]
;(result[key] ??= []).push(row)
return result
}, {}),
).flatMap((group) => {
@@ -569,7 +569,7 @@ function rankModelRows(rows: ModelStatRow[]) {
return Object.values(
rows.reduce<Record<string, ModelStatRow[]>>((result, row) => {
const key = statPeriodKey(row)
result[key] = [...(result[key] ?? []), row]
;(result[key] ??= []).push(row)
return result
}, {}),
).flatMap((group) => {
+34 -3
View File
@@ -1,4 +1,35 @@
import { Effect } from "effect"
import { layer, migrate } from "./database"
import { migrate } from "drizzle-orm/planetscale-serverless/migrator"
import { Config, ConfigProvider, Effect } from "effect"
import { DrizzleClient, layer } from "./database"
await Effect.runPromise(migrate().pipe(Effect.provide(layer)))
// The migrator reads migration files from disk, so it stays out of the
// Worker-safe database module.
class MigrationError extends Error {
readonly _tag = "MigrationError"
constructor(input: { message: string; cause?: unknown }) {
super(input.message, { cause: input.cause })
this.name = "MigrationError"
}
}
const applyMigrations = Effect.fn("Database.migrate")(function* () {
const migrationsDir = yield* Config.NonEmptyString("DATABASE_MIGRATIONS_DIR")
.pipe(Config.withDefault("./migrations"))
.parse(ConfigProvider.fromEnv())
const db = yield* DrizzleClient
yield* Effect.logInfo("applying database migrations").pipe(Effect.annotateLogs({ migrationsDir }))
const result = yield* Effect.tryPromise({
try: () => migrate(db, { migrationsFolder: migrationsDir }),
catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }),
})
if (result)
return yield* Effect.fail(
new MigrationError({
message: `Failed to initialize database migrations: ${result.exitCode}`,
}),
)
yield* Effect.logInfo("database migrations complete").pipe(Effect.annotateLogs({ migrationsDir }))
})
await Effect.runPromise(applyMigrations().pipe(Effect.provide(layer)))
+34 -1
View File
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite"
import { Effect, Fiber } from "effect"
import { TestClock } from "effect/testing"
import { expect, test } from "bun:test"
import { queryR2SqlPages, R2SqlQueryError, type R2SqlData } from "./r2-sql"
import { isDeterministicR2SqlError, queryR2SqlPages, R2SqlQueryError, type R2SqlData } from "./r2-sql"
test("includes the transport failure in the message used by sync logging", () => {
const cause = new DOMException("The operation timed out.", "TimeoutError")
@@ -158,3 +158,36 @@ test("bounds transient retries and preserves the final error", async () => {
expect(calls).toHaveLength(3)
expect(result).toBe(error)
}, 20000)
test("treats rejected queries, rejected credentials and row-limit results as deterministic", async () => {
const rowLimit = await Effect.runPromise(
queryR2SqlPages("SELECT * FROM aggregates LIMIT 10000", undefined, () =>
Effect.succeed(Array.from({ length: 10000 }, () => ({}))),
).pipe(Effect.flip),
)
expect(
[
rowLimit,
new R2SqlQueryError({ message: "syntax error", status: 400 }),
new R2SqlQueryError({ message: "unauthorized", status: 401 }),
new R2SqlQueryError({ message: "forbidden", status: 403 }),
].map(isDeterministicR2SqlError),
).toEqual([true, true, true, true])
})
test("leaves transient and unexplained failures retryable", () => {
expect(
[
new R2SqlQueryError({ message: "query timeout", status: 400, code: 40005 }),
new R2SqlQueryError({ message: "Unprocessable Entity", status: 422, code: 60104 }),
new R2SqlQueryError({ message: "rate limited", status: 429 }),
new R2SqlQueryError({ message: "unavailable", status: 503 }),
new R2SqlQueryError({ message: "Failed to run R2 SQL stats query", cause: new TypeError("connection closed") }),
new R2SqlQueryError({
message: "Failed to run R2 SQL stats query",
cause: new DOMException("The operation timed out.", "TimeoutError"),
}),
].map(isDeterministicR2SqlError),
).toEqual([false, false, false, false, false, false])
})
+37 -15
View File
@@ -2,7 +2,7 @@ import { Context, Effect, Layer, Schedule, Schema } from "effect"
import { Resource } from "sst/resource"
const R2_SQL_MAX_ROWS = 10_000
const R2_SQL_TIMEOUT_MS = 15 * 60_000
const R2_SQL_TIMEOUT_MS = 10 * 60_000
const R2SqlValue = Schema.Union([Schema.String, Schema.Number, Schema.Boolean, Schema.Null])
const R2SqlResponse = Schema.Struct({
success: Schema.Boolean,
@@ -26,8 +26,16 @@ export class R2SqlQueryError extends Error {
readonly requestId?: string
readonly status?: number
readonly code?: number
readonly rowLimit: boolean
constructor(input: { message: string; requestId?: string; status?: number; code?: number; cause?: unknown }) {
constructor(input: {
message: string
requestId?: string
status?: number
code?: number
rowLimit?: boolean
cause?: unknown
}) {
super(input.cause instanceof Error ? `${input.message}: ${input.cause.toString()}` : input.message, {
cause: input.cause,
})
@@ -35,9 +43,29 @@ export class R2SqlQueryError extends Error {
this.requestId = input.requestId
this.status = input.status
this.code = input.code
this.rowLimit = input.rowLimit ?? false
}
}
// Retrying these repeats the same R2 SQL spend for the same failure: a rejected
// query, rejected credentials, or a result that will hit the row limit again.
export function isDeterministicR2SqlError(error: R2SqlQueryError) {
if (isTransientR2SqlError(error)) return false
return error.rowLimit || error.status === 400 || error.status === 401 || error.status === 403
}
// 40005 is a query timeout and 60104 an intermittent "Unprocessable Entity", both
// sent with 4xx statuses. fetch rejects with a TypeError when the connection drops.
function isTransientR2SqlError(error: R2SqlQueryError) {
return (
error.code === 40005 ||
error.code === 60104 ||
error.status === 429 ||
(error.status !== undefined && error.status >= 500) ||
error.cause instanceof TypeError
)
}
export declare namespace R2Sql {
export interface Service {
readonly query: (query: string, columns?: readonly string[]) => Effect.Effect<R2SqlData[], R2SqlQueryError>
@@ -67,23 +95,16 @@ export const queryR2SqlPages = Effect.fn("R2Sql.query")(function* (
fetchRows(columns?.length ? pageQuery(query, columns, cursor) : query),
).pipe(
// Retry only this page; a transient R2 failure must not discard the whole
// pass. 40005 is a query timeout, 60104 an intermittent "Unprocessable
// Entity", and fetch rejects with a TypeError when the connection drops.
// Syntax, auth, row-limit, and 15-minute request timeout errors still fail.
// period. Syntax, auth, row-limit, and request timeout errors still fail.
Effect.retry({
times: 2,
schedule: Schedule.exponential("5 seconds"),
while: (error) =>
error.code === 40005 ||
error.code === 60104 ||
error.status === 429 ||
(error.status !== undefined && error.status >= 500) ||
error.cause instanceof TypeError,
while: isTransientR2SqlError,
}),
)
if (page.length >= R2_SQL_MAX_ROWS && !columns?.length)
return yield* Effect.fail(
new R2SqlQueryError({ message: `R2 SQL stats query reached the ${R2_SQL_MAX_ROWS} row limit` }),
new R2SqlQueryError({ message: `R2 SQL stats query reached the ${R2_SQL_MAX_ROWS} row limit`, rowLimit: true }),
)
rows.push(...page)
if (page.length < R2_SQL_MAX_ROWS) return rows
@@ -112,8 +133,9 @@ const fetchRows = Effect.fn("R2Sql.fetchRows")(function* (query: string) {
try: (signal) => {
const options = {
method: "POST",
// Analytical queries can exceed Bun's default five-minute idle timer.
// Bound the whole request and cancel it when the sync is interrupted.
// Bun's fetch, used by the local CLI, otherwise stops at a five-minute
// idle timer; Workers ignore the option. Bound the whole request and
// cancel it when the sync is interrupted.
timeout: false,
signal: AbortSignal.any([signal, AbortSignal.timeout(R2_SQL_TIMEOUT_MS)]),
headers: {
@@ -122,7 +144,7 @@ const fetchRows = Effect.fn("R2Sql.fetchRows")(function* (query: string) {
},
body: JSON.stringify({ query }),
}
return Bun.fetch(
return fetch(
`https://api.sql.cloudflarestorage.com/api/v1/accounts/${Resource.R2Sql.accountId}/r2-sql/query/${Resource.R2Sql.bucket}`,
options,
)
+267
View File
@@ -0,0 +1,267 @@
import { describe, expect, test } from "bun:test"
import { Effect, Layer } from "effect"
import { catalogIdentity, type CatalogSnapshot } from "./domain/catalog-identity"
import type { StatPeriodScope } from "./domain/stat"
import type { R2SqlData } from "./r2-sql"
process.env.SST_RESOURCE_App = JSON.stringify({ name: "opencode", stage: "test" })
process.env.SST_RESOURCE_R2Sql = JSON.stringify({
accountId: "account",
bucket: "bucket",
namespace: "inference",
table: "generation",
})
process.env.SST_RESOURCE_StatsSyncConfig = JSON.stringify({ dataset: "zen" })
process.env.SST_RESOURCE_StatsHiddenModels = JSON.stringify({ value: "secret-model" })
const { GeoStatRepo } = await import("./domain/geo")
const { ModelStatRepo } = await import("./domain/model")
const { ProviderStatRepo } = await import("./domain/provider")
const { RetentionStatRepo } = await import("./domain/retention")
const { R2Sql, R2SqlQueryError } = await import("./r2-sql")
const { CatalogUnavailableError, deterministicFailureAsResult, planStatsSync, syncRetentionCohort, syncStatPeriod } =
await import("./stat-sync")
const { DatabaseError } = await import("./database")
const catalog: CatalogSnapshot = { offerings: [["opencode/model-a", "lab"]], models: [["model-a", "lab"]] }
const loadCatalog = () =>
Promise.resolve({
catalog: catalogIdentity({
models: { "lab/model-a": {} },
providers: { opencode: { models: { "model-a": { canonical_model_id: "lab/model-a" } } } },
}),
stale: false,
})
describe("stats sync plan", () => {
test("fails instead of planning a run without the model catalog", async () => {
const error = await Effect.runPromise(
planStatsSync(new Date("2026-10-23T19:05:00.000Z"), {
history: false,
loadCatalog: () => Promise.resolve({ catalog: undefined, stale: true }),
}).pipe(Effect.flip, Effect.provide(services({ events: [] }))),
)
expect(error).toBeInstanceOf(CatalogUnavailableError)
})
test("plans the current week for hourly runs and the rest of the window for history runs", async () => {
const startedAt = new Date("2026-10-23T19:05:00.000Z")
const plan = (history: boolean) =>
Effect.runPromise(
planStatsSync(startedAt, { history, loadCatalog }).pipe(Effect.provide(services({ events: [] }))),
)
const hourly = await plan(false)
expect(hourly.catalog).toEqual({ offerings: [["opencode/model-a", "lab"]], models: [["model-a", "lab"]] })
expect(hourly.periods.map((period) => `${period.grain} ${period.key}`)).toEqual([
"week 2026-W43",
"day 2026-10-19",
"day 2026-10-20",
"day 2026-10-21",
"day 2026-10-22",
"day 2026-10-23",
])
expect(hourly.periods.at(-1)).toEqual({
grain: "day",
key: "2026-10-23",
start: "2026-10-23T00:00:00.000Z",
end: "2026-10-23T19:00:00.000Z",
})
expect(hourly.retentionPeriods).toEqual([
{
start: "2026-10-05T00:00:00.000Z",
end: "2026-10-12T00:00:00.000Z",
returnStart: "2026-10-12T00:00:00.000Z",
returnEnd: "2026-10-19T00:00:00.000Z",
},
])
const history = await plan(true)
expect(history.periods.filter((period) => period.grain === "week").map((period) => period.key)).toEqual([
"2026-W35",
"2026-W36",
"2026-W37",
"2026-W38",
"2026-W39",
"2026-W40",
"2026-W41",
"2026-W42",
])
expect(history.periods.filter((period) => period.grain === "day")).toHaveLength(52)
expect(history.periods.at(-1)?.key).toBe("2026-10-18")
expect(history.retentionPeriods.map((period) => period.start.slice(0, 10))).toEqual([
"2026-08-24",
"2026-08-31",
"2026-09-07",
"2026-09-14",
"2026-09-21",
"2026-09-28",
])
})
})
describe("stats sync steps", () => {
test("drops retired and hidden dimensions from a period with no remaining rows", async () => {
const events: string[] = []
const result = await Effect.runPromise(
syncStatPeriod(
{ grain: "day", key: "2026-10-22", start: "2026-10-22T00:00:00.000Z", end: "2026-10-23T00:00:00.000Z" },
catalog,
).pipe(Effect.provide(services({ events }))),
)
expect(result).toEqual({ grain: "day", periodKey: "2026-10-22", modelRows: 0, providerRows: 0, geoRows: 0 })
expect(events.filter((event) => event.startsWith("deleteRetired")).toSorted()).toEqual([
'deleteRetired geo {"grain":"day","periodKey":"2026-10-22","dataset":"zen"} secret-model',
'deleteRetired model {"grain":"day","periodKey":"2026-10-22","dataset":"zen"} secret-model',
'deleteRetired provider {"grain":"day","periodKey":"2026-10-22","dataset":"zen"}',
])
})
test("replaces a retention cohort only after its query returns", async () => {
const events: string[] = []
const result = await Effect.runPromise(
syncRetentionCohort(retentionPeriod, catalog).pipe(
Effect.provide(
services({
events,
rows: [
{
cohort_date: "2026-10-05",
dataset: "zen",
tier: "Go",
provider: "lab",
model: "model-a",
eligible_users: "10",
retained_users: "4",
},
],
}),
),
),
)
expect(result).toEqual({ cohortDate: "2026-10-05", rows: 1 })
expect(events).toEqual(["query", 'replace ["2026-10-05"] zen Go model-a:10:4'])
})
test("keeps the existing cohort when its query fails", async () => {
const events: string[] = []
const failure = new R2SqlQueryError({ message: "unavailable", status: 400 })
const error = await Effect.runPromise(
syncRetentionCohort(retentionPeriod, catalog).pipe(Effect.flip, Effect.provide(services({ events, failure }))),
)
expect(error).toBe(failure)
expect(events).toEqual(["query"])
})
})
describe("stats sync failures", () => {
test("returns a deterministic R2 SQL failure as the result", async () => {
const result = await Effect.runPromise(
deterministicFailureAsResult(
Effect.fail(new R2SqlQueryError({ message: "R2 SQL stats query failed: syntax error", status: 400 })),
),
)
expect(result).toEqual({ failed: "R2 SQL stats query failed: syntax error" })
})
test("keeps transient R2 SQL and database failures as failures so the step retries", async () => {
const transient = new R2SqlQueryError({ message: "query timeout", status: 400, code: 40005 })
const database = new DatabaseError({ cause: new Error("connection reset") })
expect(await Effect.runPromise(deterministicFailureAsResult(Effect.fail(transient)).pipe(Effect.flip))).toBe(
transient,
)
expect(await Effect.runPromise(deterministicFailureAsResult(Effect.fail(database)).pipe(Effect.flip))).toBe(
database,
)
})
test("names the driver's error in a database failure without the wrapped query", () => {
const driver = new Error("target: stats.-.primary: vttablet: rpc error: code = Unavailable")
const wrapped = Object.assign(new Error("Failed query: insert into model_stat ...\nparams: 1,2,3"), {
cause: driver,
})
expect(new DatabaseError({ cause: wrapped }).message).toBe(
"Database operation failed: Error: target: stats.-.primary: vttablet: rpc error: code = Unavailable",
)
expect(new DatabaseError({ cause: "timeout" }).message).toBe("Database operation failed")
})
})
const retentionPeriod = {
start: "2026-10-05T00:00:00.000Z",
end: "2026-10-12T00:00:00.000Z",
returnStart: "2026-10-12T00:00:00.000Z",
returnEnd: "2026-10-19T00:00:00.000Z",
}
function services(input: { events: string[]; rows?: R2SqlData[]; failure?: InstanceType<typeof R2SqlQueryError> }) {
const unused = () => Effect.die("unused")
const retired = (table: string) => (scope: StatPeriodScope, hidden?: readonly string[]) =>
Effect.sync(() => {
input.events.push(`deleteRetired ${table} ${JSON.stringify(scope)}${hidden ? ` ${hidden.join(",")}` : ""}`)
})
return Layer.mergeAll(
Layer.succeed(
R2Sql,
R2Sql.of({
query: () =>
Effect.suspend(() => {
input.events.push("query")
return input.failure ? Effect.fail(input.failure) : Effect.succeed(input.rows ?? [])
}),
}),
),
Layer.succeed(
ModelStatRepo,
ModelStatRepo.of({
listDaily: unused,
upsert: () => Effect.void,
deleteRetiredDimensions: retired("model"),
deleteUnknownDimensions: () => Effect.void,
}),
),
Layer.succeed(
ProviderStatRepo,
ProviderStatRepo.of({
listDaily: unused,
listByPeriod: unused,
upsert: () => Effect.void,
deleteRetiredDimensions: retired("provider"),
}),
),
Layer.succeed(
GeoStatRepo,
GeoStatRepo.of({
listDaily: unused,
listByPeriod: unused,
upsert: () => Effect.void,
deleteRetiredDimensions: retired("geo"),
deleteUnknownDimensions: () => Effect.void,
}),
),
Layer.succeed(
RetentionStatRepo,
RetentionStatRepo.of({
available: () => Effect.succeed(true),
replace: (rows, scope) =>
Effect.sync(() => {
input.events.push(
`replace ${JSON.stringify(scope.cohortDates)} ${scope.dataset} ${scope.tier} ${rows
.map((row) => `${row.model}:${row.eligible_users}:${row.retained_users}`)
.join(",")}`,
)
}),
}),
),
)
}
+243 -149
View File
@@ -1,12 +1,19 @@
import { DateTime, Effect } from "effect"
import { Resource } from "sst/resource"
import { DatabaseError } from "./database"
import { loadCatalogIdentity } from "./domain/catalog-identity"
import {
fromCatalogSnapshot,
loadCatalogIdentity,
toCatalogSnapshot,
type CatalogSnapshot,
} from "./domain/catalog-identity"
import { GeoStatRepo, rowsFromAggregates as geoRowsFromAggregates } from "./domain/geo"
import {
buildRetentionQueries,
buildStatsQueries,
buildPeriodRetentionQuery,
buildPeriodStatsQueries,
hiddenStatModels,
retentionPeriods,
statPeriods,
toGeoAggregate,
toModelAggregate,
toProviderAggregate,
@@ -16,169 +23,256 @@ import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./
import { statProvider } from "./domain/model-normalization"
import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider"
import { RetentionStatRepo, rowsFromAggregates as retentionRowsFromAggregates } from "./domain/retention"
import { R2Sql, R2SqlQueryError } from "./r2-sql"
import type { StatGrain } from "./domain/stat"
import { isDeterministicR2SqlError, R2Sql, R2SqlQueryError } from "./r2-sql"
import { syncRange } from "./sync-range"
// Plans cross a Workflow step boundary, so every value is plain JSON.
export type StatsSyncPeriod = { grain: StatGrain; key: string; start: string; end: string }
export type StatsSyncRetentionPeriod = { start: string; end: string; returnStart: string; returnEnd: string }
export type StatsSyncPlan = {
startedAt: string
history: boolean
periodStart: string
periodEnd: string
catalog: CatalogSnapshot
periods: StatsSyncPeriod[]
retentionPeriods: StatsSyncRetentionPeriod[]
}
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
export type SyncStatsError = R2SqlQueryError | DatabaseError
type SyncStatsServices = R2Sql | ModelStatRepo | ProviderStatRepo | GeoStatRepo | RetentionStatRepo
export type SyncStatsError = R2SqlQueryError | DatabaseError | CatalogUnavailableError
export const syncStats: (options?: {
history?: boolean
}) => Effect.Effect<SyncStatsResult, SyncStatsError, SyncStatsServices> = Effect.fn("StatSync.sync")(
function* (options?: { history?: boolean }) {
const startedAt = yield* DateTime.nowAsDate
const range = syncRange(startedAt, options?.history)
const r2Sql = yield* R2Sql
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
const geoStats = yield* GeoStatRepo
const retentionStats = yield* RetentionStatRepo
const identity = yield* Effect.promise(loadCatalogIdentity)
const catalog = identity.catalog
if (identity.stale)
yield* Effect.logWarning(
catalog
? "model catalog unavailable; using cached identity"
: "model catalog unavailable; using legacy stats attribution",
)
export class CatalogUnavailableError extends Error {
readonly _tag = "CatalogUnavailableError"
yield* logRuntimeCheck()
constructor() {
super("Model catalog unavailable")
this.name = "CatalogUnavailableError"
}
}
const queries = buildStatsQueries(range.periodStart, range.periodEnd, undefined, catalog)
yield* Effect.logInfo(
`stats sync started ${JSON.stringify({ history: options?.history ?? false, periodStart: range.periodStart, periodEnd: range.periodEnd, queries: queries.length })}`,
)
const rows = yield* Effect.forEach(
queries,
(query, index) =>
r2Sql.query(query, ["dimension", "tier", "provider", "model", "country"]).pipe(
Effect.tap((rows) =>
Effect.logInfo(
`stats query complete ${JSON.stringify({ index, total: queries.length, rows: rows.length })}`,
),
),
const STATS_CURSOR = ["dimension", "tier", "provider", "model", "country"]
// Each stat period and retention cohort is independent: ranks, upserts and
// deletes stay inside one (grain, period_key) or one cohort. A run is one plan
// and one write per period and cohort, so each can be retried on its own.
export const syncStats = Effect.fn("StatSync.sync")(function* (options?: { history?: boolean }) {
const plan = yield* planStatsSync(yield* DateTime.nowAsDate, { history: options?.history ?? false })
const periods = yield* Effect.forEach(plan.periods, (period) => syncStatPeriod(period, plan.catalog), {
concurrency: 2,
})
const cohorts = yield* Effect.forEach(plan.retentionPeriods, (period) => syncRetentionCohort(period, plan.catalog), {
concurrency: 4,
})
yield* Effect.logInfo(
`stats sync complete ${JSON.stringify({
history: plan.history,
startedAt: plan.startedAt,
periodStart: plan.periodStart,
periodEnd: plan.periodEnd,
rows: periods.reduce((sum, item) => sum + item.modelRows, 0),
providerRows: periods.reduce((sum, item) => sum + item.providerRows, 0),
geoRows: periods.reduce((sum, item) => sum + item.geoRows, 0),
retentionRows: cohorts.reduce((sum, item) => sum + item.rows, 0),
stage: Resource.App.stage,
})}`,
)
return {
ok: true,
rows: periods.reduce((sum, item) => sum + item.modelRows, 0),
startedAt: plan.startedAt,
periodStart: plan.periodStart,
periodEnd: plan.periodEnd,
} satisfies SyncStatsResult
})
export const planStatsSync = Effect.fn("StatSync.plan")(function* (
startedAt: Date,
options: { history: boolean; loadCatalog?: typeof loadCatalogIdentity },
) {
const retentionStats = yield* RetentionStatRepo
const range = syncRange(startedAt, options.history)
const identity = yield* Effect.promise(options.loadCatalog ?? loadCatalogIdentity)
// Without the catalog every period written would attribute models to
// "unknown"; fail so the run retries instead.
if (!identity.catalog) return yield* Effect.fail(new CatalogUnavailableError())
if (identity.stale) yield* Effect.logWarning("model catalog unavailable; using cached identity")
const retentionAvailable = yield* retentionStats.available()
yield* logRuntimeCheck()
const plan: StatsSyncPlan = {
startedAt: startedAt.toISOString(),
history: options.history,
periodStart: range.periodStart.toISOString(),
periodEnd: range.periodEnd.toISOString(),
catalog: toCatalogSnapshot(identity.catalog),
periods: statPeriods(range.periodStart, range.periodEnd).map((period) => ({
grain: period.grain,
key: period.key,
start: period.start.toISOString(),
end: period.end.toISOString(),
})),
retentionPeriods: retentionAvailable
? retentionPeriods(range.retentionStart, range.retentionEnd).map((period) => ({
start: period.start.toISOString(),
end: period.end.toISOString(),
returnStart: period.returnStart.toISOString(),
returnEnd: period.returnEnd.toISOString(),
}))
: [],
}
yield* Effect.logInfo(
`stats sync planned ${JSON.stringify({
history: plan.history,
periodStart: plan.periodStart,
periodEnd: plan.periodEnd,
periods: plan.periods.length,
retentionPeriods: plan.retentionPeriods.length,
retentionAvailable,
})}`,
)
return plan
})
export const syncStatPeriod = Effect.fn("StatSync.period")(function* (
period: StatsSyncPeriod,
snapshot: CatalogSnapshot,
) {
const r2Sql = yield* R2Sql
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
const geoStats = yield* GeoStatRepo
const catalog = fromCatalogSnapshot(snapshot)
const queries = buildPeriodStatsQueries(
{ grain: period.grain, key: period.key, start: new Date(period.start), end: new Date(period.end) },
undefined,
catalog,
)
const rows = yield* Effect.forEach(
queries,
(query) =>
r2Sql
.query(query, STATS_CURSOR)
.pipe(
Effect.tapError((error) =>
Effect.logError(
`stats query failed ${JSON.stringify({ index, total: queries.length, error: error.message })}`,
`stats query failed ${JSON.stringify({ grain: period.grain, periodKey: period.key, error: error.message })}`,
),
),
),
{
concurrency: 4,
},
).pipe(Effect.map((batches) => batches.flat()))
const modelRows = modelRowsFromAggregates(
rows.filter((row) => row.dimension === "model").flatMap((row) => toModelAggregate(row, catalog)),
)
const providerRows = providerRowsFromAggregates(
rows.filter((row) => row.dimension === "provider").flatMap((row) => toProviderAggregate(row, catalog)),
)
const geoRows = geoRowsFromAggregates(
rows
.filter((row) => row.dimension === "geo" || row.dimension === "geo_model")
.flatMap((row) => toGeoAggregate(row, catalog)),
)
const retentionAvailable = yield* retentionStats.available()
const retentionQueries = retentionAvailable
? buildRetentionQueries(range.retentionStart, range.retentionEnd, undefined, catalog)
: []
yield* Effect.logInfo(`stats sync querying retention ${JSON.stringify({ queries: retentionQueries.length })}`)
const retentionRows = retentionRowsFromAggregates(
yield* Effect.forEach(
retentionQueries,
(item) =>
r2Sql.query(item.query).pipe(
Effect.tap((rows) =>
Effect.logInfo(
`retention query complete ${JSON.stringify({ cohortDates: item.cohortDates, rows: rows.length })}`,
),
),
Effect.tapError((error) =>
Effect.logError(
`retention query failed ${JSON.stringify({ cohortDates: item.cohortDates, error: error.message })}`,
),
),
),
{ concurrency: 4 },
).pipe(
Effect.map((batches) => batches.flatMap((batch) => batch.flatMap((row) => toRetentionAggregate(row, catalog)))),
{ concurrency: "unbounded" },
).pipe(Effect.map((batches) => batches.flat()))
const modelRows = modelRowsFromAggregates(
rows.filter((row) => row.dimension === "model").flatMap((row) => toModelAggregate(row, catalog)),
)
const providerRows = providerRowsFromAggregates(
rows.filter((row) => row.dimension === "provider").flatMap((row) => toProviderAggregate(row, catalog)),
)
const geoRows = geoRowsFromAggregates(
rows
.filter((row) => row.dimension === "geo" || row.dimension === "geo_model")
.flatMap((row) => toGeoAggregate(row, catalog)),
)
yield* Effect.all([modelStats.upsert(modelRows), providerStats.upsert(providerRows), geoStats.upsert(geoRows)], {
concurrency: "unbounded",
discard: true,
})
// Scope by the period rather than the rows written, so a period whose only
// rows were just hidden still drops them.
const scope = { grain: period.grain, periodKey: period.key, dataset: Resource.StatsSyncConfig.dataset }
yield* Effect.all(
[
modelStats.deleteRetiredDimensions(scope, hiddenStatModels()),
providerStats.deleteRetiredDimensions(scope),
geoStats.deleteRetiredDimensions(scope, hiddenStatModels()),
],
{ concurrency: "unbounded", discard: true },
)
yield* Effect.all(
[
modelStats.deleteUnknownDimensions(
modelRows.filter(
(row) =>
row.provider !== "unknown" &&
(catalog.models.get(row.model) ?? statProvider(row.model, "", "unknown")) === row.provider,
),
),
)
geoStats.deleteUnknownDimensions(
geoRows.filter(
(row) =>
row.model !== "all" &&
row.provider !== "unknown" &&
(catalog.models.get(row.model ?? "all") ?? statProvider(row.model ?? "all", "", "unknown")) ===
row.provider,
),
),
],
{ concurrency: "unbounded", discard: true },
)
yield* Effect.logInfo(
`stats sync writing aggregates ${JSON.stringify({ modelRows: modelRows.length, providerRows: providerRows.length, geoRows: geoRows.length, retentionRows: retentionRows.length })}`,
)
yield* Effect.all(
[
modelStats.upsert(modelRows),
providerStats.upsert(providerRows),
geoStats.upsert(geoRows),
retentionStats.replace(retentionRows, {
cohortDates: retentionQueries.flatMap((item) => item.cohortDates),
dataset: Resource.StatsSyncConfig.dataset,
tier: "Go",
}),
],
{
concurrency: "unbounded",
discard: true,
},
)
yield* Effect.all(
[
modelStats.deleteRetiredDimensions(modelRows, hiddenStatModels()),
providerStats.deleteRetiredDimensions(providerRows),
geoStats.deleteRetiredDimensions(geoRows, hiddenStatModels()),
],
{ concurrency: "unbounded", discard: true },
)
yield* Effect.all(
[
modelStats.deleteUnknownDimensions(
modelRows.filter(
(row) =>
row.provider !== "unknown" &&
(catalog?.models.get(row.model) ?? statProvider(row.model, "", "unknown")) === row.provider,
const result = {
grain: period.grain,
periodKey: period.key,
modelRows: modelRows.length,
providerRows: providerRows.length,
geoRows: geoRows.length,
}
yield* Effect.logInfo(`stats period synced ${JSON.stringify(result)}`)
return result
})
export const syncRetentionCohort = Effect.fn("StatSync.retentionCohort")(function* (
period: StatsSyncRetentionPeriod,
snapshot: CatalogSnapshot,
) {
const r2Sql = yield* R2Sql
const retentionStats = yield* RetentionStatRepo
const catalog = fromCatalogSnapshot(snapshot)
const item = buildPeriodRetentionQuery(
{
start: new Date(period.start),
end: new Date(period.end),
returnStart: new Date(period.returnStart),
returnEnd: new Date(period.returnEnd),
},
undefined,
catalog,
)
const rows = retentionRowsFromAggregates(
(yield* r2Sql
.query(item.query)
.pipe(
Effect.tapError((error) =>
Effect.logError(
`retention query failed ${JSON.stringify({ cohortDates: item.cohortDates, error: error.message })}`,
),
),
geoStats.deleteUnknownDimensions(
geoRows.filter(
(row) =>
row.model !== "all" &&
row.provider !== "unknown" &&
(catalog?.models.get(row.model ?? "all") ?? statProvider(row.model ?? "all", "", "unknown")) ===
row.provider,
),
),
],
{ concurrency: "unbounded", discard: true },
)
)).flatMap((row) => toRetentionAggregate(row, catalog)),
)
// Replace only after the query succeeds, so a failed query keeps the last cohort.
yield* retentionStats.replace(rows, {
cohortDates: item.cohortDates,
dataset: Resource.StatsSyncConfig.dataset,
tier: "Go",
})
yield* Effect.logInfo(
`stats sync complete ${JSON.stringify({
history: options?.history ?? false,
startedAt: startedAt.toISOString(),
periodStart: range.periodStart.toISOString(),
periodEnd: range.periodEnd.toISOString(),
rows: modelRows.length,
providerRows: providerRows.length,
geoRows: geoRows.length,
retentionRows: retentionRows.length,
retentionAvailable,
stage: Resource.App.stage,
})}`,
)
const result = { cohortDate: item.cohortDates[0]!, rows: rows.length }
yield* Effect.logInfo(`retention cohort synced ${JSON.stringify(result)}`)
return result
})
return {
ok: true,
rows: modelRows.length,
startedAt: startedAt.toISOString(),
periodStart: range.periodStart.toISOString(),
periodEnd: range.periodEnd.toISOString(),
}
},
)
// A rejected R2 SQL query fails the same way on every attempt, and each attempt
// repeats its spend. Return the failure as a result so a Workflow step stores it
// instead of retrying, and a replay reads it instead of running the query again.
export function deterministicFailureAsResult<A, E, R>(effect: Effect.Effect<A, E, R>) {
return effect.pipe(
Effect.catchIf(
(error): error is E & R2SqlQueryError => error instanceof R2SqlQueryError,
(error) => (isDeterministicR2SqlError(error) ? Effect.succeed({ failed: error.message }) : Effect.fail(error)),
),
)
}
function logRuntimeCheck() {
return Effect.logInfo(
-32
View File
@@ -1,32 +0,0 @@
FROM oven/bun:1.4.4-alpine AS base
WORKDIR /app
ENV NODE_ENV=production
ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=0
FROM base AS pruner
COPY . .
RUN bunx turbo@2.8.13 prune @opencode/stats-server --docker --no-update-notifier --no-color
FROM base AS installer
COPY --from=pruner /app/out/json/ ./
COPY --from=pruner /app/bun.lock ./bun.lock
# Bun 1.3.x needs the pruned workspace globs and lockfile metadata refreshed before the frozen production install.
RUN bun -e 'const packageJson = await Bun.file("package.json").json(); packageJson.workspaces.packages = Array.from(new Bun.Glob("packages/**/package.json").scanSync(".")).map((file) => file.slice(0, -"/package.json".length)).sort(); await Bun.write("package.json", JSON.stringify(packageJson, null, 2) + "\n")'
# Preserve locked transitive versions while refreshing the pruned workspace metadata.
RUN bun install --filter @opencode/stats-server --lockfile-only --ignore-scripts
RUN bun install --filter @opencode/stats-server --frozen-lockfile --production --ignore-scripts
FROM base AS runner
COPY --from=installer /app ./
COPY --from=pruner /app/out/full/ ./
WORKDIR /app/packages/stats/server
CMD ["bun", "src/stat-sync.ts"]
+3 -6
View File
@@ -5,22 +5,19 @@
"private": true,
"type": "module",
"license": "MIT",
"main": "./src/stat-sync.ts",
"exports": {
".": "./src/stat-sync.ts"
},
"scripts": {
"start": "bun src/stat-sync.ts",
"sync": "bun src/sync.ts",
"test": "bun test --only-failures",
"typecheck": "bun check --no-script",
"typecheck:tsc": "tsc --noEmit"
},
"dependencies": {
"@effect/platform-node": "catalog:",
"@opencode/stats-core": "workspace:*",
"effect": "catalog:",
"sst": "catalog:"
},
"devDependencies": {
"@cloudflare/workers-types": "catalog:",
"@tsconfig/node22": "catalog:",
"@types/bun": "catalog:",
"@types/node": "catalog:",
+78
View File
@@ -0,0 +1,78 @@
import { describe, expect, test } from "bun:test"
import cron from "./cron"
const controller = { cron: "5 * * * *", scheduledTime: Date.parse("2026-11-03T04:05:00.000Z") }
describe("stats sync cron", () => {
test("starts this hour's run when no recent run exists", async () => {
const workflow = fakeWorkflow({})
await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })
expect(workflow.created).toEqual([
{ id: "stats-hourly-2026-11-03T04", params: { history: false, startedAt: "2026-11-03T04:05:00.000Z" } },
])
})
test("fails the tick instead of starting a run when a lookup fails for another reason", async () => {
const workflow = fakeWorkflow({ get: { "stats-hourly-2026-11-03T02": new Error("internal error") } })
await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow(
"internal error",
)
expect(workflow.created).toEqual([])
})
test("fails the tick when an instance's status cannot be read", async () => {
const workflow = fakeWorkflow({ status: { "stats-hourly-2026-11-03T03": new Error("status unavailable") } })
await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow(
"status unavailable",
)
expect(workflow.created).toEqual([])
})
test("treats a concurrent create of the same slot as a skip", async () => {
const workflow = fakeWorkflow({ create: new Error("(instance.already_exists) Instance already exists") })
await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })
expect(workflow.created).toEqual([])
})
test("still fails on other create errors", async () => {
const workflow = fakeWorkflow({ create: new Error("internal error") })
await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow(
"internal error",
)
})
})
// Instances not listed in `status` do not exist, and `get` rejects for them the
// way the Workflow binding does.
function fakeWorkflow(input: { get?: Record<string, Error>; status?: Record<string, Error | string>; create?: Error }) {
const created: { id?: string; params?: unknown }[] = []
return {
created,
binding: {
get: async (id: string) => {
const failure = input.get?.[id]
if (failure) throw failure
const status = input.status?.[id]
if (status === undefined) throw new Error("instance.not_found")
return {
status: async () => {
if (status instanceof Error) throw status
return { status }
},
}
},
create: async (options: { id?: string; params?: unknown }) => {
if (input.create) throw input.create
created.push(options)
return { id: options.id }
},
},
}
}
+53
View File
@@ -0,0 +1,53 @@
import {
HISTORY_SCHEDULE,
instanceId,
isWorkflowError,
recentInstanceIds,
skipReason,
type StatsSyncParams,
} from "./schedule"
type Env = { StatsSyncWorkflow: Workflow<StatsSyncParams> }
export default {
async scheduled(controller: ScheduledController, env: Env) {
const scheduledAt = new Date(controller.scheduledTime)
const history = controller.cron === HISTORY_SCHEDULE
const id = instanceId(scheduledAt, history)
// Any lookup failure other than a missing instance fails the tick, so an
// unreadable status never starts a duplicate run.
const instances = await Promise.all(
recentInstanceIds(scheduledAt, history).map(async (item) => ({
id: item,
status: await env.StatsSyncWorkflow.get(item)
.then((instance) => instance.status())
.then((value) => value.status)
.catch((error) => {
if (isWorkflowError(error, "instance.not_found")) return undefined
throw error
}),
})),
)
const skip = skipReason(id, instances)
if (skip) {
console.log(`stats sync skipped ${JSON.stringify({ id, ...skip })}`)
return
}
// Cron delivery is at least once, so a second tick for this slot can get here too.
const created = await env.StatsSyncWorkflow.create({
id,
params: { history, startedAt: scheduledAt.toISOString() },
}).then(
() => true,
(error) => {
if (isWorkflowError(error, "instance.already_exists")) return false
throw error
},
)
if (!created) {
console.log(`stats sync skipped ${JSON.stringify({ id, reason: "already created" })}`)
return
}
console.log(`stats sync started ${JSON.stringify({ id, history })}`)
},
}
@@ -0,0 +1,97 @@
import { describe, expect, test } from "bun:test"
import { instanceId, isWorkflowError, recentInstanceIds, skipReason } from "./schedule"
describe("stats sync schedule", () => {
test("names one instance per hourly slot and per history day", () => {
const at = new Date("2026-11-03T03:20:00.000Z")
expect(instanceId(at, false)).toBe("stats-hourly-2026-11-03T03")
expect(instanceId(at, true)).toBe("stats-history-2026-11-03")
expect(instanceId(new Date("2026-11-03T03:05:59.999Z"), false)).toBe("stats-hourly-2026-11-03T03")
})
test("probes this hour and the previous five for an hourly run", () => {
expect(recentInstanceIds(new Date("2026-11-01T02:05:00.000Z"), false)).toEqual([
"stats-hourly-2026-11-01T02",
"stats-hourly-2026-11-01T01",
"stats-hourly-2026-11-01T00",
"stats-hourly-2026-10-31T23",
"stats-hourly-2026-10-31T22",
"stats-hourly-2026-10-31T21",
])
})
test("probes today and the previous two days for a history run", () => {
expect(recentInstanceIds(new Date("2026-11-01T03:20:00.000Z"), true)).toEqual([
"stats-history-2026-11-01",
"stats-history-2026-10-31",
"stats-history-2026-10-30",
])
})
test("lets hourly and history runs overlap", () => {
const at = new Date("2026-11-03T04:05:00.000Z")
const running = new Set(["stats-history-2026-11-03", "stats-hourly-2026-11-03T03"])
const decide = (history: boolean) =>
skipReason(
instanceId(at, history),
recentInstanceIds(at, history).map((id) => ({ id, status: running.has(id) ? "running" : undefined })),
)
expect(decide(false)).toEqual({
reason: "previous run active",
instance: "stats-hourly-2026-11-03T03",
status: "running",
})
running.delete("stats-hourly-2026-11-03T03")
expect(decide(false)).toBeUndefined()
expect(decide(true)).toEqual({ reason: "already created", instance: "stats-history-2026-11-03", status: "running" })
running.delete("stats-history-2026-11-03")
running.add("stats-hourly-2026-11-03T04")
expect(decide(true)).toBeUndefined()
})
test("starts when no recent instance is active", () => {
expect(
skipReason("stats-hourly-2026-11-03T04", [
{ id: "stats-hourly-2026-11-03T04", status: undefined },
{ id: "stats-hourly-2026-11-03T03", status: "complete" },
{ id: "stats-hourly-2026-11-03T02", status: "errored" },
{ id: "stats-hourly-2026-11-03T01", status: "terminated" },
]),
).toBeUndefined()
})
test.each(["queued", "running", "waiting", "paused", "waitingForPause"])(
"skips while a previous run is %s",
(status) => {
expect(
skipReason("stats-hourly-2026-11-03T04", [
{ id: "stats-hourly-2026-11-03T04", status: undefined },
{ id: "stats-hourly-2026-11-03T03", status },
]),
).toEqual({ reason: "previous run active", instance: "stats-hourly-2026-11-03T03", status })
},
)
test("skips a slot that already has an instance in any state", () => {
expect(skipReason("stats-history-2026-11-03", [{ id: "stats-history-2026-11-03", status: "complete" }])).toEqual({
reason: "already created",
instance: "stats-history-2026-11-03",
status: "complete",
})
})
test("recognizes Workflow binding errors by their code only", () => {
expect(isWorkflowError(new Error("instance.not_found"), "instance.not_found")).toBe(true)
expect(isWorkflowError(new Error("(instance.not_found) Instance does not exist"), "instance.not_found")).toBe(true)
expect(
isWorkflowError(new Error("(instance.already_exists) Instance already exists"), "instance.already_exists"),
).toBe(true)
expect(isWorkflowError(new Error("internal error"), "instance.not_found")).toBe(false)
expect(isWorkflowError(new Error("(instance.already_exists) Instance already exists"), "instance.not_found")).toBe(
false,
)
expect(isWorkflowError("instance.not_found", "instance.not_found")).toBe(false)
})
})
+43
View File
@@ -0,0 +1,43 @@
// Keep in sync with the StatsSyncCron schedules in infra/stats.ts.
export const HOURLY_SCHEDULE = "5 * * * *"
export const HISTORY_SCHEDULE = "20 3 * * *"
const HOUR_MS = 3_600_000
const DAY_MS = 24 * HOUR_MS
// How far back a run can still be active: hourly runs finish within hours, history
// runs within days.
const HOURLY_LOOKBACK = 6
const HISTORY_LOOKBACK = 3
const ACTIVE = new Set(["queued", "running", "waiting", "paused", "waitingForPause"])
export type StatsSyncParams = { history?: boolean; startedAt?: string }
// One ID per schedule slot, so a repeated trigger cannot start a second run.
export function instanceId(scheduledAt: Date, history: boolean) {
if (history) return `stats-history-${scheduledAt.toISOString().slice(0, 10)}`
return `stats-hourly-${scheduledAt.toISOString().slice(0, 13)}`
}
// The newest IDs of the same kind a run may still be using, starting with this
// slot's own. Hourly and history runs cover different periods and their writes
// are idempotent, so one kind never waits for the other.
export function recentInstanceIds(scheduledAt: Date, history: boolean) {
return Array.from({ length: history ? HISTORY_LOOKBACK : HOURLY_LOOKBACK }, (_, index) =>
instanceId(new Date(scheduledAt.getTime() - index * (history ? DAY_MS : HOUR_MS)), history),
)
}
// Runs of one kind never overlap: two would repeat R2 SQL spend for the same
// periods. `undefined` means the instance does not exist.
export function skipReason(id: string, instances: { id: string; status: string | undefined }[]) {
const own = instances.find((instance) => instance.id === id && instance.status !== undefined)
if (own) return { reason: "already created", instance: own.id, status: own.status }
const active = instances.find((instance) => instance.status !== undefined && ACTIVE.has(instance.status))
if (active) return { reason: "previous run active", instance: active.id, status: active.status }
}
// Workflow bindings reject with plain Errors whose message carries the code, as
// "instance.not_found" or "(instance.already_exists) Instance already exists".
export function isWorkflowError(error: unknown, code: "instance.not_found" | "instance.already_exists") {
return error instanceof Error && error.message.includes(code)
}
-74
View File
@@ -1,74 +0,0 @@
import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
import { ModelStatRepo } from "@opencode/stats-core/domain/model"
import { R2Sql } from "@opencode/stats-core/r2-sql"
import { layer as statsLayer } from "@opencode/stats-core/runtime"
import { syncStats } from "@opencode/stats-core/stat-sync"
import { syncRange } from "@opencode/stats-core/sync-range"
import { Cause, Duration, Effect, Layer, Schedule } from "effect"
const SYNC_INTERVAL = "1 hour"
const SYNC_INTERVAL_MS = 3_600_000
const runtimeLayer = Layer.mergeAll(statsLayer, R2Sql.layer)
const daemon = Effect.gen(function* () {
yield* Effect.logInfo("stats sync daemon started")
yield* initialDelay()
// Every pass recomputes the current ISO week. Once per UTC day, including after
// boot, it also starts a history pass for the rest of the display window. That
// pass takes over an hour, so it runs in the background instead of holding back
// the next hourly pass.
let lastHistoryDay = ""
let historyRunning = false
const history = syncStats({ history: true }).pipe(
Effect.catchCause((cause) =>
Effect.logWarning(`stats history sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
),
Effect.ensuring(
Effect.sync(() => {
historyRunning = false
}),
),
)
const pass = Effect.gen(function* () {
yield* syncStats()
const today = new Date().toISOString().slice(0, 10)
// A history pass still running past midnight defers the next day's until it ends.
if (lastHistoryDay === today || historyRunning) return
lastHistoryDay = today
historyRunning = true
yield* Effect.forkChild(history)
}).pipe(
Effect.tapCause((cause) =>
Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
),
// Without a retry, one transient R2 SQL failure leaves the site stale until the
// next hourly pass. Two retries cap the extra query spend per interval.
Effect.retry({ times: 2, schedule: Schedule.spaced("10 minutes") }),
Effect.ignoreCause,
)
yield* pass.pipe(Effect.repeat(Schedule.fixed(SYNC_INTERVAL)))
}).pipe(Effect.forkScoped)
// A restarted daemon must not immediately re-run the R2 SQL pass; resume the
// hourly cadence from the last completed sync instead. This caps the query spend
// of a crash loop at one pass per interval. Only hourly passes write the current
// week, so a history pass that rewrote older periods cannot push this back.
const initialDelay = Effect.fnUntraced(function* () {
const modelStats = yield* ModelStatRepo
const lastSynced = yield* modelStats
.lastSyncedAt(syncRange(new Date()).periodStart)
.pipe(Effect.catchCause(() => Effect.succeed(null)))
if (!lastSynced) return
const delayMs = Math.min(SYNC_INTERVAL_MS - (Date.now() - lastSynced.getTime()), SYNC_INTERVAL_MS)
if (delayMs <= 0) return
yield* Effect.logInfo(
`stats sync delaying first pass ${JSON.stringify({ lastSyncedAt: lastSynced.toISOString(), delayMs })}`,
)
yield* Effect.sleep(Duration.millis(delayMs))
})
NodeRuntime.runMain(Layer.launch(Layer.effectDiscard(daemon).pipe(Layer.provide(runtimeLayer))), {
disableErrorReporting: true,
})
+10
View File
@@ -0,0 +1,10 @@
import { R2Sql } from "@opencode/stats-core/r2-sql"
import { layer } from "@opencode/stats-core/runtime"
import { syncStats } from "@opencode/stats-core/stat-sync"
import { Effect, Layer } from "effect"
// Runs one sync pass in this process, for local use. Deployed stages run the
// same steps in the StatsSyncWorkflow.
await Effect.runPromise(
syncStats({ history: process.argv.includes("--history") }).pipe(Effect.provide(Layer.mergeAll(layer, R2Sql.layer))),
)
+78
View File
@@ -0,0 +1,78 @@
import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep, type WorkflowStepConfig } from "cloudflare:workers"
import { R2Sql } from "@opencode/stats-core/r2-sql"
import { layer } from "@opencode/stats-core/runtime"
import {
deterministicFailureAsResult,
planStatsSync,
syncRetentionCohort,
syncStatPeriod,
type SyncStatsError,
} from "@opencode/stats-core/stat-sync"
import { Effect, Layer } from "effect"
import type { StatsSyncParams } from "./schedule"
const runtimeLayer = Layer.mergeAll(layer, R2Sql.layer)
// Each step runs one period's R2 SQL queries (10-minute request timeout, with
// page retries) and its PlanetScale writes.
const STEP: WorkflowStepConfig = {
timeout: "30 minutes",
retries: { limit: 2, delay: "1 minute", backoff: "exponential" },
}
export class StatsSyncWorkflow extends WorkflowEntrypoint<unknown, StatsSyncParams> {
async run(event: WorkflowEvent<StatsSyncParams | undefined>, step: WorkflowStep) {
// A dashboard trigger can start an instance without params.
const plan = await step.do("plan", STEP, () =>
run(
planStatsSync(new Date(event.payload?.startedAt ?? event.timestamp), {
history: event.payload?.history ?? false,
}),
),
)
// Steps run one at a time to keep one period's rows in memory. A failed
// period does not stop the others; the run still ends as errored.
const failed: string[] = []
const fail = (name: string, error: unknown) => {
failed.push(name)
console.error(
`stats sync step failed ${JSON.stringify({
step: name,
error: error instanceof Error ? error.message : String(error),
cause: error instanceof Error && error.cause !== undefined ? String(error.cause) : undefined,
})}`,
)
}
const settle = (name: string, attempt: Promise<object>) =>
attempt.then(
(result) => {
if ("failed" in result) fail(name, result.failed)
},
(error) => fail(name, error),
)
for (const period of plan.periods) {
const name = `${period.grain} ${period.key}`
await settle(
name,
step.do(name, STEP, () => run(deterministicFailureAsResult(syncStatPeriod(period, plan.catalog)))),
)
}
for (const period of plan.retentionPeriods) {
const name = `retention ${period.start.slice(0, 10)}`
await settle(
name,
step.do(name, STEP, () => run(deterministicFailureAsResult(syncRetentionCohort(period, plan.catalog)))),
)
}
if (failed.length > 0) throw new Error(`stats sync failed for ${failed.join(", ")}`)
return { periods: plan.periods.length, retentionPeriods: plan.retentionPeriods.length }
}
}
// SST's generated entry wraps the default export; workerd rejects a module whose
// default export is undefined.
export default {}
function run<A>(effect: Effect.Effect<A, SyncStatsError, Layer.Success<typeof runtimeLayer>>) {
return Effect.runPromise(effect.pipe(Effect.provide(runtimeLayer)))
}
+3 -2
View File
@@ -7,7 +7,8 @@
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"types": ["bun", "node"]
"types": ["@cloudflare/workers-types", "node"]
},
"include": ["src", "../core/src/resource.d.ts"]
"include": ["src", "../core/src/resource.d.ts"],
"exclude": ["src/**/*.test.ts"]
}
+1 -1
View File
@@ -38,7 +38,7 @@ export default $config({
const stage = await import("./infra/stage.js")
await import("./infra/app.js")
const stats = stage.deployAws ? await import("./infra/stats.js") : undefined
const stats = stage.deployStats ? await import("./infra/stats.js") : undefined
await import("./infra/console.js")
await import("./infra/share.js")