From 2abdf9604b3ebcf85ba4c505fca955038aefe1b9 Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:14:28 -0500 Subject: [PATCH] fix(stats): keep hourly sync running during the daily history refresh (#58535) --- packages/stats/core/package.json | 3 +- packages/stats/core/src/domain/model.ts | 13 ++-- packages/stats/core/src/stat-sync.ts | 65 ++++--------------- packages/stats/core/src/sync-range.test.ts | 72 ++++++++++++++++++++++ packages/stats/core/src/sync-range.ts | 53 ++++++++++++++++ packages/stats/server/src/stat-sync.ts | 46 ++++++++------ 6 files changed, 176 insertions(+), 76 deletions(-) create mode 100644 packages/stats/core/src/sync-range.test.ts create mode 100644 packages/stats/core/src/sync-range.ts diff --git a/packages/stats/core/package.json b/packages/stats/core/package.json index 223a4fa1cbd..d3852a75a05 100644 --- a/packages/stats/core/package.json +++ b/packages/stats/core/package.json @@ -13,7 +13,8 @@ "./domain/*": "./src/domain/*.ts", "./r2-sql": "./src/r2-sql.ts", "./runtime": "./src/runtime.ts", - "./stat-sync": "./src/stat-sync.ts" + "./stat-sync": "./src/stat-sync.ts", + "./sync-range": "./src/sync-range.ts" }, "scripts": { "db:generate": "drizzle-kit generate --config=drizzle.config.ts", diff --git a/packages/stats/core/src/domain/model.ts b/packages/stats/core/src/domain/model.ts index 3db009c56db..5b5f507d56e 100644 --- a/packages/stats/core/src/domain/model.ts +++ b/packages/stats/core/src/domain/model.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, inArray, max, or } from "drizzle-orm" +import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm" import { Effect, Layer } from "effect" import * as Context from "effect/Context" import { DatabaseError, DrizzleClient } from "../database" @@ -10,6 +10,7 @@ import { DATA_SITE_TIERS, inserted, omitUniqueUsers, + periodKeyFor, rankBy, statPeriodKey, statRowScope, @@ -44,7 +45,7 @@ export type ModelStatMetric = { export declare namespace ModelStatRepo { export interface Service { readonly listDaily: () => Effect.Effect - readonly lastSyncedAt: () => Effect.Effect + readonly lastSyncedAt: (since: Date) => Effect.Effect readonly upsert: (rows: ModelStatRow[]) => Effect.Effect readonly deleteRetiredDimensions: ( rows: ModelStatRow[], @@ -116,9 +117,13 @@ export class ModelStatRepo extends Context.Service db.select({ value: max(modelStat.updated_at) }).from(modelStat), + try: () => + db + .select({ value: max(modelStat.updated_at) }) + .from(modelStat) + .where(and(eq(modelStat.grain, "day"), gte(modelStat.period_key, periodKeyFor("day", since)))), catch: (cause) => DatabaseError.make({ cause }), }) return result[0]?.value ?? null diff --git a/packages/stats/core/src/stat-sync.ts b/packages/stats/core/src/stat-sync.ts index c5622f65191..138f20beaa8 100644 --- a/packages/stats/core/src/stat-sync.ts +++ b/packages/stats/core/src/stat-sync.ts @@ -16,32 +16,19 @@ import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./ import { statProvider } from "./domain/model-normalization" import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider" import { RetentionStatRepo, rowsFromAggregates as retentionRowsFromAggregates } from "./domain/retention" -import { startOfIsoWeek, startOfUtcDay } from "./domain/stat" import { R2Sql, R2SqlQueryError } from "./r2-sql" - -const DATALAKE_INGESTION_LAG_MS = 5 * 60_000 -const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime() -const WEEK_MS = 7 * 86_400_000 -const DISPLAY_WINDOW_MS = 56 * 86_400_000 -// A retention result needs one complete activity week plus its complete return -// week. Keep another partial week of slack around the ISO-week boundary. -const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000 -// Anchor incremental passes to the ISO week containing this lookback, so the pass -// after a week boundary still recomputes the previous week's final aggregates even -// if the boundary pass itself failed. -const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000 +import { syncRange } from "./sync-range" export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string } export type SyncStatsError = R2SqlQueryError | DatabaseError type SyncStatsServices = R2Sql | ModelStatRepo | ProviderStatRepo | GeoStatRepo | RetentionStatRepo export const syncStats: (options?: { - full?: boolean + history?: boolean }) => Effect.Effect = Effect.fn("StatSync.sync")( - function* (options?: { full?: boolean }) { + function* (options?: { history?: boolean }) { const startedAt = yield* DateTime.nowAsDate - const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000) - const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd) + const range = syncRange(startedAt, options?.history) const r2Sql = yield* R2Sql const modelStats = yield* ModelStatRepo const providerStats = yield* ProviderStatRepo @@ -58,9 +45,9 @@ export const syncStats: (options?: { yield* logRuntimeCheck() - const queries = buildStatsQueries(periodStart, periodEnd, undefined, catalog) + const queries = buildStatsQueries(range.periodStart, range.periodEnd, undefined, catalog) yield* Effect.logInfo( - `stats sync started ${JSON.stringify({ full: options?.full ?? false, periodStart, periodEnd, queries: queries.length })}`, + `stats sync started ${JSON.stringify({ history: options?.history ?? false, periodStart: range.periodStart, periodEnd: range.periodEnd, queries: queries.length })}`, ) const rows = yield* Effect.forEach( queries, @@ -94,16 +81,7 @@ export const syncStats: (options?: { ) const retentionAvailable = yield* retentionStats.available() const retentionQueries = retentionAvailable - ? buildRetentionQueries( - options?.full - ? periodStart - : new Date( - Math.max(startOfUtcDay(periodEnd).getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS), - ), - startOfUtcDay(periodEnd), - undefined, - catalog, - ) + ? buildRetentionQueries(range.retentionStart, range.retentionEnd, undefined, catalog) : [] yield* Effect.logInfo(`stats sync querying retention ${JSON.stringify({ queries: retentionQueries.length })}`) const retentionRows = retentionRowsFromAggregates( @@ -179,9 +157,10 @@ export const syncStats: (options?: { yield* Effect.logInfo( `stats sync complete ${JSON.stringify({ + history: options?.history ?? false, startedAt: startedAt.toISOString(), - periodStart: periodStart.toISOString(), - periodEnd: periodEnd.toISOString(), + periodStart: range.periodStart.toISOString(), + periodEnd: range.periodEnd.toISOString(), rows: modelRows.length, providerRows: providerRows.length, geoRows: geoRows.length, @@ -195,32 +174,12 @@ export const syncStats: (options?: { ok: true, rows: modelRows.length, startedAt: startedAt.toISOString(), - periodStart: periodStart.toISOString(), - periodEnd: periodEnd.toISOString(), + periodStart: range.periodStart.toISOString(), + periodEnd: range.periodEnd.toISOString(), } }, ) -// May 27 was partial, so keep stats anchored at the first complete day. -function fullPeriodStart(periodEnd: Date) { - return new Date( - Math.max( - Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS), - STATS_DATA_START_MS, - ), - ) -} - -// Events are append-only, so completed periods never change once synced; hourly -// passes only recompute the periods the current ISO week can still touch. The daily -// full pass refreshes the whole display window (normalization changes, retired -// dimension cleanup). -function incrementalPeriodStart(periodEnd: Date) { - return new Date( - Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS), - ) -} - function logRuntimeCheck() { return Effect.logInfo( `r2 sql stats runtime check ${JSON.stringify({ diff --git a/packages/stats/core/src/sync-range.test.ts b/packages/stats/core/src/sync-range.test.ts new file mode 100644 index 00000000000..17bd8af7c69 --- /dev/null +++ b/packages/stats/core/src/sync-range.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test" +import { buildRetentionQueries, buildStatsQueries } from "./domain/inference" +import { syncRange } from "./sync-range" + +const source = { namespace: "inference", table: "generation", dataset: "zen" } + +describe("sync range", () => { + test("keeps hourly passes on the current ISO week and leaves older periods to the history pass", () => { + const startedAt = new Date("2026-10-23T19:05:00.000Z") + + expect(syncRange(startedAt)).toEqual({ + periodStart: new Date("2026-10-19T00:00:00.000Z"), + periodEnd: new Date("2026-10-23T19:00:00.000Z"), + retentionStart: new Date("2026-10-07T00:00:00.000Z"), + retentionEnd: new Date("2026-10-23T00:00:00.000Z"), + }) + expect(periodKeys(syncRange(startedAt))).toEqual([ + "week 2026-W43", + "day 2026-10-19", + "day 2026-10-20", + "day 2026-10-21", + "day 2026-10-22", + "day 2026-10-23", + ]) + expect(cohorts(syncRange(startedAt))).toEqual(["2026-10-05"]) + + expect(syncRange(startedAt, true)).toEqual({ + periodStart: new Date("2026-08-28T19:00:00.000Z"), + periodEnd: new Date("2026-10-19T00:00:00.000Z"), + retentionStart: new Date("2026-08-28T19:00:00.000Z"), + retentionEnd: new Date("2026-10-12T00:00:00.000Z"), + }) + expect(cohorts(syncRange(startedAt, true))).toEqual([ + "2026-08-24", + "2026-08-31", + "2026-09-07", + "2026-09-14", + "2026-09-21", + "2026-09-28", + ]) + }) + + test.each([ + ["mid-week", "2026-10-23T19:05:00.000Z"], + ["late Sunday", "2026-10-25T23:59:00.000Z"], + ["Monday while hourly passes still finish the previous week", "2026-10-26T01:05:00.000Z"], + ["Monday after the previous week is final", "2026-10-26T03:05:00.000Z"], + ])("splits the display window without gaps or overlap: %s", (_, time) => { + const incremental = syncRange(new Date(time)) + const history = syncRange(new Date(time), true) + const display = { ...history, periodEnd: incremental.periodEnd, retentionEnd: incremental.retentionEnd } + + expect([...periodKeys(history), ...periodKeys(incremental)].toSorted()).toEqual(periodKeys(display).toSorted()) + expect([...cohorts(history), ...cohorts(incremental)].toSorted()).toEqual(cohorts(display).toSorted()) + }) +}) + +function periodKeys(range: ReturnType) { + return [ + ...new Set( + buildStatsQueries(range.periodStart, range.periodEnd, source).flatMap((query) => + [...query.matchAll(/'(day|week)' AS grain,\s+'([^']+)' AS period_key/g)].map( + (match) => `${match[1]} ${match[2]}`, + ), + ), + ), + ] +} + +function cohorts(range: ReturnType) { + return buildRetentionQueries(range.retentionStart, range.retentionEnd, source).flatMap((item) => item.cohortDates) +} diff --git a/packages/stats/core/src/sync-range.ts b/packages/stats/core/src/sync-range.ts new file mode 100644 index 00000000000..cfe6f737eca --- /dev/null +++ b/packages/stats/core/src/sync-range.ts @@ -0,0 +1,53 @@ +import { startOfIsoWeek, startOfUtcDay } from "./domain/stat" + +const DATALAKE_INGESTION_LAG_MS = 5 * 60_000 +const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime() +const WEEK_MS = 7 * 86_400_000 +const DISPLAY_WINDOW_MS = 56 * 86_400_000 +// A retention result needs one complete activity week plus its complete return +// week. Keep another partial week of slack around the ISO-week boundary. +const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000 +// Anchor incremental passes to the ISO week containing this lookback, so the pass +// after a week boundary still recomputes the previous week's final aggregates even +// if the boundary pass itself failed. +const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000 + +// Events are append-only, so completed periods never change once synced. Hourly +// incremental passes recompute only the periods the current ISO week can still +// touch and the latest retention cohorts. The daily history pass recomputes the rest +// of the display window (normalization changes, retired dimension cleanup). The two +// never share a period or cohort, so the slow history pass can run beside them. +export function syncRange(startedAt: Date, history = false) { + const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000) + const incrementalStart = incrementalPeriodStart(periodEnd) + const retentionEnd = startOfUtcDay(periodEnd) + const retentionStart = new Date( + Math.max(retentionEnd.getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS), + ) + if (!history) return { periodStart: incrementalStart, periodEnd, retentionStart, retentionEnd } + const windowStart = displayWindowStart(periodEnd) + return { + periodStart: windowStart, + periodEnd: incrementalStart, + retentionStart: windowStart, + // A cohort is computed only once its return week ends by this bound, so history + // stops one cohort before the first one the incremental pass recomputes. + retentionEnd: new Date(startOfIsoWeek(retentionStart).getTime() + WEEK_MS), + } +} + +// May 27 was partial, so keep stats anchored at the first complete day. +function displayWindowStart(periodEnd: Date) { + return new Date( + Math.max( + Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS), + STATS_DATA_START_MS, + ), + ) +} + +function incrementalPeriodStart(periodEnd: Date) { + return new Date( + Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS), + ) +} diff --git a/packages/stats/server/src/stat-sync.ts b/packages/stats/server/src/stat-sync.ts index e02f2725c94..6cf47e0c27d 100644 --- a/packages/stats/server/src/stat-sync.ts +++ b/packages/stats/server/src/stat-sync.ts @@ -3,6 +3,7 @@ import { ModelStatRepo } from "@opencode/stats-core/domain/model" import { R2Sql } from "@opencode/stats-core/r2-sql" import { layer as statsLayer } from "@opencode/stats-core/runtime" import { syncStats } from "@opencode/stats-core/stat-sync" +import { syncRange } from "@opencode/stats-core/sync-range" import { Cause, Duration, Effect, Layer, Schedule } from "effect" const SYNC_INTERVAL = "1 hour" @@ -14,24 +15,30 @@ const daemon = Effect.gen(function* () { yield* Effect.logInfo("stats sync daemon started") yield* initialDelay() - // One full pass per UTC day (including the first pass after boot) refreshes the - // whole display window; every other pass only recomputes the current ISO week. - let lastFullDay = "" + // Every pass recomputes the current ISO week. Once per UTC day, including after + // boot, it also starts a history pass for the rest of the display window. That + // pass takes over an hour, so it runs in the background instead of holding back + // the next hourly pass. + let lastHistoryDay = "" + let historyRunning = false + const history = syncStats({ history: true }).pipe( + Effect.catchCause((cause) => + Effect.logWarning(`stats history sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`), + ), + Effect.ensuring( + Effect.sync(() => { + historyRunning = false + }), + ), + ) const pass = Effect.gen(function* () { + yield* syncStats() const today = new Date().toISOString().slice(0, 10) - if (lastFullDay !== today) { - const completed = yield* syncStats({ full: true }).pipe( - Effect.as(true), - Effect.catchCause((cause) => - Effect.logWarning(`full stats sync failed; falling back to incremental sync ${Cause.pretty(cause)}`).pipe( - Effect.as(false), - ), - ), - ) - lastFullDay = today - if (completed) return - } - yield* syncStats({ full: false }) + // A history pass still running past midnight defers the next day's until it ends. + if (lastHistoryDay === today || historyRunning) return + lastHistoryDay = today + historyRunning = true + yield* Effect.forkChild(history) }).pipe( Effect.tapCause((cause) => Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`), @@ -46,10 +53,13 @@ const daemon = Effect.gen(function* () { // A restarted daemon must not immediately re-run the R2 SQL pass; resume the // hourly cadence from the last completed sync instead. This caps the query spend -// of a crash loop at one pass per interval. +// of a crash loop at one pass per interval. Only hourly passes write the current +// week, so a history pass that rewrote older periods cannot push this back. const initialDelay = Effect.fnUntraced(function* () { const modelStats = yield* ModelStatRepo - const lastSynced = yield* modelStats.lastSyncedAt().pipe(Effect.catchCause(() => Effect.succeed(null))) + const lastSynced = yield* modelStats + .lastSyncedAt(syncRange(new Date()).periodStart) + .pipe(Effect.catchCause(() => Effect.succeed(null))) if (!lastSynced) return const delayMs = Math.min(SYNC_INTERVAL_MS - (Date.now() - lastSynced.getTime()), SYNC_INTERVAL_MS) if (delayMs <= 0) return