fix(stats): keep hourly sync running during the daily history refresh (#58535)

This commit is contained in:
Adam
2026-10-23 16:14:28 -05:00
committed by GitHub
Unverified
parent 09e6ddf630
commit 2abdf9604b
6 changed files with 176 additions and 76 deletions
+2 -1
View File
@@ -13,7 +13,8 @@
"./domain/*": "./src/domain/*.ts",
"./r2-sql": "./src/r2-sql.ts",
"./runtime": "./src/runtime.ts",
"./stat-sync": "./src/stat-sync.ts"
"./stat-sync": "./src/stat-sync.ts",
"./sync-range": "./src/sync-range.ts"
},
"scripts": {
"db:generate": "drizzle-kit generate --config=drizzle.config.ts",
+9 -4
View File
@@ -1,4 +1,4 @@
import { and, asc, eq, inArray, max, or } from "drizzle-orm"
import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm"
import { Effect, Layer } from "effect"
import * as Context from "effect/Context"
import { DatabaseError, DrizzleClient } from "../database"
@@ -10,6 +10,7 @@ import {
DATA_SITE_TIERS,
inserted,
omitUniqueUsers,
periodKeyFor,
rankBy,
statPeriodKey,
statRowScope,
@@ -44,7 +45,7 @@ export type ModelStatMetric = {
export declare namespace ModelStatRepo {
export interface Service {
readonly listDaily: () => Effect.Effect<ModelStatMetric[], DatabaseError>
readonly lastSyncedAt: () => Effect.Effect<Date | null, DatabaseError>
readonly lastSyncedAt: (since: Date) => Effect.Effect<Date | null, DatabaseError>
readonly upsert: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
readonly deleteRetiredDimensions: (
rows: ModelStatRow[],
@@ -116,9 +117,13 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
})
})
const lastSyncedAt = Effect.fn("ModelStatRepo.lastSyncedAt")(function* () {
const lastSyncedAt = Effect.fn("ModelStatRepo.lastSyncedAt")(function* (since: Date) {
const result = yield* Effect.tryPromise({
try: () => db.select({ value: max(modelStat.updated_at) }).from(modelStat),
try: () =>
db
.select({ value: max(modelStat.updated_at) })
.from(modelStat)
.where(and(eq(modelStat.grain, "day"), gte(modelStat.period_key, periodKeyFor("day", since)))),
catch: (cause) => DatabaseError.make({ cause }),
})
return result[0]?.value ?? null
+12 -53
View File
@@ -16,32 +16,19 @@ import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./
import { statProvider } from "./domain/model-normalization"
import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider"
import { RetentionStatRepo, rowsFromAggregates as retentionRowsFromAggregates } from "./domain/retention"
import { startOfIsoWeek, startOfUtcDay } from "./domain/stat"
import { R2Sql, R2SqlQueryError } from "./r2-sql"
const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
const WEEK_MS = 7 * 86_400_000
const DISPLAY_WINDOW_MS = 56 * 86_400_000
// A retention result needs one complete activity week plus its complete return
// week. Keep another partial week of slack around the ISO-week boundary.
const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000
// Anchor incremental passes to the ISO week containing this lookback, so the pass
// after a week boundary still recomputes the previous week's final aggregates even
// if the boundary pass itself failed.
const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
import { syncRange } from "./sync-range"
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
export type SyncStatsError = R2SqlQueryError | DatabaseError
type SyncStatsServices = R2Sql | ModelStatRepo | ProviderStatRepo | GeoStatRepo | RetentionStatRepo
export const syncStats: (options?: {
full?: boolean
history?: boolean
}) => Effect.Effect<SyncStatsResult, SyncStatsError, SyncStatsServices> = Effect.fn("StatSync.sync")(
function* (options?: { full?: boolean }) {
function* (options?: { history?: boolean }) {
const startedAt = yield* DateTime.nowAsDate
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd)
const range = syncRange(startedAt, options?.history)
const r2Sql = yield* R2Sql
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
@@ -58,9 +45,9 @@ export const syncStats: (options?: {
yield* logRuntimeCheck()
const queries = buildStatsQueries(periodStart, periodEnd, undefined, catalog)
const queries = buildStatsQueries(range.periodStart, range.periodEnd, undefined, catalog)
yield* Effect.logInfo(
`stats sync started ${JSON.stringify({ full: options?.full ?? false, periodStart, periodEnd, queries: queries.length })}`,
`stats sync started ${JSON.stringify({ history: options?.history ?? false, periodStart: range.periodStart, periodEnd: range.periodEnd, queries: queries.length })}`,
)
const rows = yield* Effect.forEach(
queries,
@@ -94,16 +81,7 @@ export const syncStats: (options?: {
)
const retentionAvailable = yield* retentionStats.available()
const retentionQueries = retentionAvailable
? buildRetentionQueries(
options?.full
? periodStart
: new Date(
Math.max(startOfUtcDay(periodEnd).getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS),
),
startOfUtcDay(periodEnd),
undefined,
catalog,
)
? buildRetentionQueries(range.retentionStart, range.retentionEnd, undefined, catalog)
: []
yield* Effect.logInfo(`stats sync querying retention ${JSON.stringify({ queries: retentionQueries.length })}`)
const retentionRows = retentionRowsFromAggregates(
@@ -179,9 +157,10 @@ export const syncStats: (options?: {
yield* Effect.logInfo(
`stats sync complete ${JSON.stringify({
history: options?.history ?? false,
startedAt: startedAt.toISOString(),
periodStart: periodStart.toISOString(),
periodEnd: periodEnd.toISOString(),
periodStart: range.periodStart.toISOString(),
periodEnd: range.periodEnd.toISOString(),
rows: modelRows.length,
providerRows: providerRows.length,
geoRows: geoRows.length,
@@ -195,32 +174,12 @@ export const syncStats: (options?: {
ok: true,
rows: modelRows.length,
startedAt: startedAt.toISOString(),
periodStart: periodStart.toISOString(),
periodEnd: periodEnd.toISOString(),
periodStart: range.periodStart.toISOString(),
periodEnd: range.periodEnd.toISOString(),
}
},
)
// May 27 was partial, so keep stats anchored at the first complete day.
function fullPeriodStart(periodEnd: Date) {
return new Date(
Math.max(
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
STATS_DATA_START_MS,
),
)
}
// Events are append-only, so completed periods never change once synced; hourly
// passes only recompute the periods the current ISO week can still touch. The daily
// full pass refreshes the whole display window (normalization changes, retired
// dimension cleanup).
function incrementalPeriodStart(periodEnd: Date) {
return new Date(
Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS),
)
}
function logRuntimeCheck() {
return Effect.logInfo(
`r2 sql stats runtime check ${JSON.stringify({
@@ -0,0 +1,72 @@
import { describe, expect, test } from "bun:test"
import { buildRetentionQueries, buildStatsQueries } from "./domain/inference"
import { syncRange } from "./sync-range"
const source = { namespace: "inference", table: "generation", dataset: "zen" }
describe("sync range", () => {
test("keeps hourly passes on the current ISO week and leaves older periods to the history pass", () => {
const startedAt = new Date("2026-10-23T19:05:00.000Z")
expect(syncRange(startedAt)).toEqual({
periodStart: new Date("2026-10-19T00:00:00.000Z"),
periodEnd: new Date("2026-10-23T19:00:00.000Z"),
retentionStart: new Date("2026-10-07T00:00:00.000Z"),
retentionEnd: new Date("2026-10-23T00:00:00.000Z"),
})
expect(periodKeys(syncRange(startedAt))).toEqual([
"week 2026-W43",
"day 2026-10-19",
"day 2026-10-20",
"day 2026-10-21",
"day 2026-10-22",
"day 2026-10-23",
])
expect(cohorts(syncRange(startedAt))).toEqual(["2026-10-05"])
expect(syncRange(startedAt, true)).toEqual({
periodStart: new Date("2026-08-28T19:00:00.000Z"),
periodEnd: new Date("2026-10-19T00:00:00.000Z"),
retentionStart: new Date("2026-08-28T19:00:00.000Z"),
retentionEnd: new Date("2026-10-12T00:00:00.000Z"),
})
expect(cohorts(syncRange(startedAt, true))).toEqual([
"2026-08-24",
"2026-08-31",
"2026-09-07",
"2026-09-14",
"2026-09-21",
"2026-09-28",
])
})
test.each([
["mid-week", "2026-10-23T19:05:00.000Z"],
["late Sunday", "2026-10-25T23:59:00.000Z"],
["Monday while hourly passes still finish the previous week", "2026-10-26T01:05:00.000Z"],
["Monday after the previous week is final", "2026-10-26T03:05:00.000Z"],
])("splits the display window without gaps or overlap: %s", (_, time) => {
const incremental = syncRange(new Date(time))
const history = syncRange(new Date(time), true)
const display = { ...history, periodEnd: incremental.periodEnd, retentionEnd: incremental.retentionEnd }
expect([...periodKeys(history), ...periodKeys(incremental)].toSorted()).toEqual(periodKeys(display).toSorted())
expect([...cohorts(history), ...cohorts(incremental)].toSorted()).toEqual(cohorts(display).toSorted())
})
})
function periodKeys(range: ReturnType<typeof syncRange>) {
return [
...new Set(
buildStatsQueries(range.periodStart, range.periodEnd, source).flatMap((query) =>
[...query.matchAll(/'(day|week)' AS grain,\s+'([^']+)' AS period_key/g)].map(
(match) => `${match[1]} ${match[2]}`,
),
),
),
]
}
function cohorts(range: ReturnType<typeof syncRange>) {
return buildRetentionQueries(range.retentionStart, range.retentionEnd, source).flatMap((item) => item.cohortDates)
}
+53
View File
@@ -0,0 +1,53 @@
import { startOfIsoWeek, startOfUtcDay } from "./domain/stat"
const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
const WEEK_MS = 7 * 86_400_000
const DISPLAY_WINDOW_MS = 56 * 86_400_000
// A retention result needs one complete activity week plus its complete return
// week. Keep another partial week of slack around the ISO-week boundary.
const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000
// Anchor incremental passes to the ISO week containing this lookback, so the pass
// after a week boundary still recomputes the previous week's final aggregates even
// if the boundary pass itself failed.
const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
// Events are append-only, so completed periods never change once synced. Hourly
// incremental passes recompute only the periods the current ISO week can still
// touch and the latest retention cohorts. The daily history pass recomputes the rest
// of the display window (normalization changes, retired dimension cleanup). The two
// never share a period or cohort, so the slow history pass can run beside them.
export function syncRange(startedAt: Date, history = false) {
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
const incrementalStart = incrementalPeriodStart(periodEnd)
const retentionEnd = startOfUtcDay(periodEnd)
const retentionStart = new Date(
Math.max(retentionEnd.getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS),
)
if (!history) return { periodStart: incrementalStart, periodEnd, retentionStart, retentionEnd }
const windowStart = displayWindowStart(periodEnd)
return {
periodStart: windowStart,
periodEnd: incrementalStart,
retentionStart: windowStart,
// A cohort is computed only once its return week ends by this bound, so history
// stops one cohort before the first one the incremental pass recomputes.
retentionEnd: new Date(startOfIsoWeek(retentionStart).getTime() + WEEK_MS),
}
}
// May 27 was partial, so keep stats anchored at the first complete day.
function displayWindowStart(periodEnd: Date) {
return new Date(
Math.max(
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
STATS_DATA_START_MS,
),
)
}
function incrementalPeriodStart(periodEnd: Date) {
return new Date(
Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS),
)
}
+28 -18
View File
@@ -3,6 +3,7 @@ import { ModelStatRepo } from "@opencode/stats-core/domain/model"
import { R2Sql } from "@opencode/stats-core/r2-sql"
import { layer as statsLayer } from "@opencode/stats-core/runtime"
import { syncStats } from "@opencode/stats-core/stat-sync"
import { syncRange } from "@opencode/stats-core/sync-range"
import { Cause, Duration, Effect, Layer, Schedule } from "effect"
const SYNC_INTERVAL = "1 hour"
@@ -14,24 +15,30 @@ const daemon = Effect.gen(function* () {
yield* Effect.logInfo("stats sync daemon started")
yield* initialDelay()
// One full pass per UTC day (including the first pass after boot) refreshes the
// whole display window; every other pass only recomputes the current ISO week.
let lastFullDay = ""
// Every pass recomputes the current ISO week. Once per UTC day, including after
// boot, it also starts a history pass for the rest of the display window. That
// pass takes over an hour, so it runs in the background instead of holding back
// the next hourly pass.
let lastHistoryDay = ""
let historyRunning = false
const history = syncStats({ history: true }).pipe(
Effect.catchCause((cause) =>
Effect.logWarning(`stats history sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
),
Effect.ensuring(
Effect.sync(() => {
historyRunning = false
}),
),
)
const pass = Effect.gen(function* () {
yield* syncStats()
const today = new Date().toISOString().slice(0, 10)
if (lastFullDay !== today) {
const completed = yield* syncStats({ full: true }).pipe(
Effect.as(true),
Effect.catchCause((cause) =>
Effect.logWarning(`full stats sync failed; falling back to incremental sync ${Cause.pretty(cause)}`).pipe(
Effect.as(false),
),
),
)
lastFullDay = today
if (completed) return
}
yield* syncStats({ full: false })
// A history pass still running past midnight defers the next day's until it ends.
if (lastHistoryDay === today || historyRunning) return
lastHistoryDay = today
historyRunning = true
yield* Effect.forkChild(history)
}).pipe(
Effect.tapCause((cause) =>
Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
@@ -46,10 +53,13 @@ const daemon = Effect.gen(function* () {
// A restarted daemon must not immediately re-run the R2 SQL pass; resume the
// hourly cadence from the last completed sync instead. This caps the query spend
// of a crash loop at one pass per interval.
// of a crash loop at one pass per interval. Only hourly passes write the current
// week, so a history pass that rewrote older periods cannot push this back.
const initialDelay = Effect.fnUntraced(function* () {
const modelStats = yield* ModelStatRepo
const lastSynced = yield* modelStats.lastSyncedAt().pipe(Effect.catchCause(() => Effect.succeed(null)))
const lastSynced = yield* modelStats
.lastSyncedAt(syncRange(new Date()).periodStart)
.pipe(Effect.catchCause(() => Effect.succeed(null)))
if (!lastSynced) return
const delayMs = Math.min(SYNC_INTERVAL_MS - (Date.now() - lastSynced.getTime()), SYNC_INTERVAL_MS)
if (delayMs <= 0) return