mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(stats): keep hourly sync running during the daily history refresh (#58535)
This commit is contained in:
@@ -13,7 +13,8 @@
|
||||
"./domain/*": "./src/domain/*.ts",
|
||||
"./r2-sql": "./src/r2-sql.ts",
|
||||
"./runtime": "./src/runtime.ts",
|
||||
"./stat-sync": "./src/stat-sync.ts"
|
||||
"./stat-sync": "./src/stat-sync.ts",
|
||||
"./sync-range": "./src/sync-range.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"db:generate": "drizzle-kit generate --config=drizzle.config.ts",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, eq, inArray, max, or } from "drizzle-orm"
|
||||
import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm"
|
||||
import { Effect, Layer } from "effect"
|
||||
import * as Context from "effect/Context"
|
||||
import { DatabaseError, DrizzleClient } from "../database"
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
DATA_SITE_TIERS,
|
||||
inserted,
|
||||
omitUniqueUsers,
|
||||
periodKeyFor,
|
||||
rankBy,
|
||||
statPeriodKey,
|
||||
statRowScope,
|
||||
@@ -44,7 +45,7 @@ export type ModelStatMetric = {
|
||||
export declare namespace ModelStatRepo {
|
||||
export interface Service {
|
||||
readonly listDaily: () => Effect.Effect<ModelStatMetric[], DatabaseError>
|
||||
readonly lastSyncedAt: () => Effect.Effect<Date | null, DatabaseError>
|
||||
readonly lastSyncedAt: (since: Date) => Effect.Effect<Date | null, DatabaseError>
|
||||
readonly upsert: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
|
||||
readonly deleteRetiredDimensions: (
|
||||
rows: ModelStatRow[],
|
||||
@@ -116,9 +117,13 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
||||
})
|
||||
})
|
||||
|
||||
const lastSyncedAt = Effect.fn("ModelStatRepo.lastSyncedAt")(function* () {
|
||||
const lastSyncedAt = Effect.fn("ModelStatRepo.lastSyncedAt")(function* (since: Date) {
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () => db.select({ value: max(modelStat.updated_at) }).from(modelStat),
|
||||
try: () =>
|
||||
db
|
||||
.select({ value: max(modelStat.updated_at) })
|
||||
.from(modelStat)
|
||||
.where(and(eq(modelStat.grain, "day"), gte(modelStat.period_key, periodKeyFor("day", since)))),
|
||||
catch: (cause) => DatabaseError.make({ cause }),
|
||||
})
|
||||
return result[0]?.value ?? null
|
||||
|
||||
@@ -16,32 +16,19 @@ import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./
|
||||
import { statProvider } from "./domain/model-normalization"
|
||||
import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider"
|
||||
import { RetentionStatRepo, rowsFromAggregates as retentionRowsFromAggregates } from "./domain/retention"
|
||||
import { startOfIsoWeek, startOfUtcDay } from "./domain/stat"
|
||||
import { R2Sql, R2SqlQueryError } from "./r2-sql"
|
||||
|
||||
const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
|
||||
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
|
||||
const WEEK_MS = 7 * 86_400_000
|
||||
const DISPLAY_WINDOW_MS = 56 * 86_400_000
|
||||
// A retention result needs one complete activity week plus its complete return
|
||||
// week. Keep another partial week of slack around the ISO-week boundary.
|
||||
const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000
|
||||
// Anchor incremental passes to the ISO week containing this lookback, so the pass
|
||||
// after a week boundary still recomputes the previous week's final aggregates even
|
||||
// if the boundary pass itself failed.
|
||||
const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
|
||||
import { syncRange } from "./sync-range"
|
||||
|
||||
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
|
||||
export type SyncStatsError = R2SqlQueryError | DatabaseError
|
||||
type SyncStatsServices = R2Sql | ModelStatRepo | ProviderStatRepo | GeoStatRepo | RetentionStatRepo
|
||||
|
||||
export const syncStats: (options?: {
|
||||
full?: boolean
|
||||
history?: boolean
|
||||
}) => Effect.Effect<SyncStatsResult, SyncStatsError, SyncStatsServices> = Effect.fn("StatSync.sync")(
|
||||
function* (options?: { full?: boolean }) {
|
||||
function* (options?: { history?: boolean }) {
|
||||
const startedAt = yield* DateTime.nowAsDate
|
||||
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
|
||||
const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd)
|
||||
const range = syncRange(startedAt, options?.history)
|
||||
const r2Sql = yield* R2Sql
|
||||
const modelStats = yield* ModelStatRepo
|
||||
const providerStats = yield* ProviderStatRepo
|
||||
@@ -58,9 +45,9 @@ export const syncStats: (options?: {
|
||||
|
||||
yield* logRuntimeCheck()
|
||||
|
||||
const queries = buildStatsQueries(periodStart, periodEnd, undefined, catalog)
|
||||
const queries = buildStatsQueries(range.periodStart, range.periodEnd, undefined, catalog)
|
||||
yield* Effect.logInfo(
|
||||
`stats sync started ${JSON.stringify({ full: options?.full ?? false, periodStart, periodEnd, queries: queries.length })}`,
|
||||
`stats sync started ${JSON.stringify({ history: options?.history ?? false, periodStart: range.periodStart, periodEnd: range.periodEnd, queries: queries.length })}`,
|
||||
)
|
||||
const rows = yield* Effect.forEach(
|
||||
queries,
|
||||
@@ -94,16 +81,7 @@ export const syncStats: (options?: {
|
||||
)
|
||||
const retentionAvailable = yield* retentionStats.available()
|
||||
const retentionQueries = retentionAvailable
|
||||
? buildRetentionQueries(
|
||||
options?.full
|
||||
? periodStart
|
||||
: new Date(
|
||||
Math.max(startOfUtcDay(periodEnd).getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS),
|
||||
),
|
||||
startOfUtcDay(periodEnd),
|
||||
undefined,
|
||||
catalog,
|
||||
)
|
||||
? buildRetentionQueries(range.retentionStart, range.retentionEnd, undefined, catalog)
|
||||
: []
|
||||
yield* Effect.logInfo(`stats sync querying retention ${JSON.stringify({ queries: retentionQueries.length })}`)
|
||||
const retentionRows = retentionRowsFromAggregates(
|
||||
@@ -179,9 +157,10 @@ export const syncStats: (options?: {
|
||||
|
||||
yield* Effect.logInfo(
|
||||
`stats sync complete ${JSON.stringify({
|
||||
history: options?.history ?? false,
|
||||
startedAt: startedAt.toISOString(),
|
||||
periodStart: periodStart.toISOString(),
|
||||
periodEnd: periodEnd.toISOString(),
|
||||
periodStart: range.periodStart.toISOString(),
|
||||
periodEnd: range.periodEnd.toISOString(),
|
||||
rows: modelRows.length,
|
||||
providerRows: providerRows.length,
|
||||
geoRows: geoRows.length,
|
||||
@@ -195,32 +174,12 @@ export const syncStats: (options?: {
|
||||
ok: true,
|
||||
rows: modelRows.length,
|
||||
startedAt: startedAt.toISOString(),
|
||||
periodStart: periodStart.toISOString(),
|
||||
periodEnd: periodEnd.toISOString(),
|
||||
periodStart: range.periodStart.toISOString(),
|
||||
periodEnd: range.periodEnd.toISOString(),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
// May 27 was partial, so keep stats anchored at the first complete day.
|
||||
function fullPeriodStart(periodEnd: Date) {
|
||||
return new Date(
|
||||
Math.max(
|
||||
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
|
||||
STATS_DATA_START_MS,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// Events are append-only, so completed periods never change once synced; hourly
|
||||
// passes only recompute the periods the current ISO week can still touch. The daily
|
||||
// full pass refreshes the whole display window (normalization changes, retired
|
||||
// dimension cleanup).
|
||||
function incrementalPeriodStart(periodEnd: Date) {
|
||||
return new Date(
|
||||
Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS),
|
||||
)
|
||||
}
|
||||
|
||||
function logRuntimeCheck() {
|
||||
return Effect.logInfo(
|
||||
`r2 sql stats runtime check ${JSON.stringify({
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { buildRetentionQueries, buildStatsQueries } from "./domain/inference"
|
||||
import { syncRange } from "./sync-range"
|
||||
|
||||
const source = { namespace: "inference", table: "generation", dataset: "zen" }
|
||||
|
||||
describe("sync range", () => {
|
||||
test("keeps hourly passes on the current ISO week and leaves older periods to the history pass", () => {
|
||||
const startedAt = new Date("2026-10-23T19:05:00.000Z")
|
||||
|
||||
expect(syncRange(startedAt)).toEqual({
|
||||
periodStart: new Date("2026-10-19T00:00:00.000Z"),
|
||||
periodEnd: new Date("2026-10-23T19:00:00.000Z"),
|
||||
retentionStart: new Date("2026-10-07T00:00:00.000Z"),
|
||||
retentionEnd: new Date("2026-10-23T00:00:00.000Z"),
|
||||
})
|
||||
expect(periodKeys(syncRange(startedAt))).toEqual([
|
||||
"week 2026-W43",
|
||||
"day 2026-10-19",
|
||||
"day 2026-10-20",
|
||||
"day 2026-10-21",
|
||||
"day 2026-10-22",
|
||||
"day 2026-10-23",
|
||||
])
|
||||
expect(cohorts(syncRange(startedAt))).toEqual(["2026-10-05"])
|
||||
|
||||
expect(syncRange(startedAt, true)).toEqual({
|
||||
periodStart: new Date("2026-08-28T19:00:00.000Z"),
|
||||
periodEnd: new Date("2026-10-19T00:00:00.000Z"),
|
||||
retentionStart: new Date("2026-08-28T19:00:00.000Z"),
|
||||
retentionEnd: new Date("2026-10-12T00:00:00.000Z"),
|
||||
})
|
||||
expect(cohorts(syncRange(startedAt, true))).toEqual([
|
||||
"2026-08-24",
|
||||
"2026-08-31",
|
||||
"2026-09-07",
|
||||
"2026-09-14",
|
||||
"2026-09-21",
|
||||
"2026-09-28",
|
||||
])
|
||||
})
|
||||
|
||||
test.each([
|
||||
["mid-week", "2026-10-23T19:05:00.000Z"],
|
||||
["late Sunday", "2026-10-25T23:59:00.000Z"],
|
||||
["Monday while hourly passes still finish the previous week", "2026-10-26T01:05:00.000Z"],
|
||||
["Monday after the previous week is final", "2026-10-26T03:05:00.000Z"],
|
||||
])("splits the display window without gaps or overlap: %s", (_, time) => {
|
||||
const incremental = syncRange(new Date(time))
|
||||
const history = syncRange(new Date(time), true)
|
||||
const display = { ...history, periodEnd: incremental.periodEnd, retentionEnd: incremental.retentionEnd }
|
||||
|
||||
expect([...periodKeys(history), ...periodKeys(incremental)].toSorted()).toEqual(periodKeys(display).toSorted())
|
||||
expect([...cohorts(history), ...cohorts(incremental)].toSorted()).toEqual(cohorts(display).toSorted())
|
||||
})
|
||||
})
|
||||
|
||||
function periodKeys(range: ReturnType<typeof syncRange>) {
|
||||
return [
|
||||
...new Set(
|
||||
buildStatsQueries(range.periodStart, range.periodEnd, source).flatMap((query) =>
|
||||
[...query.matchAll(/'(day|week)' AS grain,\s+'([^']+)' AS period_key/g)].map(
|
||||
(match) => `${match[1]} ${match[2]}`,
|
||||
),
|
||||
),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
function cohorts(range: ReturnType<typeof syncRange>) {
|
||||
return buildRetentionQueries(range.retentionStart, range.retentionEnd, source).flatMap((item) => item.cohortDates)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { startOfIsoWeek, startOfUtcDay } from "./domain/stat"
|
||||
|
||||
const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
|
||||
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
|
||||
const WEEK_MS = 7 * 86_400_000
|
||||
const DISPLAY_WINDOW_MS = 56 * 86_400_000
|
||||
// A retention result needs one complete activity week plus its complete return
|
||||
// week. Keep another partial week of slack around the ISO-week boundary.
|
||||
const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000
|
||||
// Anchor incremental passes to the ISO week containing this lookback, so the pass
|
||||
// after a week boundary still recomputes the previous week's final aggregates even
|
||||
// if the boundary pass itself failed.
|
||||
const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
|
||||
|
||||
// Events are append-only, so completed periods never change once synced. Hourly
|
||||
// incremental passes recompute only the periods the current ISO week can still
|
||||
// touch and the latest retention cohorts. The daily history pass recomputes the rest
|
||||
// of the display window (normalization changes, retired dimension cleanup). The two
|
||||
// never share a period or cohort, so the slow history pass can run beside them.
|
||||
export function syncRange(startedAt: Date, history = false) {
|
||||
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
|
||||
const incrementalStart = incrementalPeriodStart(periodEnd)
|
||||
const retentionEnd = startOfUtcDay(periodEnd)
|
||||
const retentionStart = new Date(
|
||||
Math.max(retentionEnd.getTime() - RETENTION_INCREMENTAL_LOOKBACK_MS, STATS_DATA_START_MS),
|
||||
)
|
||||
if (!history) return { periodStart: incrementalStart, periodEnd, retentionStart, retentionEnd }
|
||||
const windowStart = displayWindowStart(periodEnd)
|
||||
return {
|
||||
periodStart: windowStart,
|
||||
periodEnd: incrementalStart,
|
||||
retentionStart: windowStart,
|
||||
// A cohort is computed only once its return week ends by this bound, so history
|
||||
// stops one cohort before the first one the incremental pass recomputes.
|
||||
retentionEnd: new Date(startOfIsoWeek(retentionStart).getTime() + WEEK_MS),
|
||||
}
|
||||
}
|
||||
|
||||
// May 27 was partial, so keep stats anchored at the first complete day.
|
||||
function displayWindowStart(periodEnd: Date) {
|
||||
return new Date(
|
||||
Math.max(
|
||||
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
|
||||
STATS_DATA_START_MS,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
function incrementalPeriodStart(periodEnd: Date) {
|
||||
return new Date(
|
||||
Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS),
|
||||
)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { ModelStatRepo } from "@opencode/stats-core/domain/model"
|
||||
import { R2Sql } from "@opencode/stats-core/r2-sql"
|
||||
import { layer as statsLayer } from "@opencode/stats-core/runtime"
|
||||
import { syncStats } from "@opencode/stats-core/stat-sync"
|
||||
import { syncRange } from "@opencode/stats-core/sync-range"
|
||||
import { Cause, Duration, Effect, Layer, Schedule } from "effect"
|
||||
|
||||
const SYNC_INTERVAL = "1 hour"
|
||||
@@ -14,24 +15,30 @@ const daemon = Effect.gen(function* () {
|
||||
yield* Effect.logInfo("stats sync daemon started")
|
||||
yield* initialDelay()
|
||||
|
||||
// One full pass per UTC day (including the first pass after boot) refreshes the
|
||||
// whole display window; every other pass only recomputes the current ISO week.
|
||||
let lastFullDay = ""
|
||||
// Every pass recomputes the current ISO week. Once per UTC day, including after
|
||||
// boot, it also starts a history pass for the rest of the display window. That
|
||||
// pass takes over an hour, so it runs in the background instead of holding back
|
||||
// the next hourly pass.
|
||||
let lastHistoryDay = ""
|
||||
let historyRunning = false
|
||||
const history = syncStats({ history: true }).pipe(
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning(`stats history sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
|
||||
),
|
||||
Effect.ensuring(
|
||||
Effect.sync(() => {
|
||||
historyRunning = false
|
||||
}),
|
||||
),
|
||||
)
|
||||
const pass = Effect.gen(function* () {
|
||||
yield* syncStats()
|
||||
const today = new Date().toISOString().slice(0, 10)
|
||||
if (lastFullDay !== today) {
|
||||
const completed = yield* syncStats({ full: true }).pipe(
|
||||
Effect.as(true),
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning(`full stats sync failed; falling back to incremental sync ${Cause.pretty(cause)}`).pipe(
|
||||
Effect.as(false),
|
||||
),
|
||||
),
|
||||
)
|
||||
lastFullDay = today
|
||||
if (completed) return
|
||||
}
|
||||
yield* syncStats({ full: false })
|
||||
// A history pass still running past midnight defers the next day's until it ends.
|
||||
if (lastHistoryDay === today || historyRunning) return
|
||||
lastHistoryDay = today
|
||||
historyRunning = true
|
||||
yield* Effect.forkChild(history)
|
||||
}).pipe(
|
||||
Effect.tapCause((cause) =>
|
||||
Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
|
||||
@@ -46,10 +53,13 @@ const daemon = Effect.gen(function* () {
|
||||
|
||||
// A restarted daemon must not immediately re-run the R2 SQL pass; resume the
|
||||
// hourly cadence from the last completed sync instead. This caps the query spend
|
||||
// of a crash loop at one pass per interval.
|
||||
// of a crash loop at one pass per interval. Only hourly passes write the current
|
||||
// week, so a history pass that rewrote older periods cannot push this back.
|
||||
const initialDelay = Effect.fnUntraced(function* () {
|
||||
const modelStats = yield* ModelStatRepo
|
||||
const lastSynced = yield* modelStats.lastSyncedAt().pipe(Effect.catchCause(() => Effect.succeed(null)))
|
||||
const lastSynced = yield* modelStats
|
||||
.lastSyncedAt(syncRange(new Date()).periodStart)
|
||||
.pipe(Effect.catchCause(() => Effect.succeed(null)))
|
||||
if (!lastSynced) return
|
||||
const delayMs = Math.min(SYNC_INTERVAL_MS - (Date.now() - lastSynced.getTime()), SYNC_INTERVAL_MS)
|
||||
if (delayMs <= 0) return
|
||||
|
||||
Reference in New Issue
Block a user