|
|
|
@@ -1,9 +1,7 @@
|
|
|
|
|
import { Client } from "@planetscale/database"
|
|
|
|
|
import { Effect } from "effect"
|
|
|
|
|
import { Resource } from "sst/resource"
|
|
|
|
|
import { DatabaseError } from "../database"
|
|
|
|
|
import type { GeoStatMetric } from "./geo"
|
|
|
|
|
import { ModelStatRepo, type ModelStatMetric } from "./model"
|
|
|
|
|
import type { ModelStatMetric } from "./model"
|
|
|
|
|
import { statProvider } from "./model-normalization"
|
|
|
|
|
import { isMissingRetentionTable } from "./retention"
|
|
|
|
|
import { DATA_SITE_TIERS, normalizeTier } from "./stat"
|
|
|
|
@@ -77,7 +75,7 @@ export type StatsModelData = {
|
|
|
|
|
}
|
|
|
|
|
usage: ModelUsagePoint[]
|
|
|
|
|
tokenMix: ModelMixEntry[]
|
|
|
|
|
country: Record<UsageRange, CountryEntry[]>
|
|
|
|
|
country: CountryEntry[]
|
|
|
|
|
peers: ModelPeerEntry[]
|
|
|
|
|
}
|
|
|
|
|
export type StatsLabData = {
|
|
|
|
@@ -127,7 +125,7 @@ export type StatsHomeData = {
|
|
|
|
|
cacheRatio: Record<TokenProduct, CacheRatioEntry[]>
|
|
|
|
|
sessionCost: Record<TokenProduct, SessionCostEntry[]>
|
|
|
|
|
retention: RetentionEntry[]
|
|
|
|
|
country: Record<UsageRange, CountryEntry[]>
|
|
|
|
|
country: CountryEntry[]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export class StatsDataError extends Error {
|
|
|
|
@@ -146,6 +144,8 @@ const RETENTION_MODEL_LIMIT = 15
|
|
|
|
|
const RETENTION_MIN_ELIGIBLE_USER_WEEKS = 100
|
|
|
|
|
const RETENTION_COHORT_WEEKS = 7
|
|
|
|
|
const TOP_MODEL_SEGMENT_LIMIT = 9
|
|
|
|
|
const QUERY_CACHE_TTL_MS = 5 * 60 * 1000
|
|
|
|
|
const QUERY_CACHE_MAX_ENTRIES = 256
|
|
|
|
|
// Preserve the response shape while the public site presents Go and Free as one cohort.
|
|
|
|
|
const SITE_PRODUCT = "Go"
|
|
|
|
|
const SITE_TIER_PLACEHOLDERS = DATA_SITE_TIERS.map(() => "?").join(", ")
|
|
|
|
@@ -156,8 +156,10 @@ type StatMetricRow = Omit<ModelStatMetric, "updatedAt"> & {
|
|
|
|
|
periodStart: number
|
|
|
|
|
updatedAt: number
|
|
|
|
|
}
|
|
|
|
|
type GeoMetricRow = Omit<GeoStatMetric, "updatedAt"> & {
|
|
|
|
|
periodStart: number
|
|
|
|
|
type CountryTotalRow = {
|
|
|
|
|
country: string
|
|
|
|
|
continent: string
|
|
|
|
|
tokens: number
|
|
|
|
|
updatedAt: number
|
|
|
|
|
}
|
|
|
|
|
export type RetentionMetricRow = {
|
|
|
|
@@ -187,15 +189,16 @@ type ModelAggregate = {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type RawRow = Record<string, unknown>
|
|
|
|
|
type CachedQuery = { expiresAt: number; value: Promise<RawRow[]> }
|
|
|
|
|
|
|
|
|
|
const queryCache = new Map<string, CachedQuery>()
|
|
|
|
|
|
|
|
|
|
export function getStatsHomeData(): Effect.Effect<StatsHomeData, StatsDataError> {
|
|
|
|
|
return Effect.tryPromise({
|
|
|
|
|
try: async () => {
|
|
|
|
|
const [modelRows, geoRows, retentionRows] = await Promise.all([
|
|
|
|
|
listModelDaily(),
|
|
|
|
|
listGeoDaily(),
|
|
|
|
|
listRetentionWeekly(),
|
|
|
|
|
])
|
|
|
|
|
const [modelRows, retentionRows] = await Promise.all([listModelDaily(), listRetentionWeekly()])
|
|
|
|
|
const window = modelRowsWindow(modelRows, "2M")
|
|
|
|
|
const geoRows = window ? await listCountryTotals(window) : []
|
|
|
|
|
return buildStatsHomeData(modelRows, geoRows, retentionRows)
|
|
|
|
|
},
|
|
|
|
|
catch: (cause) => new StatsDataError(cause),
|
|
|
|
@@ -212,13 +215,12 @@ export function getStatsModelData(
|
|
|
|
|
const normalized = modelRows.flatMap(normalizeStatRow)
|
|
|
|
|
const resolvedModel = resolveModelName(model, normalized, provider)
|
|
|
|
|
if (!resolvedModel) return null
|
|
|
|
|
const window = modelRowsWindow(modelRows, "2M")
|
|
|
|
|
const resolvedProvider = resolveModelProvider(resolvedModel, normalized, provider)
|
|
|
|
|
return buildStatsModelData(
|
|
|
|
|
resolvedModel,
|
|
|
|
|
modelRows,
|
|
|
|
|
await listGeoDaily({
|
|
|
|
|
model: resolvedModel,
|
|
|
|
|
provider: resolveModelProvider(resolvedModel, normalized, provider),
|
|
|
|
|
}),
|
|
|
|
|
window ? await listCountryTotals(window, { model: resolvedModel, provider: resolvedProvider }) : [],
|
|
|
|
|
provider,
|
|
|
|
|
retentionRows,
|
|
|
|
|
)
|
|
|
|
@@ -239,8 +241,8 @@ async function listModelDaily(): Promise<ModelStatMetric[]> {
|
|
|
|
|
await queryRows(
|
|
|
|
|
`select period_key, updated_at, tier, provider, model, sessions, unique_users, input_tokens,
|
|
|
|
|
output_tokens, reasoning_tokens, cache_read_tokens, total_tokens, input_cost_microcents, output_cost_microcents,
|
|
|
|
|
total_cost_microcents from model_stat where grain = 'day' and client = 'all' and source = 'all'
|
|
|
|
|
and tier in (${SITE_TIER_PLACEHOLDERS}) order by period_key`,
|
|
|
|
|
total_cost_microcents from model_stat where grain = 'day' and dataset = 'zen' and client = 'all'
|
|
|
|
|
and source = 'all' and tier in (${SITE_TIER_PLACEHOLDERS}) order by period_key`,
|
|
|
|
|
DATA_SITE_TIERS,
|
|
|
|
|
)
|
|
|
|
|
).map((row) => ({
|
|
|
|
@@ -262,7 +264,10 @@ async function listModelDaily(): Promise<ModelStatMetric[]> {
|
|
|
|
|
}))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function listGeoDaily(opts?: { provider?: string; model?: string }): Promise<GeoStatMetric[]> {
|
|
|
|
|
async function listCountryTotals(
|
|
|
|
|
window: DateWindow,
|
|
|
|
|
opts?: { provider?: string; model?: string },
|
|
|
|
|
): Promise<CountryTotalRow[]> {
|
|
|
|
|
const scope =
|
|
|
|
|
opts?.model && opts.provider
|
|
|
|
|
? "and provider = ? and model = ?"
|
|
|
|
@@ -272,20 +277,16 @@ async function listGeoDaily(opts?: { provider?: string; model?: string }): Promi
|
|
|
|
|
const params = opts?.model && opts.provider ? [opts.provider, opts.model] : opts?.model ? [opts.model] : []
|
|
|
|
|
return (
|
|
|
|
|
await queryRows(
|
|
|
|
|
`select period_key, updated_at, tier, provider, model, country, continent, total_tokens from geo_stat
|
|
|
|
|
where grain = 'day' and client = 'all' and source = 'all'
|
|
|
|
|
and tier in (${SITE_TIER_PLACEHOLDERS}) ${scope} order by period_key`,
|
|
|
|
|
[...DATA_SITE_TIERS, ...params],
|
|
|
|
|
`select country, max(continent) as continent, sum(total_tokens) as total_tokens, max(updated_at) as updated_at
|
|
|
|
|
from geo_stat where grain = 'day' and dataset = 'zen' and client = 'all' and source = 'all'
|
|
|
|
|
and tier in (${SITE_TIER_PLACEHOLDERS}) ${scope} and period_key >= ? and period_key < ? group by country`,
|
|
|
|
|
[...DATA_SITE_TIERS, ...params, periodKey(window.start), periodKey(window.end)],
|
|
|
|
|
)
|
|
|
|
|
).map((row) => ({
|
|
|
|
|
periodKey: stringValue(row.period_key),
|
|
|
|
|
updatedAt: dateValue(row.updated_at),
|
|
|
|
|
tier: stringValue(row.tier),
|
|
|
|
|
provider: stringValue(row.provider),
|
|
|
|
|
model: stringValue(row.model),
|
|
|
|
|
country: stringValue(row.country),
|
|
|
|
|
updatedAt: dateValue(row.updated_at).getTime(),
|
|
|
|
|
country: stringValue(row.country) || "ZZ",
|
|
|
|
|
continent: stringValue(row.continent),
|
|
|
|
|
totalTokens: numberValue(row.total_tokens),
|
|
|
|
|
tokens: numberValue(row.total_tokens),
|
|
|
|
|
}))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -311,7 +312,21 @@ async function listRetentionWeekly(): Promise<RetentionMetricRow[]> {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function queryRows(query: string, params: string[] = []) {
|
|
|
|
|
return (await new Client({ url: databaseUrl() }).execute(query, params)).rows as RawRow[]
|
|
|
|
|
const key = JSON.stringify([query, params])
|
|
|
|
|
const now = Date.now()
|
|
|
|
|
const cached = queryCache.get(key)
|
|
|
|
|
if (cached && cached.expiresAt > now) return cached.value
|
|
|
|
|
if (cached) queryCache.delete(key)
|
|
|
|
|
|
|
|
|
|
const value = new Client({ url: databaseUrl() }).execute(query, params).then((result) => result.rows as RawRow[])
|
|
|
|
|
const entry = { expiresAt: now + QUERY_CACHE_TTL_MS, value }
|
|
|
|
|
queryCache.set(key, entry)
|
|
|
|
|
if (queryCache.size > QUERY_CACHE_MAX_ENTRIES) queryCache.delete(queryCache.keys().next().value!)
|
|
|
|
|
|
|
|
|
|
return value.catch((cause) => {
|
|
|
|
|
if (queryCache.get(key) === entry) queryCache.delete(key)
|
|
|
|
|
throw cause
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function databaseUrl() {
|
|
|
|
@@ -330,31 +345,27 @@ function dateValue(value: unknown) {
|
|
|
|
|
return value instanceof Date ? value : new Date(stringValue(value))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export const getStatsModelsComparisonData: (
|
|
|
|
|
export function getStatsModelsComparisonData(
|
|
|
|
|
models: readonly StatsModelComparisonInput[],
|
|
|
|
|
) => Effect.Effect<StatsModelComparisonData, DatabaseError, ModelStatRepo> = Effect.fn("StatsModelsComparison.getData")(
|
|
|
|
|
function* (models) {
|
|
|
|
|
const modelStats = yield* ModelStatRepo
|
|
|
|
|
const [rows, retentionRows] = yield* Effect.all([
|
|
|
|
|
modelStats.listDaily(),
|
|
|
|
|
Effect.tryPromise({
|
|
|
|
|
try: listRetentionWeekly,
|
|
|
|
|
catch: (cause) => DatabaseError.make({ cause }),
|
|
|
|
|
}),
|
|
|
|
|
])
|
|
|
|
|
const entries = models.map((model) =>
|
|
|
|
|
toComparisonEntry(buildStatsModelData(model.model, rows, [], model.provider, retentionRows)),
|
|
|
|
|
)
|
|
|
|
|
const latest = entries
|
|
|
|
|
.map((model) => model?.updatedAt)
|
|
|
|
|
.flatMap((value) => (value ? [dateTime(value)] : []))
|
|
|
|
|
.toSorted((a, b) => b - a)[0]
|
|
|
|
|
return {
|
|
|
|
|
updatedAt: latest === undefined ? null : new Date(latest).toISOString(),
|
|
|
|
|
models: entries,
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
): Effect.Effect<StatsModelComparisonData, StatsDataError> {
|
|
|
|
|
return Effect.tryPromise({
|
|
|
|
|
try: async () => {
|
|
|
|
|
const [rows, retentionRows] = await Promise.all([listModelDaily(), listRetentionWeekly()])
|
|
|
|
|
const entries = models.map((model) =>
|
|
|
|
|
toComparisonEntry(buildStatsModelData(model.model, rows, [], model.provider, retentionRows)),
|
|
|
|
|
)
|
|
|
|
|
const latest = entries
|
|
|
|
|
.map((model) => model?.updatedAt)
|
|
|
|
|
.flatMap((value) => (value ? [dateTime(value)] : []))
|
|
|
|
|
.toSorted((a, b) => b - a)[0]
|
|
|
|
|
return {
|
|
|
|
|
updatedAt: latest === undefined ? null : new Date(latest).toISOString(),
|
|
|
|
|
models: entries,
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
catch: (cause) => new StatsDataError(cause),
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export const getStatsModelComparisonData = (
|
|
|
|
|
firstProvider: string,
|
|
|
|
@@ -369,17 +380,15 @@ export const getStatsModelComparisonData = (
|
|
|
|
|
|
|
|
|
|
function buildStatsHomeData(
|
|
|
|
|
modelRows: ModelStatMetric[],
|
|
|
|
|
geoRows: GeoStatMetric[],
|
|
|
|
|
countryRows: CountryTotalRow[],
|
|
|
|
|
retentionRows: RetentionMetricRow[],
|
|
|
|
|
): StatsHomeData {
|
|
|
|
|
const normalized = modelRows.flatMap(normalizeStatRow)
|
|
|
|
|
const geo = geoRows.flatMap(normalizeGeoRow)
|
|
|
|
|
const periods = [...normalized, ...geo]
|
|
|
|
|
if (periods.length === 0) return emptyStatsHomeData()
|
|
|
|
|
if (normalized.length === 0) return emptyStatsHomeData()
|
|
|
|
|
|
|
|
|
|
const earliest = Math.min(...periods.map((row) => row.periodStart))
|
|
|
|
|
const latest = Math.max(...periods.map((row) => row.periodStart))
|
|
|
|
|
const latestUpdate = Math.max(...periods.map((row) => row.updatedAt))
|
|
|
|
|
const earliest = Math.min(...normalized.map((row) => row.periodStart))
|
|
|
|
|
const latest = Math.max(...normalized.map((row) => row.periodStart))
|
|
|
|
|
const latestUpdate = Math.max(...normalized.map((row) => row.updatedAt), ...countryRows.map((row) => row.updatedAt))
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
updatedAt: new Date(latestUpdate).toISOString(),
|
|
|
|
@@ -407,7 +416,7 @@ function buildStatsHomeData(
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
leaderboard: createUsageProductRecord((product) =>
|
|
|
|
|
createRangeRecord((range) => buildLeaderboard(normalized, product, getWindow("1W", earliest, latest))),
|
|
|
|
|
createRangeRecord((_range) => buildLeaderboard(normalized, product, getWindow("1W", earliest, latest))),
|
|
|
|
|
),
|
|
|
|
|
market: createRangeRecord((range) => buildMarketShare(normalized, "Go", range, getWindow(range, earliest, latest))),
|
|
|
|
|
tokenCost: createTokenProductRecord((product) =>
|
|
|
|
@@ -422,19 +431,18 @@ function buildStatsHomeData(
|
|
|
|
|
retention: buildRetentionEntries(retentionRows)
|
|
|
|
|
.filter((item) => item.rank !== null)
|
|
|
|
|
.slice(0, RETENTION_MODEL_LIMIT),
|
|
|
|
|
country: createRangeRecord((range) => buildCountryStats(geo, getWindow(range, earliest, latest))),
|
|
|
|
|
country: buildCountryStats(countryRows),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildStatsModelData(
|
|
|
|
|
modelParam: string,
|
|
|
|
|
modelRows: ModelStatMetric[],
|
|
|
|
|
geoRows: GeoStatMetric[],
|
|
|
|
|
countryRows: CountryTotalRow[],
|
|
|
|
|
providerParam?: string,
|
|
|
|
|
retentionRows: RetentionMetricRow[] = [],
|
|
|
|
|
): StatsModelData | null {
|
|
|
|
|
const normalized = modelRows.flatMap(normalizeStatRow)
|
|
|
|
|
const geo = geoRows.flatMap(normalizeGeoRow)
|
|
|
|
|
if (normalized.length === 0) return null
|
|
|
|
|
|
|
|
|
|
const model = resolveModelName(modelParam, normalized, providerParam)
|
|
|
|
@@ -497,7 +505,7 @@ function buildStatsModelData(
|
|
|
|
|
},
|
|
|
|
|
usage: buildModelUsage(currentRows, window, "2M"),
|
|
|
|
|
tokenMix: buildModelTokenMix(current),
|
|
|
|
|
country: createRangeRecord((range) => buildCountryStats(geo, getWindow(range, earliest, latest))),
|
|
|
|
|
country: buildCountryStats(countryRows),
|
|
|
|
|
peers: buildModelPeers(rankPeers, peerRank, peerTokens),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@@ -579,7 +587,7 @@ function emptyStatsHomeData(): StatsHomeData {
|
|
|
|
|
cacheRatio: createTokenProductRecord(() => []),
|
|
|
|
|
sessionCost: createTokenProductRecord(() => []),
|
|
|
|
|
retention: [],
|
|
|
|
|
country: createRangeRecord(() => []),
|
|
|
|
|
country: [],
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -707,8 +715,8 @@ function buildMarketShare(rows: StatMetricRow[], product: UsageProduct, range: U
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildCountryStats(rows: GeoMetricRow[], window: DateWindow) {
|
|
|
|
|
const countries = aggregateByCountry(rowsForProduct(rows, SITE_PRODUCT, window.start, window.end))
|
|
|
|
|
function buildCountryStats(rows: CountryTotalRow[]) {
|
|
|
|
|
const countries = rows
|
|
|
|
|
.filter((item) => item.tokens > 0 && item.country !== "AQ")
|
|
|
|
|
.toSorted((a, b) => b.tokens - a.tokens)
|
|
|
|
|
const totalTokens = countries.reduce((sum, item) => sum + item.tokens, 0)
|
|
|
|
@@ -862,19 +870,6 @@ function aggregateByProvider(rows: { provider: string; totalTokens: number }[])
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function aggregateByCountry(rows: GeoMetricRow[]) {
|
|
|
|
|
return Object.values(
|
|
|
|
|
rows.reduce<Record<string, { country: string; continent: string; tokens: number }>>((result, row) => {
|
|
|
|
|
result[row.country] = {
|
|
|
|
|
country: row.country,
|
|
|
|
|
continent: result[row.country]?.continent || row.continent,
|
|
|
|
|
tokens: (result[row.country]?.tokens ?? 0) + row.totalTokens,
|
|
|
|
|
}
|
|
|
|
|
return result
|
|
|
|
|
}, {}),
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function combineRowsForModel(model: string, rows: StatMetricRow[]): ModelAggregate {
|
|
|
|
|
const aggregate = rows.reduce<ModelAggregate | undefined>(
|
|
|
|
|
(result, row) => combineModelAggregate(result, row),
|
|
|
|
@@ -1000,28 +995,20 @@ function normalizeStatRow(row: ModelStatMetric): StatMetricRow[] {
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function normalizeGeoRow(row: GeoStatMetric): GeoMetricRow[] {
|
|
|
|
|
const periodStart = periodKeyTime(row.periodKey)
|
|
|
|
|
const updatedAt = dateTime(row.updatedAt)
|
|
|
|
|
if (!Number.isFinite(periodStart) || !Number.isFinite(updatedAt)) return []
|
|
|
|
|
return [
|
|
|
|
|
{
|
|
|
|
|
...row,
|
|
|
|
|
periodStart,
|
|
|
|
|
updatedAt,
|
|
|
|
|
tier: normalizeTier(row.tier),
|
|
|
|
|
provider: row.provider === "all" ? "all" : statProvider(row.model, undefined, row.provider) || "unknown",
|
|
|
|
|
model: row.model || "all",
|
|
|
|
|
country: row.country || "ZZ",
|
|
|
|
|
continent: row.continent || "",
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
function modelRowsWindow(rows: ModelStatMetric[], range: UsageRange): DateWindow | undefined {
|
|
|
|
|
const periods = rows.map((row) => periodKeyTime(row.periodKey)).filter(Number.isFinite)
|
|
|
|
|
if (periods.length === 0) return undefined
|
|
|
|
|
return getWindow(range, Math.min(...periods), Math.max(...periods))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function dateTime(value: Date | string) {
|
|
|
|
|
return (value instanceof Date ? value : new Date(value)).getTime()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function periodKey(value: number) {
|
|
|
|
|
return new Date(value).toISOString().slice(0, 10)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function periodKeyTime(value: string) {
|
|
|
|
|
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value)
|
|
|
|
|
if (!match) return Number.NaN
|
|
|
|
|