diff --git a/bun.lock b/bun.lock index 316cb9e3fc3..a8fb016d7c2 100644 --- a/bun.lock +++ b/bun.lock @@ -277,6 +277,67 @@ "@anthropic-ai/claude-agent-sdk", ], }, + "packages/data/app": { + "name": "@opencode/data-app", + "version": "2.2.7", + "dependencies": { + "@kobalte/core": "catalog:", + "@opencode/data-core": "workspace:*", + "@opencode/nav-menus": "workspace:*", + "@opencode/ui": "workspace:*", + "@resvg/resvg-wasm": "2.6.2", + "@solidjs/meta": "catalog:", + "@solidjs/router": "catalog:", + "@solidjs/start": "catalog:", + "d3-scale": "4.0.2", + "effect": "catalog:", + "i18n-iso-countries": "7.14.0", + "nitro": "3.0.260903-beta", + "satori": "0.63.2", + "solid-js": "catalog:", + "vite": "8.2.4", + }, + "devDependencies": { + "@types/bun": "catalog:", + "@types/d3-scale": "4.0.9", + "@typescript/native": "catalog:", + "typescript": "catalog:", + "wawoff2": "2.0.1", + }, + }, + "packages/data/core": { + "name": "@opencode/data-core", + "version": "2.2.7", + "dependencies": { + "@planetscale/database": "1.19.0", + "drizzle-orm": "catalog:", + "effect": "catalog:", + }, + "devDependencies": { + "@tsconfig/node22": "catalog:", + "@types/bun": "catalog:", + "@types/node": "catalog:", + "@typescript/native": "catalog:", + "drizzle-kit": "catalog:", + "typescript": "catalog:", + }, + }, + "packages/data/server": { + "name": "@opencode/data-server", + "version": "2.2.7", + "dependencies": { + "@opencode/data-core": "workspace:*", + "effect": "catalog:", + }, + "devDependencies": { + "@cloudflare/workers-types": "catalog:", + "@tsconfig/node22": "catalog:", + "@types/bun": "catalog:", + "@types/node": "catalog:", + "@typescript/native": "catalog:", + "typescript": "catalog:", + }, + }, "packages/desktop": { "name": "@opencode/desktop", "version": "2.2.7", @@ -665,67 +726,6 @@ "@typescript/native": "catalog:", }, }, - "packages/stats/app": { - "name": "@opencode/stats-app", - "version": "2.2.7", - "dependencies": { - "@kobalte/core": "catalog:", - "@opencode/nav-menus": "workspace:*", - "@opencode/stats-core": "workspace:*", - "@opencode/ui": "workspace:*", - "@resvg/resvg-wasm": "2.6.2", - "@solidjs/meta": "catalog:", - "@solidjs/router": "catalog:", - "@solidjs/start": "catalog:", - "d3-scale": "4.0.2", - "effect": "catalog:", - "i18n-iso-countries": "7.14.0", - "nitro": "3.0.260903-beta", - "satori": "0.63.2", - "solid-js": "catalog:", - "vite": "8.2.4", - }, - "devDependencies": { - "@types/bun": "catalog:", - "@types/d3-scale": "4.0.9", - "@typescript/native": "catalog:", - "typescript": "catalog:", - "wawoff2": "2.0.1", - }, - }, - "packages/stats/core": { - "name": "@opencode/stats-core", - "version": "2.2.7", - "dependencies": { - "@planetscale/database": "1.19.0", - "drizzle-orm": "catalog:", - "effect": "catalog:", - }, - "devDependencies": { - "@tsconfig/node22": "catalog:", - "@types/bun": "catalog:", - "@types/node": "catalog:", - "@typescript/native": "catalog:", - "drizzle-kit": "catalog:", - "typescript": "catalog:", - }, - }, - "packages/stats/server": { - "name": "@opencode/stats-server", - "version": "2.2.7", - "dependencies": { - "@opencode/stats-core": "workspace:*", - "effect": "catalog:", - }, - "devDependencies": { - "@cloudflare/workers-types": "catalog:", - "@tsconfig/node22": "catalog:", - "@types/bun": "catalog:", - "@types/node": "catalog:", - "@typescript/native": "catalog:", - "typescript": "catalog:", - }, - }, "packages/stories": { "name": "@opencode/stories", "devDependencies": { @@ -870,8 +870,8 @@ "devDependencies": { "@cloudflare/build-output-utils": "0.8.5", "@cloudflare/config": "0.23.0", - "@opencode/stats-app": "workspace:*", - "@opencode/stats-server": "workspace:*", + "@opencode/data-app": "workspace:*", + "@opencode/data-server": "workspace:*", "@types/bun": "catalog:", "@typescript/native": "catalog:", "cf": "1.0.0-beta.12", @@ -1798,6 +1798,12 @@ "@opencode/data": ["@opencode/data@workspace:services/data"], + "@opencode/data-app": ["@opencode/data-app@workspace:packages/data/app"], + + "@opencode/data-core": ["@opencode/data-core@workspace:packages/data/core"], + + "@opencode/data-server": ["@opencode/data-server@workspace:packages/data/server"], + "@opencode/desktop": ["@opencode/desktop@workspace:packages/desktop"], "@opencode/files": ["@opencode/files@workspace:services/files"], @@ -1858,12 +1864,6 @@ "@opencode/site": ["@opencode/site@workspace:services/site"], - "@opencode/stats-app": ["@opencode/stats-app@workspace:packages/stats/app"], - - "@opencode/stats-core": ["@opencode/stats-core@workspace:packages/stats/core"], - - "@opencode/stats-server": ["@opencode/stats-server@workspace:packages/stats/server"], - "@opencode/stories": ["@opencode/stories@workspace:packages/stories"], "@opencode/theme": ["@opencode/theme@workspace:packages/theme"], @@ -3084,7 +3084,7 @@ "ffi-rs": ["ffi-rs@1.3.7", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.7", "@yuuang/ffi-rs-darwin-arm64": "1.3.7", "@yuuang/ffi-rs-darwin-x64": "1.3.7", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.7", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.7", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.7", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.7", "@yuuang/ffi-rs-linux-x64-musl": "1.3.7", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.7", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.7", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.7" } }, "sha512-5MMZQS2t/6f/ec3sx0f51gBcKGZPGfolPAW04lvLOSYnr7gmOKs8YzH1Rh+34WGO+VtcgcTuxaUUoJLcR4Tirg=="], - "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + "fflate": ["fflate@0.7.3", "", {}, "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw=="], "filelist": ["filelist@1.0.6", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA=="], @@ -3800,7 +3800,7 @@ "path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="], - "path-to-regexp": ["path-to-regexp@6.3.0", "", {}, "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="], + "path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], @@ -4646,10 +4646,6 @@ "@sentry/server-utils/@sentry/core": ["@sentry/core@10.73.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0" } }, "sha512-FLO1UgH19RyasVpofu612WCOgb2nEH0dZy+R72d7p65XU9i0wxlMKm3+sgfwKmiSJp1Qhilaaxs4Jg6BbiM5HA=="], - "@shuding/opentype.js/fflate": ["fflate@0.7.3", "", {}, "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw=="], - - "@solidjs/start/path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], - "@tailwindcss/node/lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" }, "bundled": true }, "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg=="], @@ -4668,6 +4664,8 @@ "@textjs/app/vite": ["vite@6.4.5", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-4enwjFfi4IwtEh0w+jPzpLVOFQiPHmmzPxcHXQOyEur47o6u0jE573DmGpjA+7irXoMLYRZD5ZaFUEneS3abPg=="], + "@types/three/fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + "app-builder-lib/js-yaml": ["js-yaml@4.3.2", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA=="], "app-builder-lib/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], @@ -4820,14 +4818,10 @@ "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], - "router/path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], - "safe-array-concat/isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], "safe-push-apply/isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], - "satori/fflate": ["fflate@0.7.3", "", {}, "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw=="], - "serialize-error/type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], "slice-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], @@ -4884,6 +4878,8 @@ "wrangler/miniflare": ["miniflare@4.20260708.1", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.34.5", "undici": "7.28.0", "workerd": "1.20260708.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-c94O9zRDISdqO18EHt6l0iF/fWgWt8p18PJvRsA/L/NJZ9Cfke3s/F5Blg1XXF7WDutVRzWVWy8Vy4LaT5ifsA=="], + "wrangler/path-to-regexp": ["path-to-regexp@6.3.0", "", {}, "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="], + "wrap-ansi/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "wrap-ansi/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -4958,6 +4954,8 @@ "@opencode/legacy/wrangler/miniflare": ["miniflare@5.20261102.0-beta", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.5", "undici": "7.30.1", "workerd": "1.20261102.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" } }, "sha512-FGqoi2BF7Vq6v8aSraq9Tmhg8ORpUBC5RkNGhfa11J597cWlLJ5QP9vJwDMOzwJx+W4U+IuHWSidVceBK6VRZQ=="], + "@opencode/legacy/wrangler/path-to-regexp": ["path-to-regexp@6.3.0", "", {}, "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="], + "@opencode/legacy/wrangler/workerd": ["workerd@1.20261102.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20261102.1", "@cloudflare/workerd-darwin-arm64": "1.20261102.1", "@cloudflare/workerd-linux-64": "1.20261102.1", "@cloudflare/workerd-linux-arm64": "1.20261102.1", "@cloudflare/workerd-windows-64": "1.20261102.1" }, "hasInstallScript": true, "bin": { "workerd": "bin/workerd" } }, "sha512-5A+/vm0M5oc98pB3aR+DZ5cIOvRMhVWVS9tg+CrjUap7PIo8tqAnvNdvuiHBIE47/wkxtNM3jToAaU38dBsZTw=="], "@opencode/stories/@tailwindcss/vite/@tailwindcss/node": ["@tailwindcss/node@4.1.11", "", { "dependencies": { "@ampproject/remapping": "^2.3.0", "enhanced-resolve": "^5.18.1", "jiti": "^2.4.2", "lightningcss": "1.30.1", "magic-string": "^0.30.17", "source-map-js": "^1.2.1", "tailwindcss": "4.1.11" } }, "sha512-yzhzuGRmv5QyU9qLNg4GTlYI6STedBWRE7NjxP45CsFYYq9taI0zJXZBMqIC/c8fViNLhmrbpSFS57EoxUmD6Q=="], diff --git a/package.json b/package.json index afe66bf1f87..8e4a41f7bc3 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "packages/*", "packages/plugins/*", "services/*", - "packages/stats/*" + "packages/data/*" ], "catalog": { "@effect/opentelemetry": "4.1.0", diff --git a/packages/data/AGENTS.md b/packages/data/AGENTS.md new file mode 100644 index 00000000000..5f5378e3ece --- /dev/null +++ b/packages/data/AGENTS.md @@ -0,0 +1 @@ +To start the data site locally, run `bun run --cwd packages/data/app dev` with the database variables in README.md exported. diff --git a/packages/stats/README.md b/packages/data/README.md similarity index 61% rename from packages/stats/README.md rename to packages/data/README.md index 18d1dde8950..163917d3291 100644 --- a/packages/stats/README.md +++ b/packages/data/README.md @@ -1,48 +1,48 @@ -# OpenCode Stats +# OpenCode Data -Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; the Cloudflare Workers that sync stats live in `server`. +The data site (opencode.ai/data) is separate from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; the Cloudflare Workers that sync its data live in `server`. ## Packages - `app`: SolidStart frontend/site. -- `core`: Effect services, app config, Drizzle schema/migrations, and stats domains. -- `server`: the `StatsSyncWorkflow` Workflow, the cron handler that starts it, and a local sync script. +- `core`: Effect services, app config, Drizzle schema/migrations, and the data domains. +- `server`: the `DataSyncWorkflow` Workflow, the cron handler that starts it, and a local sync script. `services/data` builds and deploys them as the `opencode-data-`, `opencode-data-og-` and `opencode-data-sync-` Workers. ## Commands -- `bun run --cwd packages/stats/app dev` starts the SolidStart app (see Stats sync for its environment). -- `bun run --cwd packages/stats/app typecheck` typechecks the site. -- `bun run --cwd packages/stats/core typecheck` typechecks the Effect/database package. -- `bun run --cwd packages/stats/server typecheck` typechecks the sync Workers. +- `bun run --cwd packages/data/app dev` starts the SolidStart app (see Data sync for its environment). +- `bun run --cwd packages/data/app typecheck` typechecks the site. +- `bun run --cwd packages/data/core typecheck` typechecks the Effect/database package. +- `bun run --cwd packages/data/server typecheck` typechecks the sync Workers. -## Stats sync +## Data sync -The sync reads aggregates from R2 SQL and writes them to the stats PlanetScale database. It runs on the `dev` and `production` stages as a Cloudflare Workflow, `StatsSyncWorkflow`, in the `opencode-data-sync-` Worker that `services/data` deploys. +The sync reads aggregates from R2 SQL and writes them to the `opencode-stats` PlanetScale database. It runs on the `dev` and `production` stages as a Cloudflare Workflow, `DataSyncWorkflow`, in the `opencode-data-sync-` Worker that `services/data` deploys. Each run has a `plan` step, then one step per stat period (`week 2026-W45`, `day 2026-11-03`) and one per retention cohort (`retention 2026-10-05`). The plan snapshots the model catalog once, so every step attributes models the same way. Steps run one at a time and retry twice with exponential backoff. A rejected query (HTTP 400, 401 or 403) or a result over the 10,000-row limit fails its step after one attempt, since a retry repeats the same R2 SQL spend. A failed step does not stop the others, but the run ends as errored and logs each failed step with its error. Because queries run per period, expect an hourly run to take about 15 to 25 minutes and a history run about 2 to 3.5 hours. These are estimates until measured on a deployed stage. The same Worker's cron handler starts runs on two schedules: -- `5 * * * *`: an hourly run of the current ISO week and the most recent complete retention cohort, with instance ID `stats-hourly-YYYY-MM-DDTHH`. -- `20 3 * * *`: a daily history run of the rest of the 56-day display window, with instance ID `stats-history-YYYY-MM-DD`. +- `5 * * * *`: an hourly run of the current ISO week and the most recent complete retention cohort, with instance ID `data-hourly-YYYY-MM-DDTHH`. +- `20 3 * * *`: a daily history run of the rest of the 56-day display window, with instance ID `data-history-YYYY-MM-DD`. Times and IDs are in UTC. Before starting a run, the cron handler checks this slot's ID and the recent IDs of the same kind: the five previous hours for an hourly run, the two previous days for a history run. It skips the run if any of them is queued, running, waiting or paused, or if this slot's instance already exists, and logs the reason. Hourly and history runs can overlap, since they cover different periods and their writes are idempotent. To start a run by hand, trigger the `opencode-data-sync-` Workflow with an ID from the cron handler's convention so scheduled runs of the same kind see it and wait: ```sh -cf workflows instances create opencode-data-sync-dev --instance-id stats-history-2026-11-03 --params '{"history":true}' -cf workflows instances get stats-history-2026-11-03 --workflow-name opencode-data-sync-dev +cf workflows instances create opencode-data-sync-dev --instance-id data-history-2026-11-03 --params '{"history":true}' +cf workflows instances get data-history-2026-11-03 --workflow-name opencode-data-sync-dev ``` Omit `history` for an hourly run. If the ID already exists, that slot has run; wait for the next one. You can also trigger a run from the Workflow's page in the Cloudflare dashboard; without params it runs as an hourly pass. -To run one sync pass on your machine against a stage's resources, export `STAGE`, `DATA_DATABASE_HOST`, `DATA_DATABASE_USERNAME`, `DATA_DATABASE_PASSWORD`, `R2_SQL_AUTH_TOKEN` and `DATA_HIDDEN_MODELS` (AWS Secrets Manager `opencode//*`; `packages/stats/core/src/resource.ts` reads them), then run `bun run --cwd packages/stats/server sync`, adding `--history` for a history pass. `bun run --cwd packages/stats/app dev` reads the same database variables. +To run one sync pass on your machine against a stage's resources, export `STAGE`, `DATA_DATABASE_HOST`, `DATA_DATABASE_USERNAME`, `DATA_DATABASE_PASSWORD`, `R2_SQL_AUTH_TOKEN` and `DATA_HIDDEN_MODELS` (AWS Secrets Manager `opencode//*`; `packages/data/core/src/resource.ts` reads them), then run `bun run --cwd packages/data/server sync`, adding `--history` for a history pass. `bun run --cwd packages/data/app dev` reads the same database variables. ## Public page caching -Stats HTML, JSON, and Markdown use a one-minute browser cache and a five-minute shared cache. Expired responses must revalidate instead of serving stale snapshots. +Data site HTML, JSON, and Markdown use a one-minute browser cache and a five-minute shared cache. Expired responses must revalidate instead of serving stale snapshots. The Worker has no custom domain. `services/data/cloudflare.config.ts` routes `/data*` and `//data*` on the main host straight to it, so its responses skip the Cloudflare cache and reach browsers with these headers unchanged. Static assets are uploaded under `/data`, and the build moves Nitro's `_headers` to the assets root so they stay immutable. diff --git a/packages/stats/app/.gitignore b/packages/data/app/.gitignore similarity index 100% rename from packages/stats/app/.gitignore rename to packages/data/app/.gitignore diff --git a/packages/stats/app/app.config.ts b/packages/data/app/app.config.ts similarity index 100% rename from packages/stats/app/app.config.ts rename to packages/data/app/app.config.ts diff --git a/packages/stats/app/bunfig.toml b/packages/data/app/bunfig.toml similarity index 100% rename from packages/stats/app/bunfig.toml rename to packages/data/app/bunfig.toml diff --git a/packages/stats/app/package.json b/packages/data/app/package.json similarity index 94% rename from packages/stats/app/package.json rename to packages/data/app/package.json index ea99301b4be..4f820ecc953 100644 --- a/packages/stats/app/package.json +++ b/packages/data/app/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "name": "@opencode/stats-app", + "name": "@opencode/data-app", "version": "2.2.7", "private": true, "type": "module", @@ -23,7 +23,7 @@ "dependencies": { "@kobalte/core": "catalog:", "@opencode/nav-menus": "workspace:*", - "@opencode/stats-core": "workspace:*", + "@opencode/data-core": "workspace:*", "@opencode/ui": "workspace:*", "@resvg/resvg-wasm": "2.6.2", "@solidjs/meta": "catalog:", diff --git a/packages/stats/app/public/banner.jpg b/packages/data/app/public/banner.jpg similarity index 100% rename from packages/stats/app/public/banner.jpg rename to packages/data/app/public/banner.jpg diff --git a/packages/stats/app/public/banner.png b/packages/data/app/public/banner.png similarity index 100% rename from packages/stats/app/public/banner.png rename to packages/data/app/public/banner.png diff --git a/packages/stats/app/script/build-og-worker.ts b/packages/data/app/script/build-og-worker.ts similarity index 100% rename from packages/stats/app/script/build-og-worker.ts rename to packages/data/app/script/build-og-worker.ts diff --git a/packages/stats/app/src/app.css b/packages/data/app/src/app.css similarity index 100% rename from packages/stats/app/src/app.css rename to packages/data/app/src/app.css diff --git a/packages/stats/app/src/app.tsx b/packages/data/app/src/app.tsx similarity index 100% rename from packages/stats/app/src/app.tsx rename to packages/data/app/src/app.tsx diff --git a/packages/stats/app/src/asset/fonts/OpenTUIMono-OFL.txt b/packages/data/app/src/asset/fonts/OpenTUIMono-OFL.txt similarity index 100% rename from packages/stats/app/src/asset/fonts/OpenTUIMono-OFL.txt rename to packages/data/app/src/asset/fonts/OpenTUIMono-OFL.txt diff --git a/packages/stats/app/src/asset/fonts/opentui-mono-bold-latin.woff2 b/packages/data/app/src/asset/fonts/opentui-mono-bold-latin.woff2 similarity index 100% rename from packages/stats/app/src/asset/fonts/opentui-mono-bold-latin.woff2 rename to packages/data/app/src/asset/fonts/opentui-mono-bold-latin.woff2 diff --git a/packages/stats/app/src/asset/fonts/opentui-mono-bold-rest.woff2 b/packages/data/app/src/asset/fonts/opentui-mono-bold-rest.woff2 similarity index 100% rename from packages/stats/app/src/asset/fonts/opentui-mono-bold-rest.woff2 rename to packages/data/app/src/asset/fonts/opentui-mono-bold-rest.woff2 diff --git a/packages/stats/app/src/asset/fonts/opentui-mono-regular-latin.woff2 b/packages/data/app/src/asset/fonts/opentui-mono-regular-latin.woff2 similarity index 100% rename from packages/stats/app/src/asset/fonts/opentui-mono-regular-latin.woff2 rename to packages/data/app/src/asset/fonts/opentui-mono-regular-latin.woff2 diff --git a/packages/stats/app/src/asset/fonts/opentui-mono-regular-rest.woff2 b/packages/data/app/src/asset/fonts/opentui-mono-regular-rest.woff2 similarity index 100% rename from packages/stats/app/src/asset/fonts/opentui-mono-regular-rest.woff2 rename to packages/data/app/src/asset/fonts/opentui-mono-regular-rest.woff2 diff --git a/packages/stats/app/src/asset/logo-ornate-light.svg b/packages/data/app/src/asset/logo-ornate-light.svg similarity index 100% rename from packages/stats/app/src/asset/logo-ornate-light.svg rename to packages/data/app/src/asset/logo-ornate-light.svg diff --git a/packages/stats/app/src/cloudflare-workers.d.ts b/packages/data/app/src/cloudflare-workers.d.ts similarity index 100% rename from packages/stats/app/src/cloudflare-workers.d.ts rename to packages/data/app/src/cloudflare-workers.d.ts diff --git a/packages/stats/app/src/component/agent-meta.tsx b/packages/data/app/src/component/agent-meta.tsx similarity index 100% rename from packages/stats/app/src/component/agent-meta.tsx rename to packages/data/app/src/component/agent-meta.tsx diff --git a/packages/stats/app/src/component/chart-data-table.tsx b/packages/data/app/src/component/chart-data-table.tsx similarity index 100% rename from packages/stats/app/src/component/chart-data-table.tsx rename to packages/data/app/src/component/chart-data-table.tsx diff --git a/packages/stats/app/src/component/locale-links.tsx b/packages/data/app/src/component/locale-links.tsx similarity index 100% rename from packages/stats/app/src/component/locale-links.tsx rename to packages/data/app/src/component/locale-links.tsx diff --git a/packages/stats/app/src/component/model-compare-detail.tsx b/packages/data/app/src/component/model-compare-detail.tsx similarity index 99% rename from packages/stats/app/src/component/model-compare-detail.tsx rename to packages/data/app/src/component/model-compare-detail.tsx index ffb0468fd0f..f9df797948f 100644 --- a/packages/stats/app/src/component/model-compare-detail.tsx +++ b/packages/data/app/src/component/model-compare-detail.tsx @@ -6,7 +6,7 @@ import { type RetentionEntry, type StatsModelComparisonInput, type StatsModelComparisonEntry, -} from "@opencode/stats-core/domain/home" +} from "@opencode/data-core/domain/home" import { createAsync, query, useParams, useSearchParams } from "@solidjs/router" import { createEffect, createMemo, createSignal, For, onMount, Show } from "solid-js" import { getRequestEvent } from "solid-js/web" diff --git a/packages/stats/app/src/component/not-found-meta.tsx b/packages/data/app/src/component/not-found-meta.tsx similarity index 100% rename from packages/stats/app/src/component/not-found-meta.tsx rename to packages/data/app/src/component/not-found-meta.tsx diff --git a/packages/stats/app/src/component/pattern-ripple.tsx b/packages/data/app/src/component/pattern-ripple.tsx similarity index 100% rename from packages/stats/app/src/component/pattern-ripple.tsx rename to packages/data/app/src/component/pattern-ripple.tsx diff --git a/packages/stats/app/src/component/product-lockup.css b/packages/data/app/src/component/product-lockup.css similarity index 100% rename from packages/stats/app/src/component/product-lockup.css rename to packages/data/app/src/component/product-lockup.css diff --git a/packages/stats/app/src/component/product-lockup.tsx b/packages/data/app/src/component/product-lockup.tsx similarity index 100% rename from packages/stats/app/src/component/product-lockup.tsx rename to packages/data/app/src/component/product-lockup.tsx diff --git a/packages/stats/app/src/context/i18n.tsx b/packages/data/app/src/context/i18n.tsx similarity index 100% rename from packages/stats/app/src/context/i18n.tsx rename to packages/data/app/src/context/i18n.tsx diff --git a/packages/stats/app/src/context/language.tsx b/packages/data/app/src/context/language.tsx similarity index 100% rename from packages/stats/app/src/context/language.tsx rename to packages/data/app/src/context/language.tsx diff --git a/packages/stats/app/src/entry-client.tsx b/packages/data/app/src/entry-client.tsx similarity index 100% rename from packages/stats/app/src/entry-client.tsx rename to packages/data/app/src/entry-client.tsx diff --git a/packages/stats/app/src/entry-server.tsx b/packages/data/app/src/entry-server.tsx similarity index 100% rename from packages/stats/app/src/entry-server.tsx rename to packages/data/app/src/entry-server.tsx diff --git a/packages/stats/app/src/global.d.ts b/packages/data/app/src/global.d.ts similarity index 100% rename from packages/stats/app/src/global.d.ts rename to packages/data/app/src/global.d.ts diff --git a/packages/stats/app/src/i18n.ts b/packages/data/app/src/i18n.ts similarity index 100% rename from packages/stats/app/src/i18n.ts rename to packages/data/app/src/i18n.ts diff --git a/packages/stats/app/src/i18n/ar.ts b/packages/data/app/src/i18n/ar.ts similarity index 100% rename from packages/stats/app/src/i18n/ar.ts rename to packages/data/app/src/i18n/ar.ts diff --git a/packages/stats/app/src/i18n/br.ts b/packages/data/app/src/i18n/br.ts similarity index 100% rename from packages/stats/app/src/i18n/br.ts rename to packages/data/app/src/i18n/br.ts diff --git a/packages/stats/app/src/i18n/da.ts b/packages/data/app/src/i18n/da.ts similarity index 100% rename from packages/stats/app/src/i18n/da.ts rename to packages/data/app/src/i18n/da.ts diff --git a/packages/stats/app/src/i18n/de.ts b/packages/data/app/src/i18n/de.ts similarity index 100% rename from packages/stats/app/src/i18n/de.ts rename to packages/data/app/src/i18n/de.ts diff --git a/packages/stats/app/src/i18n/es.ts b/packages/data/app/src/i18n/es.ts similarity index 100% rename from packages/stats/app/src/i18n/es.ts rename to packages/data/app/src/i18n/es.ts diff --git a/packages/stats/app/src/i18n/fr.ts b/packages/data/app/src/i18n/fr.ts similarity index 100% rename from packages/stats/app/src/i18n/fr.ts rename to packages/data/app/src/i18n/fr.ts diff --git a/packages/stats/app/src/i18n/it.ts b/packages/data/app/src/i18n/it.ts similarity index 100% rename from packages/stats/app/src/i18n/it.ts rename to packages/data/app/src/i18n/it.ts diff --git a/packages/stats/app/src/i18n/ja.ts b/packages/data/app/src/i18n/ja.ts similarity index 100% rename from packages/stats/app/src/i18n/ja.ts rename to packages/data/app/src/i18n/ja.ts diff --git a/packages/stats/app/src/i18n/ko.ts b/packages/data/app/src/i18n/ko.ts similarity index 100% rename from packages/stats/app/src/i18n/ko.ts rename to packages/data/app/src/i18n/ko.ts diff --git a/packages/stats/app/src/i18n/no.ts b/packages/data/app/src/i18n/no.ts similarity index 100% rename from packages/stats/app/src/i18n/no.ts rename to packages/data/app/src/i18n/no.ts diff --git a/packages/stats/app/src/i18n/pl.ts b/packages/data/app/src/i18n/pl.ts similarity index 100% rename from packages/stats/app/src/i18n/pl.ts rename to packages/data/app/src/i18n/pl.ts diff --git a/packages/stats/app/src/i18n/ru.ts b/packages/data/app/src/i18n/ru.ts similarity index 100% rename from packages/stats/app/src/i18n/ru.ts rename to packages/data/app/src/i18n/ru.ts diff --git a/packages/stats/app/src/i18n/th.ts b/packages/data/app/src/i18n/th.ts similarity index 100% rename from packages/stats/app/src/i18n/th.ts rename to packages/data/app/src/i18n/th.ts diff --git a/packages/stats/app/src/i18n/tr.ts b/packages/data/app/src/i18n/tr.ts similarity index 100% rename from packages/stats/app/src/i18n/tr.ts rename to packages/data/app/src/i18n/tr.ts diff --git a/packages/stats/app/src/i18n/uk.ts b/packages/data/app/src/i18n/uk.ts similarity index 100% rename from packages/stats/app/src/i18n/uk.ts rename to packages/data/app/src/i18n/uk.ts diff --git a/packages/stats/app/src/i18n/zh.ts b/packages/data/app/src/i18n/zh.ts similarity index 100% rename from packages/stats/app/src/i18n/zh.ts rename to packages/data/app/src/i18n/zh.ts diff --git a/packages/stats/app/src/i18n/zht.ts b/packages/data/app/src/i18n/zht.ts similarity index 100% rename from packages/stats/app/src/i18n/zht.ts rename to packages/data/app/src/i18n/zht.ts diff --git a/packages/stats/app/src/lib/agent-formats.ts b/packages/data/app/src/lib/agent-formats.ts similarity index 99% rename from packages/stats/app/src/lib/agent-formats.ts rename to packages/data/app/src/lib/agent-formats.ts index 088b949bd13..e1c376e5545 100644 --- a/packages/stats/app/src/lib/agent-formats.ts +++ b/packages/data/app/src/lib/agent-formats.ts @@ -1,4 +1,4 @@ -import type { UsagePoint } from "@opencode/stats-core/domain/home" +import type { UsagePoint } from "@opencode/data-core/domain/home" import { translate } from "../i18n" import { catalogModelPath, diff --git a/packages/stats/app/src/lib/chart-tooltip.ts b/packages/data/app/src/lib/chart-tooltip.ts similarity index 100% rename from packages/stats/app/src/lib/chart-tooltip.ts rename to packages/data/app/src/lib/chart-tooltip.ts diff --git a/packages/stats/app/src/lib/comparison-pages.ts b/packages/data/app/src/lib/comparison-pages.ts similarity index 100% rename from packages/stats/app/src/lib/comparison-pages.ts rename to packages/data/app/src/lib/comparison-pages.ts diff --git a/packages/stats/app/src/lib/data-path.ts b/packages/data/app/src/lib/data-path.ts similarity index 100% rename from packages/stats/app/src/lib/data-path.ts rename to packages/data/app/src/lib/data-path.ts diff --git a/packages/stats/app/src/lib/format.ts b/packages/data/app/src/lib/format.ts similarity index 100% rename from packages/stats/app/src/lib/format.ts rename to packages/data/app/src/lib/format.ts diff --git a/packages/stats/app/src/lib/language.ts b/packages/data/app/src/lib/language.ts similarity index 100% rename from packages/stats/app/src/lib/language.ts rename to packages/data/app/src/lib/language.ts diff --git a/packages/stats/app/src/lib/locale.ts b/packages/data/app/src/lib/locale.ts similarity index 100% rename from packages/stats/app/src/lib/locale.ts rename to packages/data/app/src/lib/locale.ts diff --git a/packages/stats/app/src/lib/methodology.ts b/packages/data/app/src/lib/methodology.ts similarity index 100% rename from packages/stats/app/src/lib/methodology.ts rename to packages/data/app/src/lib/methodology.ts diff --git a/packages/stats/app/src/lib/model-name.ts b/packages/data/app/src/lib/model-name.ts similarity index 89% rename from packages/stats/app/src/lib/model-name.ts rename to packages/data/app/src/lib/model-name.ts index b9fe4c85802..8d009753376 100644 --- a/packages/stats/app/src/lib/model-name.ts +++ b/packages/data/app/src/lib/model-name.ts @@ -1,4 +1,4 @@ -import { statModel } from "@opencode/stats-core/domain/model-normalization" +import { statModel } from "@opencode/data-core/domain/model-normalization" const glmFlashModel = "glm-5.3-flash" diff --git a/packages/stats/app/src/lib/noindex.ts b/packages/data/app/src/lib/noindex.ts similarity index 95% rename from packages/stats/app/src/lib/noindex.ts rename to packages/data/app/src/lib/noindex.ts index 0d1639855f1..84c55eb5fef 100644 --- a/packages/stats/app/src/lib/noindex.ts +++ b/packages/data/app/src/lib/noindex.ts @@ -1,5 +1,5 @@ import type { createMiddleware } from "@solidjs/start/middleware" -import { Resource } from "@opencode/stats-core/resource" +import { Resource } from "@opencode/data-core/resource" // Only production may be indexed: dev.opencode.ai and the personal stages run unreleased builds, and // search engines must never list them in place of opencode.ai. Every response there asks them not diff --git a/packages/stats/app/src/lib/page-data.ts b/packages/data/app/src/lib/page-data.ts similarity index 97% rename from packages/stats/app/src/lib/page-data.ts rename to packages/data/app/src/lib/page-data.ts index 6986b9e6043..8a6da4505d9 100644 --- a/packages/stats/app/src/lib/page-data.ts +++ b/packages/data/app/src/lib/page-data.ts @@ -5,8 +5,8 @@ import { getStatsModelsComparisonData, type MarketDay, type TokenCostEntry, -} from "@opencode/stats-core/domain/home" -import { statModel } from "@opencode/stats-core/domain/model-normalization" +} from "@opencode/data-core/domain/home" +import { statModel } from "@opencode/data-core/domain/model-normalization" import { runStatsEffect } from "../stats-runtime" import { catalogSlug, diff --git a/packages/stats/app/src/lib/provider-icon.ts b/packages/data/app/src/lib/provider-icon.ts similarity index 100% rename from packages/stats/app/src/lib/provider-icon.ts rename to packages/data/app/src/lib/provider-icon.ts diff --git a/packages/stats/app/src/lib/resource.cloudflare.ts b/packages/data/app/src/lib/resource.cloudflare.ts similarity index 91% rename from packages/stats/app/src/lib/resource.cloudflare.ts rename to packages/data/app/src/lib/resource.cloudflare.ts index 0d8de1683a7..97f0efd93a0 100644 --- a/packages/stats/app/src/lib/resource.cloudflare.ts +++ b/packages/data/app/src/lib/resource.cloudflare.ts @@ -1,4 +1,4 @@ -// The Stats Worker's bindings for the OG image route (src/og/response.ts). package.json `imports` picks this file for +// The data Worker's bindings for the OG image route (src/og/response.ts). package.json `imports` picks this file for // builds; resource.node.ts stands in under `vite dev` and tests. import { env } from "cloudflare:workers" import type { OgCard } from "../og/card" diff --git a/packages/stats/app/src/lib/resource.node.ts b/packages/data/app/src/lib/resource.node.ts similarity index 100% rename from packages/stats/app/src/lib/resource.node.ts rename to packages/data/app/src/lib/resource.node.ts diff --git a/packages/stats/app/src/lib/summaries.ts b/packages/data/app/src/lib/summaries.ts similarity index 100% rename from packages/stats/app/src/lib/summaries.ts rename to packages/data/app/src/lib/summaries.ts diff --git a/packages/stats/app/src/middleware.ts b/packages/data/app/src/middleware.ts similarity index 100% rename from packages/stats/app/src/middleware.ts rename to packages/data/app/src/middleware.ts diff --git a/packages/stats/app/src/og/card-data.ts b/packages/data/app/src/og/card-data.ts similarity index 98% rename from packages/stats/app/src/og/card-data.ts rename to packages/data/app/src/og/card-data.ts index 27b8f58dabf..fe14d006571 100644 --- a/packages/stats/app/src/og/card-data.ts +++ b/packages/data/app/src/og/card-data.ts @@ -1,4 +1,4 @@ -import { statModel } from "@opencode/stats-core/domain/model-normalization" +import { statModel } from "@opencode/data-core/domain/model-normalization" import { formatTokens, fromFirstUsage, isoDay } from "../lib/format" import { baseUrl } from "../lib/language" import { publicModelName } from "../lib/model-name" diff --git a/packages/stats/app/src/og/card.ts b/packages/data/app/src/og/card.ts similarity index 100% rename from packages/stats/app/src/og/card.ts rename to packages/data/app/src/og/card.ts diff --git a/packages/stats/app/src/og/modules.d.ts b/packages/data/app/src/og/modules.d.ts similarity index 100% rename from packages/stats/app/src/og/modules.d.ts rename to packages/data/app/src/og/modules.d.ts diff --git a/packages/stats/app/src/og/render-png.ts b/packages/data/app/src/og/render-png.ts similarity index 100% rename from packages/stats/app/src/og/render-png.ts rename to packages/data/app/src/og/render-png.ts diff --git a/packages/stats/app/src/og/response.ts b/packages/data/app/src/og/response.ts similarity index 100% rename from packages/stats/app/src/og/response.ts rename to packages/data/app/src/og/response.ts diff --git a/packages/stats/app/src/og/worker.ts b/packages/data/app/src/og/worker.ts similarity index 100% rename from packages/stats/app/src/og/worker.ts rename to packages/data/app/src/og/worker.ts diff --git a/packages/stats/app/src/routes/[...404].tsx b/packages/data/app/src/routes/[...404].tsx similarity index 100% rename from packages/stats/app/src/routes/[...404].tsx rename to packages/data/app/src/routes/[...404].tsx diff --git a/packages/stats/app/src/routes/[lab]/[model].tsx b/packages/data/app/src/routes/[lab]/[model].tsx similarity index 99% rename from packages/stats/app/src/routes/[lab]/[model].tsx rename to packages/data/app/src/routes/[lab]/[model].tsx index c825a9f60ec..a009edec4c8 100644 --- a/packages/stats/app/src/routes/[lab]/[model].tsx +++ b/packages/data/app/src/routes/[lab]/[model].tsx @@ -2,8 +2,8 @@ import { Meta, Title } from "@solidjs/meta" import { ProviderIcon } from "@opencode/ui/provider-icon" import { scaleSqrt } from "d3-scale" import countryCodesSource from "i18n-iso-countries/codes.json?raw" -import { type CountryEntry, type ModelUsagePoint } from "@opencode/stats-core/domain/home" -import { statModel } from "@opencode/stats-core/domain/model-normalization" +import { type CountryEntry, type ModelUsagePoint } from "@opencode/data-core/domain/home" +import { statModel } from "@opencode/data-core/domain/model-normalization" import { createAsync, query, useParams } from "@solidjs/router" import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" diff --git a/packages/stats/app/src/routes/[lab]/index.tsx b/packages/data/app/src/routes/[lab]/index.tsx similarity index 99% rename from packages/stats/app/src/routes/[lab]/index.tsx rename to packages/data/app/src/routes/[lab]/index.tsx index 98f417ccd86..db5b8f8e30c 100644 --- a/packages/stats/app/src/routes/[lab]/index.tsx +++ b/packages/data/app/src/routes/[lab]/index.tsx @@ -5,7 +5,7 @@ import { type MarketDay, type ModelUsagePoint, type StatsLabData, -} from "@opencode/stats-core/domain/home" +} from "@opencode/data-core/domain/home" import { createAsync, query, useParams } from "@solidjs/router" import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" diff --git a/packages/stats/app/src/routes/api/health.ts b/packages/data/app/src/routes/api/health.ts similarity index 100% rename from packages/stats/app/src/routes/api/health.ts rename to packages/data/app/src/routes/api/health.ts diff --git a/packages/stats/app/src/routes/breadcrumb-select.tsx b/packages/data/app/src/routes/breadcrumb-select.tsx similarity index 100% rename from packages/stats/app/src/routes/breadcrumb-select.tsx rename to packages/data/app/src/routes/breadcrumb-select.tsx diff --git a/packages/stats/app/src/routes/compare-cards.tsx b/packages/data/app/src/routes/compare-cards.tsx similarity index 100% rename from packages/stats/app/src/routes/compare-cards.tsx rename to packages/data/app/src/routes/compare-cards.tsx diff --git a/packages/stats/app/src/routes/compare-radar.tsx b/packages/data/app/src/routes/compare-radar.tsx similarity index 100% rename from packages/stats/app/src/routes/compare-radar.tsx rename to packages/data/app/src/routes/compare-radar.tsx diff --git a/packages/stats/app/src/routes/compare-selector.tsx b/packages/data/app/src/routes/compare-selector.tsx similarity index 100% rename from packages/stats/app/src/routes/compare-selector.tsx rename to packages/data/app/src/routes/compare-selector.tsx diff --git a/packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx b/packages/data/app/src/routes/compare/[firstFamily]/[secondFamily].tsx similarity index 100% rename from packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx rename to packages/data/app/src/routes/compare/[firstFamily]/[secondFamily].tsx diff --git a/packages/stats/app/src/routes/compare/[firstLab]/[firstModel]/[secondLab]/[secondModel].tsx b/packages/data/app/src/routes/compare/[firstLab]/[firstModel]/[secondLab]/[secondModel].tsx similarity index 100% rename from packages/stats/app/src/routes/compare/[firstLab]/[firstModel]/[secondLab]/[secondModel].tsx rename to packages/data/app/src/routes/compare/[firstLab]/[firstModel]/[secondLab]/[secondModel].tsx diff --git a/packages/stats/app/src/routes/compare/index.tsx b/packages/data/app/src/routes/compare/index.tsx similarity index 100% rename from packages/stats/app/src/routes/compare/index.tsx rename to packages/data/app/src/routes/compare/index.tsx diff --git a/packages/stats/app/src/routes/index.css b/packages/data/app/src/routes/index.css similarity index 100% rename from packages/stats/app/src/routes/index.css rename to packages/data/app/src/routes/index.css diff --git a/packages/stats/app/src/routes/index.tsx b/packages/data/app/src/routes/index.tsx similarity index 99% rename from packages/stats/app/src/routes/index.tsx rename to packages/data/app/src/routes/index.tsx index bb737fedfee..1db1cfe416d 100644 --- a/packages/stats/app/src/routes/index.tsx +++ b/packages/data/app/src/routes/index.tsx @@ -9,7 +9,7 @@ import { type SessionCostEntry, type TokenCostEntry, type UsagePoint, -} from "@opencode/stats-core/domain/home" +} from "@opencode/data-core/domain/home" import { createAsync, query } from "@solidjs/router" import { createEffect, createMemo, createSignal, For, onCleanup, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" diff --git a/packages/stats/app/src/routes/model-catalog.test.ts b/packages/data/app/src/routes/model-catalog.test.ts similarity index 100% rename from packages/stats/app/src/routes/model-catalog.test.ts rename to packages/data/app/src/routes/model-catalog.test.ts diff --git a/packages/stats/app/src/routes/model-catalog.ts b/packages/data/app/src/routes/model-catalog.ts similarity index 99% rename from packages/stats/app/src/routes/model-catalog.ts rename to packages/data/app/src/routes/model-catalog.ts index cc368a58788..c5f9eb5a5ae 100644 --- a/packages/stats/app/src/routes/model-catalog.ts +++ b/packages/data/app/src/routes/model-catalog.ts @@ -1,4 +1,4 @@ -import { STEALTH_MODELS, statModel } from "@opencode/stats-core/domain/model-normalization" +import { STEALTH_MODELS, statModel } from "@opencode/data-core/domain/model-normalization" import { query } from "@solidjs/router" export const modelCatalogSourceUrl = "https://models.opencode.ai/catalog.json" diff --git a/packages/stats/app/src/routes/section-heading.tsx b/packages/data/app/src/routes/section-heading.tsx similarity index 100% rename from packages/stats/app/src/routes/section-heading.tsx rename to packages/data/app/src/routes/section-heading.tsx diff --git a/packages/stats/app/src/routes/sitemap.xml.ts b/packages/data/app/src/routes/sitemap.xml.ts similarity index 97% rename from packages/stats/app/src/routes/sitemap.xml.ts rename to packages/data/app/src/routes/sitemap.xml.ts index f0ca8eb2245..30caf8f3828 100644 --- a/packages/stats/app/src/routes/sitemap.xml.ts +++ b/packages/data/app/src/routes/sitemap.xml.ts @@ -1,5 +1,5 @@ -import { getStatsHomeData } from "@opencode/stats-core/domain/home" -import { runtime } from "@opencode/stats-core/runtime" +import { getStatsHomeData } from "@opencode/data-core/domain/home" +import { runtime } from "@opencode/data-core/runtime" import { canonicalFamilyComparisonPath, canonicalModelComparisonPath, diff --git a/packages/stats/app/src/routes/stats-cache.ts b/packages/data/app/src/routes/stats-cache.ts similarity index 100% rename from packages/stats/app/src/routes/stats-cache.ts rename to packages/data/app/src/routes/stats-cache.ts diff --git a/packages/stats/app/src/routes/stats-shell.tsx b/packages/data/app/src/routes/stats-shell.tsx similarity index 100% rename from packages/stats/app/src/routes/stats-shell.tsx rename to packages/data/app/src/routes/stats-shell.tsx diff --git a/packages/stats/app/src/routes/stats/api/health.ts b/packages/data/app/src/routes/stats/api/health.ts similarity index 100% rename from packages/stats/app/src/routes/stats/api/health.ts rename to packages/data/app/src/routes/stats/api/health.ts diff --git a/packages/stats/app/src/stats-runtime.ts b/packages/data/app/src/stats-runtime.ts similarity index 100% rename from packages/stats/app/src/stats-runtime.ts rename to packages/data/app/src/stats-runtime.ts diff --git a/packages/stats/app/src/wawoff2.d.ts b/packages/data/app/src/wawoff2.d.ts similarity index 100% rename from packages/stats/app/src/wawoff2.d.ts rename to packages/data/app/src/wawoff2.d.ts diff --git a/packages/stats/app/test/agent-formats.test.ts b/packages/data/app/test/agent-formats.test.ts similarity index 100% rename from packages/stats/app/test/agent-formats.test.ts rename to packages/data/app/test/agent-formats.test.ts diff --git a/packages/stats/app/test/compare-radar.test.ts b/packages/data/app/test/compare-radar.test.ts similarity index 100% rename from packages/stats/app/test/compare-radar.test.ts rename to packages/data/app/test/compare-radar.test.ts diff --git a/packages/stats/app/test/model-catalog.test.ts b/packages/data/app/test/model-catalog.test.ts similarity index 100% rename from packages/stats/app/test/model-catalog.test.ts rename to packages/data/app/test/model-catalog.test.ts diff --git a/packages/stats/app/test/og.test.ts b/packages/data/app/test/og.test.ts similarity index 100% rename from packages/stats/app/test/og.test.ts rename to packages/data/app/test/og.test.ts diff --git a/packages/stats/app/tsconfig.json b/packages/data/app/tsconfig.json similarity index 100% rename from packages/stats/app/tsconfig.json rename to packages/data/app/tsconfig.json diff --git a/packages/stats/app/vite.config.ts b/packages/data/app/vite.config.ts similarity index 100% rename from packages/stats/app/vite.config.ts rename to packages/data/app/vite.config.ts diff --git a/packages/stats/core/drizzle.config.ts b/packages/data/core/drizzle.config.ts similarity index 60% rename from packages/stats/core/drizzle.config.ts rename to packages/data/core/drizzle.config.ts index d7ec5d75a50..73ad3653797 100644 --- a/packages/stats/core/drizzle.config.ts +++ b/packages/data/core/drizzle.config.ts @@ -8,11 +8,11 @@ export default defineConfig({ out: "./migrations/", verbose: true, dbCredentials: { - database: Resource.StatsDatabase.database, - host: Resource.StatsDatabase.host, - user: Resource.StatsDatabase.username, - password: Resource.StatsDatabase.password, - port: Resource.StatsDatabase.port, + database: Resource.DataDatabase.database, + host: Resource.DataDatabase.host, + user: Resource.DataDatabase.username, + password: Resource.DataDatabase.password, + port: Resource.DataDatabase.port, ssl: { rejectUnauthorized: false, }, diff --git a/packages/stats/core/migrations/20260522121617_common_dust/migration.sql b/packages/data/core/migrations/20260522121617_common_dust/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260522121617_common_dust/migration.sql rename to packages/data/core/migrations/20260522121617_common_dust/migration.sql diff --git a/packages/stats/core/migrations/20260522121617_common_dust/snapshot.json b/packages/data/core/migrations/20260522121617_common_dust/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260522121617_common_dust/snapshot.json rename to packages/data/core/migrations/20260522121617_common_dust/snapshot.json diff --git a/packages/stats/core/migrations/20260523110335_cool_vin_gonzales/migration.sql b/packages/data/core/migrations/20260523110335_cool_vin_gonzales/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260523110335_cool_vin_gonzales/migration.sql rename to packages/data/core/migrations/20260523110335_cool_vin_gonzales/migration.sql diff --git a/packages/stats/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json b/packages/data/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json rename to packages/data/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json diff --git a/packages/stats/core/migrations/20260528012726_worthless_ultimo/migration.sql b/packages/data/core/migrations/20260528012726_worthless_ultimo/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260528012726_worthless_ultimo/migration.sql rename to packages/data/core/migrations/20260528012726_worthless_ultimo/migration.sql diff --git a/packages/stats/core/migrations/20260528012726_worthless_ultimo/snapshot.json b/packages/data/core/migrations/20260528012726_worthless_ultimo/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260528012726_worthless_ultimo/snapshot.json rename to packages/data/core/migrations/20260528012726_worthless_ultimo/snapshot.json diff --git a/packages/stats/core/migrations/20260528114447_misty_black_knight/migration.sql b/packages/data/core/migrations/20260528114447_misty_black_knight/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260528114447_misty_black_knight/migration.sql rename to packages/data/core/migrations/20260528114447_misty_black_knight/migration.sql diff --git a/packages/stats/core/migrations/20260528114447_misty_black_knight/snapshot.json b/packages/data/core/migrations/20260528114447_misty_black_knight/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260528114447_misty_black_knight/snapshot.json rename to packages/data/core/migrations/20260528114447_misty_black_knight/snapshot.json diff --git a/packages/stats/core/migrations/20260528121120_easy_multiple_man/migration.sql b/packages/data/core/migrations/20260528121120_easy_multiple_man/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260528121120_easy_multiple_man/migration.sql rename to packages/data/core/migrations/20260528121120_easy_multiple_man/migration.sql diff --git a/packages/stats/core/migrations/20260528121120_easy_multiple_man/snapshot.json b/packages/data/core/migrations/20260528121120_easy_multiple_man/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260528121120_easy_multiple_man/snapshot.json rename to packages/data/core/migrations/20260528121120_easy_multiple_man/snapshot.json diff --git a/packages/stats/core/migrations/20260620000000_unique_users/migration.sql b/packages/data/core/migrations/20260620000000_unique_users/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260620000000_unique_users/migration.sql rename to packages/data/core/migrations/20260620000000_unique_users/migration.sql diff --git a/packages/stats/core/migrations/20260826000000_model_retention/migration.sql b/packages/data/core/migrations/20260826000000_model_retention/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260826000000_model_retention/migration.sql rename to packages/data/core/migrations/20260826000000_model_retention/migration.sql diff --git a/packages/stats/core/migrations/20260903161929_parched_patriot/migration.sql b/packages/data/core/migrations/20260903161929_parched_patriot/migration.sql similarity index 100% rename from packages/stats/core/migrations/20260903161929_parched_patriot/migration.sql rename to packages/data/core/migrations/20260903161929_parched_patriot/migration.sql diff --git a/packages/stats/core/migrations/20260903161929_parched_patriot/snapshot.json b/packages/data/core/migrations/20260903161929_parched_patriot/snapshot.json similarity index 100% rename from packages/stats/core/migrations/20260903161929_parched_patriot/snapshot.json rename to packages/data/core/migrations/20260903161929_parched_patriot/snapshot.json diff --git a/packages/stats/core/package.json b/packages/data/core/package.json similarity index 97% rename from packages/stats/core/package.json rename to packages/data/core/package.json index 8c209003ac1..fa82a434c8c 100644 --- a/packages/stats/core/package.json +++ b/packages/data/core/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "name": "@opencode/stats-core", + "name": "@opencode/data-core", "version": "2.2.7", "private": true, "type": "module", diff --git a/packages/stats/core/src/config.ts b/packages/data/core/src/config.ts similarity index 95% rename from packages/stats/core/src/config.ts rename to packages/data/core/src/config.ts index 277749af77d..40fb3417568 100644 --- a/packages/stats/core/src/config.ts +++ b/packages/data/core/src/config.ts @@ -15,7 +15,7 @@ const config = Config.all({ publicUrl: Config.String("PUBLIC_URL").pipe(Config.withDefault("http://localhost:3000")), }).pipe(Config.map((value) => decodeAppConfigValue({ ...value, stage: Resource.App.stage }))) -export class AppConfig extends Context.Service()("@opencode/stats/AppConfig") { +export class AppConfig extends Context.Service()("@opencode/data/AppConfig") { static readonly config = config static readonly layer: Layer.Layer = Layer.effect( AppConfig, diff --git a/packages/stats/core/src/database.ts b/packages/data/core/src/database.ts similarity index 95% rename from packages/stats/core/src/database.ts rename to packages/data/core/src/database.ts index 1a64980d804..089d3663e43 100644 --- a/packages/stats/core/src/database.ts +++ b/packages/data/core/src/database.ts @@ -18,12 +18,12 @@ const config = Config.all({ // touch bindings at module scope. url: Config.NonEmptyString("DATABASE_URL").pipe( Config.option, - Config.map(Option.getOrElse(() => Resource.StatsDatabase.url)), + Config.map(Option.getOrElse(() => Resource.DataDatabase.url)), ), }).pipe(Config.map(decodeDatabaseSettings)) export class DatabaseConfig extends Context.Service()( - "@opencode/stats/DatabaseConfig", + "@opencode/data/DatabaseConfig", ) { static readonly config = config static readonly layer: Layer.Layer = Layer.effect( @@ -48,7 +48,7 @@ function makeDrizzle(settings: DatabaseSettings) { export type Drizzle = ReturnType -export class DrizzleClient extends Context.Service()("@opencode/stats/DrizzleClient") { +export class DrizzleClient extends Context.Service()("@opencode/data/DrizzleClient") { static readonly layer: Layer.Layer = Layer.effect( DrizzleClient, Effect.map(DatabaseConfig, makeDrizzle), diff --git a/packages/stats/core/src/database/schema.ts b/packages/data/core/src/database/schema.ts similarity index 100% rename from packages/stats/core/src/database/schema.ts rename to packages/data/core/src/database/schema.ts diff --git a/packages/stats/core/src/domain/catalog-identity.test.ts b/packages/data/core/src/domain/catalog-identity.test.ts similarity index 100% rename from packages/stats/core/src/domain/catalog-identity.test.ts rename to packages/data/core/src/domain/catalog-identity.test.ts diff --git a/packages/stats/core/src/domain/catalog-identity.ts b/packages/data/core/src/domain/catalog-identity.ts similarity index 100% rename from packages/stats/core/src/domain/catalog-identity.ts rename to packages/data/core/src/domain/catalog-identity.ts diff --git a/packages/stats/core/src/domain/geo.ts b/packages/data/core/src/domain/geo.ts similarity index 99% rename from packages/stats/core/src/domain/geo.ts rename to packages/data/core/src/domain/geo.ts index 7408c9410ba..29a1acc4a37 100644 --- a/packages/stats/core/src/domain/geo.ts +++ b/packages/data/core/src/domain/geo.ts @@ -64,7 +64,7 @@ export declare namespace GeoStatRepo { } } -export class GeoStatRepo extends Context.Service()("@opencode/stats/GeoStatRepo") { +export class GeoStatRepo extends Context.Service()("@opencode/data/GeoStatRepo") { static readonly layer: Layer.Layer = Layer.effect( GeoStatRepo, Effect.gen(function* () { diff --git a/packages/stats/core/src/domain/home.test.ts b/packages/data/core/src/domain/home.test.ts similarity index 100% rename from packages/stats/core/src/domain/home.test.ts rename to packages/data/core/src/domain/home.test.ts diff --git a/packages/stats/core/src/domain/home.ts b/packages/data/core/src/domain/home.ts similarity index 99% rename from packages/stats/core/src/domain/home.ts rename to packages/data/core/src/domain/home.ts index 4f4a3e31cb8..a8da61c19cf 100644 --- a/packages/stats/core/src/domain/home.ts +++ b/packages/data/core/src/domain/home.ts @@ -335,7 +335,7 @@ async function queryRows(query: string, params: string[] = []) { } function databaseUrl() { - return process.env.DATABASE_URL ?? Resource.StatsDatabase.url + return process.env.DATABASE_URL ?? Resource.DataDatabase.url } function stringValue(value: unknown) { diff --git a/packages/stats/core/src/domain/inference.test.ts b/packages/data/core/src/domain/inference.test.ts similarity index 100% rename from packages/stats/core/src/domain/inference.test.ts rename to packages/data/core/src/domain/inference.test.ts diff --git a/packages/stats/core/src/domain/inference.ts b/packages/data/core/src/domain/inference.ts similarity index 98% rename from packages/stats/core/src/domain/inference.ts rename to packages/data/core/src/domain/inference.ts index bab176515a0..fd36c42a157 100644 --- a/packages/stats/core/src/domain/inference.ts +++ b/packages/data/core/src/domain/inference.ts @@ -94,7 +94,7 @@ function defaultSource(): StatsQuerySource { return { namespace: Resource.R2Sql.namespace, table: Resource.R2Sql.table, - dataset: Resource.StatsSyncConfig.dataset, + dataset: Resource.DataSyncConfig.dataset, hiddenModels: hiddenStatModels(), } } @@ -401,7 +401,7 @@ export function toRetentionAggregate(data: R2SqlData, catalog?: CatalogIdentity) return [ { cohortDate: data.cohort_date, - dataset: data.dataset || Resource.StatsSyncConfig.dataset, + dataset: data.dataset || Resource.DataSyncConfig.dataset, tier: data.tier || "all", provider: (catalog ? data.provider : statProvider(data.model, "", data.provider)) || "unknown", model: statModel(data.model, undefined), @@ -419,7 +419,7 @@ function toStatBaseAggregate(data: R2SqlData): StatBaseAggregate[] { { grain, period_key: data.period_key, - dataset: data.dataset || Resource.StatsSyncConfig.dataset, + dataset: data.dataset || Resource.DataSyncConfig.dataset, tier: normalizeTier(data.tier || "unknown"), sessions: integer(data, "sessions"), requests: integer(data, "requests"), @@ -470,7 +470,7 @@ function sqlIdentifier(value: string) { } export function hiddenStatModels() { - return Resource.StatsHiddenModels.value + return Resource.DataHiddenModels.value .split(",") .map((model) => model.trim().toLowerCase()) .filter(Boolean) diff --git a/packages/stats/core/src/domain/model-normalization.ts b/packages/data/core/src/domain/model-normalization.ts similarity index 100% rename from packages/stats/core/src/domain/model-normalization.ts rename to packages/data/core/src/domain/model-normalization.ts diff --git a/packages/stats/core/src/domain/model.ts b/packages/data/core/src/domain/model.ts similarity index 99% rename from packages/stats/core/src/domain/model.ts rename to packages/data/core/src/domain/model.ts index fe67d2f5c31..493fe623f78 100644 --- a/packages/stats/core/src/domain/model.ts +++ b/packages/data/core/src/domain/model.ts @@ -55,7 +55,7 @@ export declare namespace ModelStatRepo { } export class ModelStatRepo extends Context.Service()( - "@opencode/stats/ModelStatRepo", + "@opencode/data/ModelStatRepo", ) { static readonly layer: Layer.Layer = Layer.effect( ModelStatRepo, diff --git a/packages/stats/core/src/domain/provider.ts b/packages/data/core/src/domain/provider.ts similarity index 99% rename from packages/stats/core/src/domain/provider.ts rename to packages/data/core/src/domain/provider.ts index 11a64e4a474..e01b500a91a 100644 --- a/packages/stats/core/src/domain/provider.ts +++ b/packages/data/core/src/domain/provider.ts @@ -46,7 +46,7 @@ export declare namespace ProviderStatRepo { } export class ProviderStatRepo extends Context.Service()( - "@opencode/stats/ProviderStatRepo", + "@opencode/data/ProviderStatRepo", ) { static readonly layer: Layer.Layer = Layer.effect( ProviderStatRepo, diff --git a/packages/stats/core/src/domain/retention.ts b/packages/data/core/src/domain/retention.ts similarity index 98% rename from packages/stats/core/src/domain/retention.ts rename to packages/data/core/src/domain/retention.ts index 689c5b9312c..8cee6c58b74 100644 --- a/packages/stats/core/src/domain/retention.ts +++ b/packages/data/core/src/domain/retention.ts @@ -26,7 +26,7 @@ export declare namespace RetentionStatRepo { } export class RetentionStatRepo extends Context.Service()( - "@opencode/stats/RetentionStatRepo", + "@opencode/data/RetentionStatRepo", ) { static readonly layer: Layer.Layer = Layer.effect( RetentionStatRepo, diff --git a/packages/stats/core/src/domain/stat.test.ts b/packages/data/core/src/domain/stat.test.ts similarity index 100% rename from packages/stats/core/src/domain/stat.test.ts rename to packages/data/core/src/domain/stat.test.ts diff --git a/packages/stats/core/src/domain/stat.ts b/packages/data/core/src/domain/stat.ts similarity index 100% rename from packages/stats/core/src/domain/stat.ts rename to packages/data/core/src/domain/stat.ts diff --git a/packages/stats/core/src/ensure-unique-users.ts b/packages/data/core/src/ensure-unique-users.ts similarity index 96% rename from packages/stats/core/src/ensure-unique-users.ts rename to packages/data/core/src/ensure-unique-users.ts index ff7373f07bd..803fa1ddd5e 100644 --- a/packages/stats/core/src/ensure-unique-users.ts +++ b/packages/data/core/src/ensure-unique-users.ts @@ -40,7 +40,7 @@ function databaseUrl() { JSON.stringify({ rejectUnauthorized: true }), )}` - return process.env.DATABASE_URL ?? Resource.StatsDatabase.url + return process.env.DATABASE_URL ?? Resource.DataDatabase.url } async function hasUniqueUsersColumn(table: (typeof tables)[number]) { diff --git a/packages/stats/core/src/honeycomb-backfill.ts b/packages/data/core/src/honeycomb-backfill.ts similarity index 100% rename from packages/stats/core/src/honeycomb-backfill.ts rename to packages/data/core/src/honeycomb-backfill.ts diff --git a/packages/stats/core/src/index.ts b/packages/data/core/src/index.ts similarity index 100% rename from packages/stats/core/src/index.ts rename to packages/data/core/src/index.ts diff --git a/packages/stats/core/src/migrate.ts b/packages/data/core/src/migrate.ts similarity index 100% rename from packages/stats/core/src/migrate.ts rename to packages/data/core/src/migrate.ts diff --git a/packages/stats/core/src/r2-sql.test.ts b/packages/data/core/src/r2-sql.test.ts similarity index 100% rename from packages/stats/core/src/r2-sql.test.ts rename to packages/data/core/src/r2-sql.test.ts diff --git a/packages/stats/core/src/r2-sql.ts b/packages/data/core/src/r2-sql.ts similarity index 99% rename from packages/stats/core/src/r2-sql.ts rename to packages/data/core/src/r2-sql.ts index 061d7290768..8965c088042 100644 --- a/packages/stats/core/src/r2-sql.ts +++ b/packages/data/core/src/r2-sql.ts @@ -72,7 +72,7 @@ export declare namespace R2Sql { } } -export class R2Sql extends Context.Service()("@opencode/stats/R2Sql") { +export class R2Sql extends Context.Service()("@opencode/data/R2Sql") { static readonly layer: Layer.Layer = Layer.succeed( R2Sql, R2Sql.of({ diff --git a/packages/stats/core/src/resource.ts b/packages/data/core/src/resource.ts similarity index 89% rename from packages/stats/core/src/resource.ts rename to packages/data/core/src/resource.ts index 736b3cc52a3..101afe53032 100644 --- a/packages/stats/core/src/resource.ts +++ b/packages/data/core/src/resource.ts @@ -19,7 +19,7 @@ export const Resource = { return stage() }, }, - StatsDatabase: { + DataDatabase: { database: "opencode-stats", port: 3306, get host() { @@ -48,14 +48,14 @@ export const Resource = { return required("R2_SQL_AUTH_TOKEN") }, }, - // Comma-separated model IDs kept off the public stats site. A secret, so excluding an unreleased model does not + // Comma-separated model IDs kept off the public data site. A secret, so excluding an unreleased model does not // publish its name in git. - StatsHiddenModels: { + DataHiddenModels: { get value() { return process.env.DATA_HIDDEN_MODELS ?? "" }, }, - StatsSyncConfig: { + DataSyncConfig: { dataset: "zen", }, } diff --git a/packages/stats/core/src/runtime.ts b/packages/data/core/src/runtime.ts similarity index 100% rename from packages/stats/core/src/runtime.ts rename to packages/data/core/src/runtime.ts diff --git a/packages/stats/core/src/stat-sync.test.ts b/packages/data/core/src/stat-sync.test.ts similarity index 96% rename from packages/stats/core/src/stat-sync.test.ts rename to packages/data/core/src/stat-sync.test.ts index d1edd30131f..d8f96060517 100644 --- a/packages/stats/core/src/stat-sync.test.ts +++ b/packages/data/core/src/stat-sync.test.ts @@ -11,7 +11,7 @@ const { ModelStatRepo } = await import("./domain/model") const { ProviderStatRepo } = await import("./domain/provider") const { RetentionStatRepo } = await import("./domain/retention") const { R2Sql, R2SqlQueryError } = await import("./r2-sql") -const { CatalogUnavailableError, deterministicFailureAsResult, planStatsSync, syncRetentionCohort, syncStatPeriod } = +const { CatalogUnavailableError, deterministicFailureAsResult, planDataSync, syncRetentionCohort, syncStatPeriod } = await import("./stat-sync") const { DatabaseError } = await import("./database") @@ -25,10 +25,10 @@ const loadCatalog = () => stale: false, }) -describe("stats sync plan", () => { +describe("data sync plan", () => { test("fails instead of planning a run without the model catalog", async () => { const error = await Effect.runPromise( - planStatsSync(new Date("2026-10-23T19:05:00.000Z"), { + planDataSync(new Date("2026-10-23T19:05:00.000Z"), { history: false, loadCatalog: () => Promise.resolve({ catalog: undefined, stale: true }), }).pipe(Effect.flip, Effect.provide(services({ events: [] }))), @@ -41,7 +41,7 @@ describe("stats sync plan", () => { const startedAt = new Date("2026-10-23T19:05:00.000Z") const plan = (history: boolean) => Effect.runPromise( - planStatsSync(startedAt, { history, loadCatalog }).pipe(Effect.provide(services({ events: [] }))), + planDataSync(startedAt, { history, loadCatalog }).pipe(Effect.provide(services({ events: [] }))), ) const hourly = await plan(false) @@ -93,7 +93,7 @@ describe("stats sync plan", () => { }) }) -describe("stats sync steps", () => { +describe("data sync steps", () => { test("drops retired and hidden dimensions from a period with no remaining rows", async () => { const events: string[] = [] @@ -153,7 +153,7 @@ describe("stats sync steps", () => { }) }) -describe("stats sync failures", () => { +describe("data sync failures", () => { test("returns a deterministic R2 SQL failure as the result", async () => { const result = await Effect.runPromise( deterministicFailureAsResult( diff --git a/packages/stats/core/src/stat-sync.ts b/packages/data/core/src/stat-sync.ts similarity index 89% rename from packages/stats/core/src/stat-sync.ts rename to packages/data/core/src/stat-sync.ts index de321caeaf6..6286ea3ec4f 100644 --- a/packages/stats/core/src/stat-sync.ts +++ b/packages/data/core/src/stat-sync.ts @@ -28,19 +28,19 @@ import { isDeterministicR2SqlError, R2Sql, R2SqlQueryError } from "./r2-sql" import { syncRange } from "./sync-range" // Plans cross a Workflow step boundary, so every value is plain JSON. -export type StatsSyncPeriod = { grain: StatGrain; key: string; start: string; end: string } -export type StatsSyncRetentionPeriod = { start: string; end: string; returnStart: string; returnEnd: string } -export type StatsSyncPlan = { +export type DataSyncPeriod = { grain: StatGrain; key: string; start: string; end: string } +export type DataSyncRetentionPeriod = { start: string; end: string; returnStart: string; returnEnd: string } +export type DataSyncPlan = { startedAt: string history: boolean periodStart: string periodEnd: string catalog: CatalogSnapshot - periods: StatsSyncPeriod[] - retentionPeriods: StatsSyncRetentionPeriod[] + periods: DataSyncPeriod[] + retentionPeriods: DataSyncRetentionPeriod[] } -export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string } -export type SyncStatsError = R2SqlQueryError | DatabaseError | CatalogUnavailableError +export type DataSyncResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string } +export type DataSyncError = R2SqlQueryError | DatabaseError | CatalogUnavailableError export class CatalogUnavailableError extends Error { readonly _tag = "CatalogUnavailableError" @@ -56,8 +56,8 @@ const STATS_CURSOR = ["dimension", "tier", "provider", "model", "country"] // Each stat period and retention cohort is independent: ranks, upserts and // deletes stay inside one (grain, period_key) or one cohort. A run is one plan // and one write per period and cohort, so each can be retried on its own. -export const syncStats = Effect.fn("StatSync.sync")(function* (options?: { history?: boolean }) { - const plan = yield* planStatsSync(yield* DateTime.nowAsDate, { history: options?.history ?? false }) +export const syncData = Effect.fn("StatSync.sync")(function* (options?: { history?: boolean }) { + const plan = yield* planDataSync(yield* DateTime.nowAsDate, { history: options?.history ?? false }) const periods = yield* Effect.forEach(plan.periods, (period) => syncStatPeriod(period, plan.catalog), { concurrency: 2, }) @@ -65,7 +65,7 @@ export const syncStats = Effect.fn("StatSync.sync")(function* (options?: { histo concurrency: 4, }) yield* Effect.logInfo( - `stats sync complete ${JSON.stringify({ + `data sync complete ${JSON.stringify({ history: plan.history, startedAt: plan.startedAt, periodStart: plan.periodStart, @@ -83,10 +83,10 @@ export const syncStats = Effect.fn("StatSync.sync")(function* (options?: { histo startedAt: plan.startedAt, periodStart: plan.periodStart, periodEnd: plan.periodEnd, - } satisfies SyncStatsResult + } satisfies DataSyncResult }) -export const planStatsSync = Effect.fn("StatSync.plan")(function* ( +export const planDataSync = Effect.fn("StatSync.plan")(function* ( startedAt: Date, options: { history: boolean; loadCatalog?: typeof loadCatalogIdentity }, ) { @@ -100,7 +100,7 @@ export const planStatsSync = Effect.fn("StatSync.plan")(function* ( const retentionAvailable = yield* retentionStats.available() yield* logRuntimeCheck() - const plan: StatsSyncPlan = { + const plan: DataSyncPlan = { startedAt: startedAt.toISOString(), history: options.history, periodStart: range.periodStart.toISOString(), @@ -122,7 +122,7 @@ export const planStatsSync = Effect.fn("StatSync.plan")(function* ( : [], } yield* Effect.logInfo( - `stats sync planned ${JSON.stringify({ + `data sync planned ${JSON.stringify({ history: plan.history, periodStart: plan.periodStart, periodEnd: plan.periodEnd, @@ -135,7 +135,7 @@ export const planStatsSync = Effect.fn("StatSync.plan")(function* ( }) export const syncStatPeriod = Effect.fn("StatSync.period")(function* ( - period: StatsSyncPeriod, + period: DataSyncPeriod, snapshot: CatalogSnapshot, ) { const r2Sql = yield* R2Sql @@ -180,7 +180,7 @@ export const syncStatPeriod = Effect.fn("StatSync.period")(function* ( }) // Scope by the period rather than the rows written, so a period whose only // rows were just hidden still drops them. - const scope = { grain: period.grain, periodKey: period.key, dataset: Resource.StatsSyncConfig.dataset } + const scope = { grain: period.grain, periodKey: period.key, dataset: Resource.DataSyncConfig.dataset } yield* Effect.all( [ modelStats.deleteRetiredDimensions(scope, hiddenStatModels()), @@ -223,7 +223,7 @@ export const syncStatPeriod = Effect.fn("StatSync.period")(function* ( }) export const syncRetentionCohort = Effect.fn("StatSync.retentionCohort")(function* ( - period: StatsSyncRetentionPeriod, + period: DataSyncRetentionPeriod, snapshot: CatalogSnapshot, ) { const r2Sql = yield* R2Sql @@ -253,7 +253,7 @@ export const syncRetentionCohort = Effect.fn("StatSync.retentionCohort")(functio // Replace only after the query succeeds, so a failed query keeps the last cohort. yield* retentionStats.replace(rows, { cohortDates: item.cohortDates, - dataset: Resource.StatsSyncConfig.dataset, + dataset: Resource.DataSyncConfig.dataset, tier: "Go", }) @@ -279,7 +279,7 @@ function logRuntimeCheck() { `r2 sql stats runtime check ${JSON.stringify({ accountId: Resource.R2Sql.accountId, bucket: Resource.R2Sql.bucket, - dataset: Resource.StatsSyncConfig.dataset, + dataset: Resource.DataSyncConfig.dataset, namespace: Resource.R2Sql.namespace, table: Resource.R2Sql.table, stage: Resource.App.stage, diff --git a/packages/stats/core/src/sync-range.test.ts b/packages/data/core/src/sync-range.test.ts similarity index 100% rename from packages/stats/core/src/sync-range.test.ts rename to packages/data/core/src/sync-range.test.ts diff --git a/packages/stats/core/src/sync-range.ts b/packages/data/core/src/sync-range.ts similarity index 100% rename from packages/stats/core/src/sync-range.ts rename to packages/data/core/src/sync-range.ts diff --git a/packages/stats/core/tsconfig.json b/packages/data/core/tsconfig.json similarity index 100% rename from packages/stats/core/tsconfig.json rename to packages/data/core/tsconfig.json diff --git a/packages/stats/server/package.json b/packages/data/server/package.json similarity index 88% rename from packages/stats/server/package.json rename to packages/data/server/package.json index 2c0e1de5bb5..68b25c7e760 100644 --- a/packages/stats/server/package.json +++ b/packages/data/server/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "name": "@opencode/stats-server", + "name": "@opencode/data-server", "version": "2.2.7", "private": true, "type": "module", @@ -12,7 +12,7 @@ "typecheck:tsc": "tsc --noEmit" }, "dependencies": { - "@opencode/stats-core": "workspace:*", + "@opencode/data-core": "workspace:*", "effect": "catalog:" }, "devDependencies": { diff --git a/packages/stats/server/src/cron.test.ts b/packages/data/server/src/cron.test.ts similarity index 68% rename from packages/stats/server/src/cron.test.ts rename to packages/data/server/src/cron.test.ts index af50422ffdf..8aada7bd392 100644 --- a/packages/stats/server/src/cron.test.ts +++ b/packages/data/server/src/cron.test.ts @@ -3,30 +3,30 @@ import cron from "./cron" const controller = { cron: "5 * * * *", scheduledTime: Date.parse("2026-11-03T04:05:00.000Z") } -describe("stats sync cron", () => { +describe("data sync cron", () => { test("starts this hour's run when no recent run exists", async () => { const workflow = fakeWorkflow({}) - await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never }) + await cron.scheduled(controller as never, { DataSyncWorkflow: workflow.binding as never }) expect(workflow.created).toEqual([ - { id: "stats-hourly-2026-11-03T04", params: { history: false, startedAt: "2026-11-03T04:05:00.000Z" } }, + { id: "data-hourly-2026-11-03T04", params: { history: false, startedAt: "2026-11-03T04:05:00.000Z" } }, ]) }) test("fails the tick instead of starting a run when a lookup fails for another reason", async () => { - const workflow = fakeWorkflow({ get: { "stats-hourly-2026-11-03T02": new Error("internal error") } }) + const workflow = fakeWorkflow({ get: { "data-hourly-2026-11-03T02": new Error("internal error") } }) - await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + await expect(cron.scheduled(controller as never, { DataSyncWorkflow: workflow.binding as never })).rejects.toThrow( "internal error", ) expect(workflow.created).toEqual([]) }) test("fails the tick when an instance's status cannot be read", async () => { - const workflow = fakeWorkflow({ status: { "stats-hourly-2026-11-03T03": new Error("status unavailable") } }) + const workflow = fakeWorkflow({ status: { "data-hourly-2026-11-03T03": new Error("status unavailable") } }) - await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + await expect(cron.scheduled(controller as never, { DataSyncWorkflow: workflow.binding as never })).rejects.toThrow( "status unavailable", ) expect(workflow.created).toEqual([]) @@ -35,7 +35,7 @@ describe("stats sync cron", () => { test("treats a concurrent create of the same slot as a skip", async () => { const workflow = fakeWorkflow({ create: new Error("(instance.already_exists) Instance already exists") }) - await cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never }) + await cron.scheduled(controller as never, { DataSyncWorkflow: workflow.binding as never }) expect(workflow.created).toEqual([]) }) @@ -43,7 +43,7 @@ describe("stats sync cron", () => { test("still fails on other create errors", async () => { const workflow = fakeWorkflow({ create: new Error("internal error") }) - await expect(cron.scheduled(controller as never, { StatsSyncWorkflow: workflow.binding as never })).rejects.toThrow( + await expect(cron.scheduled(controller as never, { DataSyncWorkflow: workflow.binding as never })).rejects.toThrow( "internal error", ) }) diff --git a/packages/stats/server/src/cron.ts b/packages/data/server/src/cron.ts similarity index 75% rename from packages/stats/server/src/cron.ts rename to packages/data/server/src/cron.ts index 524c1ffac97..f6c29667486 100644 --- a/packages/stats/server/src/cron.ts +++ b/packages/data/server/src/cron.ts @@ -4,10 +4,10 @@ import { isWorkflowError, recentInstanceIds, skipReason, - type StatsSyncParams, + type DataSyncParams, } from "./schedule" -type Env = { StatsSyncWorkflow: Workflow } +type Env = { DataSyncWorkflow: Workflow } export default { async scheduled(controller: ScheduledController, env: Env) { @@ -19,7 +19,7 @@ export default { const instances = await Promise.all( recentInstanceIds(scheduledAt, history).map(async (item) => ({ id: item, - status: await env.StatsSyncWorkflow.get(item) + status: await env.DataSyncWorkflow.get(item) .then((instance) => instance.status()) .then((value) => value.status) .catch((error) => { @@ -30,11 +30,11 @@ export default { ) const skip = skipReason(id, instances) if (skip) { - console.log(`stats sync skipped ${JSON.stringify({ id, ...skip })}`) + console.log(`data sync skipped ${JSON.stringify({ id, ...skip })}`) return } // Cron delivery is at least once, so a second tick for this slot can get here too. - const created = await env.StatsSyncWorkflow.create({ + const created = await env.DataSyncWorkflow.create({ id, params: { history, startedAt: scheduledAt.toISOString() }, }).then( @@ -45,9 +45,9 @@ export default { }, ) if (!created) { - console.log(`stats sync skipped ${JSON.stringify({ id, reason: "already created" })}`) + console.log(`data sync skipped ${JSON.stringify({ id, reason: "already created" })}`) return } - console.log(`stats sync started ${JSON.stringify({ id, history })}`) + console.log(`data sync started ${JSON.stringify({ id, history })}`) }, } diff --git a/packages/stats/server/src/schedule.test.ts b/packages/data/server/src/schedule.test.ts similarity index 60% rename from packages/stats/server/src/schedule.test.ts rename to packages/data/server/src/schedule.test.ts index 154c4ee4b00..46a0bd975f5 100644 --- a/packages/stats/server/src/schedule.test.ts +++ b/packages/data/server/src/schedule.test.ts @@ -1,37 +1,37 @@ import { describe, expect, test } from "bun:test" import { instanceId, isWorkflowError, recentInstanceIds, skipReason } from "./schedule" -describe("stats sync schedule", () => { +describe("data sync schedule", () => { test("names one instance per hourly slot and per history day", () => { const at = new Date("2026-11-03T03:20:00.000Z") - expect(instanceId(at, false)).toBe("stats-hourly-2026-11-03T03") - expect(instanceId(at, true)).toBe("stats-history-2026-11-03") - expect(instanceId(new Date("2026-11-03T03:05:59.999Z"), false)).toBe("stats-hourly-2026-11-03T03") + expect(instanceId(at, false)).toBe("data-hourly-2026-11-03T03") + expect(instanceId(at, true)).toBe("data-history-2026-11-03") + expect(instanceId(new Date("2026-11-03T03:05:59.999Z"), false)).toBe("data-hourly-2026-11-03T03") }) test("probes this hour and the previous five for an hourly run", () => { expect(recentInstanceIds(new Date("2026-11-01T02:05:00.000Z"), false)).toEqual([ - "stats-hourly-2026-11-01T02", - "stats-hourly-2026-11-01T01", - "stats-hourly-2026-11-01T00", - "stats-hourly-2026-10-31T23", - "stats-hourly-2026-10-31T22", - "stats-hourly-2026-10-31T21", + "data-hourly-2026-11-01T02", + "data-hourly-2026-11-01T01", + "data-hourly-2026-11-01T00", + "data-hourly-2026-10-31T23", + "data-hourly-2026-10-31T22", + "data-hourly-2026-10-31T21", ]) }) test("probes today and the previous two days for a history run", () => { expect(recentInstanceIds(new Date("2026-11-01T03:20:00.000Z"), true)).toEqual([ - "stats-history-2026-11-01", - "stats-history-2026-10-31", - "stats-history-2026-10-30", + "data-history-2026-11-01", + "data-history-2026-10-31", + "data-history-2026-10-30", ]) }) test("lets hourly and history runs overlap", () => { const at = new Date("2026-11-03T04:05:00.000Z") - const running = new Set(["stats-history-2026-11-03", "stats-hourly-2026-11-03T03"]) + const running = new Set(["data-history-2026-11-03", "data-hourly-2026-11-03T03"]) const decide = (history: boolean) => skipReason( instanceId(at, history), @@ -40,24 +40,24 @@ describe("stats sync schedule", () => { expect(decide(false)).toEqual({ reason: "previous run active", - instance: "stats-hourly-2026-11-03T03", + instance: "data-hourly-2026-11-03T03", status: "running", }) - running.delete("stats-hourly-2026-11-03T03") + running.delete("data-hourly-2026-11-03T03") expect(decide(false)).toBeUndefined() - expect(decide(true)).toEqual({ reason: "already created", instance: "stats-history-2026-11-03", status: "running" }) - running.delete("stats-history-2026-11-03") - running.add("stats-hourly-2026-11-03T04") + expect(decide(true)).toEqual({ reason: "already created", instance: "data-history-2026-11-03", status: "running" }) + running.delete("data-history-2026-11-03") + running.add("data-hourly-2026-11-03T04") expect(decide(true)).toBeUndefined() }) test("starts when no recent instance is active", () => { expect( - skipReason("stats-hourly-2026-11-03T04", [ - { id: "stats-hourly-2026-11-03T04", status: undefined }, - { id: "stats-hourly-2026-11-03T03", status: "complete" }, - { id: "stats-hourly-2026-11-03T02", status: "errored" }, - { id: "stats-hourly-2026-11-03T01", status: "terminated" }, + skipReason("data-hourly-2026-11-03T04", [ + { id: "data-hourly-2026-11-03T04", status: undefined }, + { id: "data-hourly-2026-11-03T03", status: "complete" }, + { id: "data-hourly-2026-11-03T02", status: "errored" }, + { id: "data-hourly-2026-11-03T01", status: "terminated" }, ]), ).toBeUndefined() }) @@ -66,18 +66,18 @@ describe("stats sync schedule", () => { "skips while a previous run is %s", (status) => { expect( - skipReason("stats-hourly-2026-11-03T04", [ - { id: "stats-hourly-2026-11-03T04", status: undefined }, - { id: "stats-hourly-2026-11-03T03", status }, + skipReason("data-hourly-2026-11-03T04", [ + { id: "data-hourly-2026-11-03T04", status: undefined }, + { id: "data-hourly-2026-11-03T03", status }, ]), - ).toEqual({ reason: "previous run active", instance: "stats-hourly-2026-11-03T03", status }) + ).toEqual({ reason: "previous run active", instance: "data-hourly-2026-11-03T03", status }) }, ) test("skips a slot that already has an instance in any state", () => { - expect(skipReason("stats-history-2026-11-03", [{ id: "stats-history-2026-11-03", status: "complete" }])).toEqual({ + expect(skipReason("data-history-2026-11-03", [{ id: "data-history-2026-11-03", status: "complete" }])).toEqual({ reason: "already created", - instance: "stats-history-2026-11-03", + instance: "data-history-2026-11-03", status: "complete", }) }) diff --git a/packages/stats/server/src/schedule.ts b/packages/data/server/src/schedule.ts similarity index 90% rename from packages/stats/server/src/schedule.ts rename to packages/data/server/src/schedule.ts index 5624e242d41..32715e24f7f 100644 --- a/packages/stats/server/src/schedule.ts +++ b/packages/data/server/src/schedule.ts @@ -10,12 +10,12 @@ const HOURLY_LOOKBACK = 6 const HISTORY_LOOKBACK = 3 const ACTIVE = new Set(["queued", "running", "waiting", "paused", "waitingForPause"]) -export type StatsSyncParams = { history?: boolean; startedAt?: string } +export type DataSyncParams = { history?: boolean; startedAt?: string } // One ID per schedule slot, so a repeated trigger cannot start a second run. export function instanceId(scheduledAt: Date, history: boolean) { - if (history) return `stats-history-${scheduledAt.toISOString().slice(0, 10)}` - return `stats-hourly-${scheduledAt.toISOString().slice(0, 13)}` + if (history) return `data-history-${scheduledAt.toISOString().slice(0, 10)}` + return `data-hourly-${scheduledAt.toISOString().slice(0, 13)}` } // The newest IDs of the same kind a run may still be using, starting with this diff --git a/packages/data/server/src/sync.ts b/packages/data/server/src/sync.ts new file mode 100644 index 00000000000..67fadf58503 --- /dev/null +++ b/packages/data/server/src/sync.ts @@ -0,0 +1,10 @@ +import { R2Sql } from "@opencode/data-core/r2-sql" +import { layer } from "@opencode/data-core/runtime" +import { syncData } from "@opencode/data-core/stat-sync" +import { Effect, Layer } from "effect" + +// Runs one sync pass in this process, for local use. Deployed stages run the +// same steps in the DataSyncWorkflow. +await Effect.runPromise( + syncData({ history: process.argv.includes("--history") }).pipe(Effect.provide(Layer.mergeAll(layer, R2Sql.layer))), +) diff --git a/packages/data/server/src/worker.ts b/packages/data/server/src/worker.ts new file mode 100644 index 00000000000..12a7dec8534 --- /dev/null +++ b/packages/data/server/src/worker.ts @@ -0,0 +1,3 @@ +// The sync Worker (services/data): the DataSyncWorkflow and the cron handler that starts its runs. +export { DataSyncWorkflow } from "./workflow" +export { default } from "./cron" diff --git a/packages/stats/server/src/workflow.ts b/packages/data/server/src/workflow.ts similarity index 75% rename from packages/stats/server/src/workflow.ts rename to packages/data/server/src/workflow.ts index a3f6772c773..d6bd4383b1c 100644 --- a/packages/stats/server/src/workflow.ts +++ b/packages/data/server/src/workflow.ts @@ -1,15 +1,15 @@ import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep, type WorkflowStepConfig } from "cloudflare:workers" -import { R2Sql } from "@opencode/stats-core/r2-sql" -import { layer } from "@opencode/stats-core/runtime" +import { R2Sql } from "@opencode/data-core/r2-sql" +import { layer } from "@opencode/data-core/runtime" import { deterministicFailureAsResult, - planStatsSync, + planDataSync, syncRetentionCohort, syncStatPeriod, - type SyncStatsError, -} from "@opencode/stats-core/stat-sync" + type DataSyncError, +} from "@opencode/data-core/stat-sync" import { Effect, Layer } from "effect" -import type { StatsSyncParams } from "./schedule" +import type { DataSyncParams } from "./schedule" const runtimeLayer = Layer.mergeAll(layer, R2Sql.layer) // Each step runs one period's R2 SQL queries (10-minute request timeout, with @@ -19,12 +19,12 @@ const STEP: WorkflowStepConfig = { retries: { limit: 2, delay: "1 minute", backoff: "exponential" }, } -export class StatsSyncWorkflow extends WorkflowEntrypoint { - async run(event: WorkflowEvent, step: WorkflowStep) { +export class DataSyncWorkflow extends WorkflowEntrypoint { + async run(event: WorkflowEvent, step: WorkflowStep) { // A dashboard trigger can start an instance without params. const plan = await step.do("plan", STEP, () => run( - planStatsSync(new Date(event.payload?.startedAt ?? event.timestamp), { + planDataSync(new Date(event.payload?.startedAt ?? event.timestamp), { history: event.payload?.history ?? false, }), ), @@ -36,7 +36,7 @@ export class StatsSyncWorkflow extends WorkflowEntrypoint { failed.push(name) console.error( - `stats sync step failed ${JSON.stringify({ + `data sync step failed ${JSON.stringify({ step: name, error: error instanceof Error ? error.message : String(error), cause: error instanceof Error && error.cause !== undefined ? String(error.cause) : undefined, @@ -64,11 +64,11 @@ export class StatsSyncWorkflow extends WorkflowEntrypoint run(deterministicFailureAsResult(syncRetentionCohort(period, plan.catalog)))), ) } - if (failed.length > 0) throw new Error(`stats sync failed for ${failed.join(", ")}`) + if (failed.length > 0) throw new Error(`data sync failed for ${failed.join(", ")}`) return { periods: plan.periods.length, retentionPeriods: plan.retentionPeriods.length } } } -function run(effect: Effect.Effect>) { +function run(effect: Effect.Effect>) { return Effect.runPromise(effect.pipe(Effect.provide(runtimeLayer))) } diff --git a/packages/stats/server/tsconfig.json b/packages/data/server/tsconfig.json similarity index 100% rename from packages/stats/server/tsconfig.json rename to packages/data/server/tsconfig.json diff --git a/packages/stats/AGENTS.md b/packages/stats/AGENTS.md deleted file mode 100644 index a52cfa50a9c..00000000000 --- a/packages/stats/AGENTS.md +++ /dev/null @@ -1 +0,0 @@ -To start the stats site locally, run `bun run --cwd packages/stats/app dev` with the database variables in README.md exported. diff --git a/packages/stats/server/src/sync.ts b/packages/stats/server/src/sync.ts deleted file mode 100644 index c1f67d9d099..00000000000 --- a/packages/stats/server/src/sync.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { R2Sql } from "@opencode/stats-core/r2-sql" -import { layer } from "@opencode/stats-core/runtime" -import { syncStats } from "@opencode/stats-core/stat-sync" -import { Effect, Layer } from "effect" - -// Runs one sync pass in this process, for local use. Deployed stages run the -// same steps in the StatsSyncWorkflow. -await Effect.runPromise( - syncStats({ history: process.argv.includes("--history") }).pipe(Effect.provide(Layer.mergeAll(layer, R2Sql.layer))), -) diff --git a/packages/stats/server/src/worker.ts b/packages/stats/server/src/worker.ts deleted file mode 100644 index f342e1295c0..00000000000 --- a/packages/stats/server/src/worker.ts +++ /dev/null @@ -1,3 +0,0 @@ -// The sync Worker (services/data): the StatsSyncWorkflow and the cron handler that starts its runs. -export { StatsSyncWorkflow } from "./workflow" -export { default } from "./cron" diff --git a/script/release.ts b/script/release.ts index 1b4fd9ea18f..4beb4855c77 100755 --- a/script/release.ts +++ b/script/release.ts @@ -9,11 +9,11 @@ import path from "path" import { parseArgs } from "util" const root = path.resolve(import.meta.dir, "..") -// Commits that only touch these folders, such as the console, stats, services, docs, websites, and infra, are +// Commits that only touch these folders, such as the console, the data site, services, docs, websites, and infra, are // left out of the changelog. const changelogExclusions = [ ":(exclude)packages/console", - ":(exclude)packages/stats", + ":(exclude)packages/data", ":(exclude)services", ":(exclude)infra", ] diff --git a/services/data/README.md b/services/data/README.md index 187ab43c39e..0f6d93988fc 100644 --- a/services/data/README.md +++ b/services/data/README.md @@ -1,6 +1,6 @@ # Data -Deploys the opencode.ai/data site from `packages/stats` as three Workers with `cf`: +Deploys the opencode.ai/data site from `packages/data` as three Workers with `cf`: | Worker | Config | What it is | | ---------------------------- | ---------------------- | ----------------------------------------------------------------------------- | @@ -18,7 +18,12 @@ its config declares. The database is the `opencode-stats` PlanetScale database: the `dev` branch for dev and `production` for production, through the `data-worker-` password. It is not managed as code. +Every config, `script/build.ts` and `script/deploy.ts` take the stage from `--mode`, which must be `dev` or +`production` (`stage.ts`). Anything else, including no mode, throws instead of falling back to production, so pass +`--mode` to `cf` commands that read the configs too (`cf deploy --dry-run --mode dev`). + ```bash -bun run build --mode dev +bun run dev # the site on your machine (packages/data/app) +bun run build:dev # or build:production bun run typecheck ``` diff --git a/services/data/cloudflare.config.ts b/services/data/cloudflare.config.ts index 231ce572b36..778a2dc23c0 100644 --- a/services/data/cloudflare.config.ts +++ b/services/data/cloudflare.config.ts @@ -1,30 +1,31 @@ import { bindings, defineConfig, triggers } from "cf/config" -import { LOCALES } from "../../packages/stats/app/src/lib/language" +import { LOCALES } from "../../packages/data/app/src/lib/language" import { ogWorkerName } from "./og.config" +import { stage, stageHost } from "./stage" -// The data site (packages/stats/app, SolidStart) at /data and //data on the main host. `--mode dev` deploys -// opencode-data-dev on dev.opencode.ai; any other mode deploys opencode-data-production on opencode.ai. The routes -// are its only way in: it has no custom domain or workers.dev URL. An exact `/data` pattern misses `/data?query`, so -// each pattern is a `/data*` prefix, and route patterns can't wildcard the middle of a path, hence one per locale. +// The data site (packages/data/app, SolidStart) at /data and //data on the main host. `--mode dev` deploys +// opencode-data-dev on dev.opencode.ai and `--mode production` opencode-data-production on opencode.ai; any other +// mode throws (stage.ts). The routes are its only way in: it has no custom domain or workers.dev URL. An exact `/data` +// pattern misses `/data?query`, so each pattern is a `/data*` prefix, and route patterns can't wildcard the middle of +// a path, hence one per locale. // script/build.ts builds this, og.config.ts and sync.config.ts into one Build Output tree. export default defineConfig(({ mode }) => { - const dev = mode === "dev" - const stage = dev ? "dev" : "production" - const host = dev ? "dev.opencode.ai" : "opencode.ai" + const current = stage(mode) + const host = stageHost(current) return { // Anomaly accountId: "15d29c8639fd3733b1b5486a2acfd968", worker: { - name: `opencode-data-${stage}`, + name: `opencode-data-${current}`, compatibilityDate: "2026-07-25", // The Nitro server is built with node compatibility, which also copies string bindings into process.env - // (packages/stats/core/src/resource.ts). + // (packages/data/core/src/resource.ts). compatibilityFlags: ["nodejs_compat"], env: { ASSETS: bindings.assets(), // The private Worker that renders Open Graph images (og.config.ts). OG_WORKER: bindings.worker({ worker: ogWorkerName(mode) }), - STAGE: bindings.text(stage), + STAGE: bindings.text(current), PUBLIC_URL: bindings.text(`https://${host}/data`), // Set from AWS Secrets Manager (opencode//DATA_*) by the deploy job. DATA_DATABASE_HOST: bindings.secret(), diff --git a/services/data/og.config.ts b/services/data/og.config.ts index a943526d0d4..923ec01c803 100644 --- a/services/data/og.config.ts +++ b/services/data/og.config.ts @@ -1,8 +1,9 @@ import { defineConfig } from "cf/config" +import { stage } from "./stage" -export const ogWorkerName = (mode: string | undefined) => `opencode-data-og-${mode === "dev" ? "dev" : "production"}` +export const ogWorkerName = (mode: string | undefined) => `opencode-data-og-${stage(mode)}` -// The private Worker that renders the data pages' Open Graph images (packages/stats/app/src/og/worker.ts). It has no +// The private Worker that renders the data pages' Open Graph images (packages/data/app/src/og/worker.ts). It has no // URL or route; the data Worker reaches it through its OG_WORKER Service Binding. It is its own Worker so the data // Worker doesn't load the Wasm engines and fonts it ships as modules (script/build.ts). export default defineConfig(({ mode }) => ({ diff --git a/services/data/package.json b/services/data/package.json index f30669bd7fb..ac325499cb4 100644 --- a/services/data/package.json +++ b/services/data/package.json @@ -5,7 +5,9 @@ "type": "module", "license": "MIT", "scripts": { - "build": "bun script/build.ts", + "dev": "bun run --cwd ../../packages/data/app dev", + "build:dev": "bun script/build.ts --mode dev", + "build:production": "bun script/build.ts --mode production", "deploy:dev": "bun script/deploy.ts --mode dev", "deploy:production": "bun script/deploy.ts --mode production", "typecheck": "bun check --no-script", @@ -14,8 +16,8 @@ "devDependencies": { "@cloudflare/build-output-utils": "0.8.5", "@cloudflare/config": "0.23.0", - "@opencode/stats-app": "workspace:*", - "@opencode/stats-server": "workspace:*", + "@opencode/data-app": "workspace:*", + "@opencode/data-server": "workspace:*", "@types/bun": "catalog:", "@typescript/native": "catalog:", "cf": "1.0.0-beta.12", diff --git a/services/data/script/build.ts b/services/data/script/build.ts index b015d976cdc..e0171708074 100644 --- a/services/data/script/build.ts +++ b/services/data/script/build.ts @@ -1,11 +1,11 @@ // Builds the data site's three Workers into one Build Output Specification tree (.cloudflare/output) for // `cf deploy --prebuilt --worker ` (script/deploy.ts): -// - default: the SolidStart site (packages/stats/app, Nitro's cloudflare-module preset) with .output/public as assets +// - default: the SolidStart site (packages/data/app, Nitro's cloudflare-module preset) with .output/public as assets // - og: the Open Graph image Worker, whose Wasm engines and fonts ship as modules -// - sync: the StatsSyncWorkflow and its cron handler (packages/stats/server) +// - sync: the DataSyncWorkflow and its cron handler (packages/data/server) // cf's own build step can't detect these projects, so they deploy prebuilt. // -// bun script/build.ts [--mode dev|production] +// bun script/build.ts --mode dev|production import { cp, mkdir, readdir } from "node:fs/promises" import path from "node:path" import { @@ -16,14 +16,14 @@ import { writeWorkerConfig, } from "@cloudflare/build-output-utils" import { resolveAndParseConfig } from "@cloudflare/config" +import { stage } from "../stage" const root = path.resolve(import.meta.dir, "..") -const app = path.resolve(root, "../../packages/stats/app") -const server = path.resolve(root, "../../packages/stats/server") +const app = path.resolve(root, "../../packages/data/app") +const server = path.resolve(root, "../../packages/data/server") const modeIndex = process.argv.indexOf("--mode") -const mode = modeIndex === -1 ? "production" : process.argv[modeIndex + 1] -if (mode !== "dev" && mode !== "production") throw new Error(`Unknown --mode ${mode}; use dev or production`) -// STAGE picks the stage's headers for the static assets (packages/stats/app/src/lib/noindex.ts). +const mode = stage(modeIndex === -1 ? undefined : process.argv[modeIndex + 1]) +// STAGE picks the stage's headers for the static assets (packages/data/app/src/lib/noindex.ts). const env = { ...process.env, STAGE: mode } await Bun.$`bun run build`.cwd(app).env(env) diff --git a/services/data/script/deploy.ts b/services/data/script/deploy.ts index 3d204e37246..c80d0d432f0 100644 --- a/services/data/script/deploy.ts +++ b/services/data/script/deploy.ts @@ -6,14 +6,14 @@ import { rm } from "node:fs/promises" import path from "node:path" import { resolveAndParseConfig } from "@cloudflare/config" +import { stage } from "../stage" const root = path.resolve(import.meta.dir, "..") const flag = (name: string) => { const index = process.argv.indexOf(name) return index === -1 ? undefined : process.argv[index + 1] } -const mode = flag("--mode") -if (mode !== "dev" && mode !== "production") throw new Error(`Unknown --mode ${mode}; use dev or production`) +const mode = stage(flag("--mode")) const secretsFile = flag("--secrets-file") const secrets: Record = secretsFile ? await Bun.file(secretsFile).json() : {} diff --git a/services/data/stage.ts b/services/data/stage.ts new file mode 100644 index 00000000000..6500e1dc435 --- /dev/null +++ b/services/data/stage.ts @@ -0,0 +1,18 @@ +// The stage a `--mode` deploys. Only `dev` and `production` are stages: anything else, including no mode at all, throws, +// so a typo or a bare `cf deploy` can never fall through to production. +const hosts = { + dev: "dev.opencode.ai", + production: "opencode.ai", +} as const + +export type Stage = keyof typeof hosts + +export function stage(mode: string | undefined): Stage { + if (mode === "dev" || mode === "production") return mode + throw new Error(`Unknown mode ${JSON.stringify(mode)}; pass --mode dev or --mode production`) +} + +// The host the stage's site is served from. +export function stageHost(value: Stage) { + return hosts[value] +} diff --git a/services/data/sync.config.ts b/services/data/sync.config.ts index 81ad132a2ef..8d7985c9dde 100644 --- a/services/data/sync.config.ts +++ b/services/data/sync.config.ts @@ -1,28 +1,29 @@ import { bindings, defineConfig, exports, triggers } from "cf/config" +import { stage } from "./stage" -// The Worker that syncs R2 SQL aggregates into the data database (packages/stats/server/src/worker.ts): the -// StatsSyncWorkflow, and the cron handler that starts its runs. The schedules must match -// packages/stats/server/src/schedule.ts, which maps the daily one to a history run. +// The Worker that syncs R2 SQL aggregates into the data database (packages/data/server/src/worker.ts): the +// DataSyncWorkflow, and the cron handler that starts its runs. The schedules must match +// packages/data/server/src/schedule.ts, which maps the daily one to a history run. export default defineConfig(({ mode }) => { - const stage = mode === "dev" ? "dev" : "production" - const name = `opencode-data-sync-${stage}` + const current = stage(mode) + const name = `opencode-data-sync-${current}` return { // Anomaly accountId: "15d29c8639fd3733b1b5486a2acfd968", worker: { name, compatibilityDate: "2026-07-25", - // Also copies string bindings into process.env (packages/stats/core/src/resource.ts). + // Also copies string bindings into process.env (packages/data/core/src/resource.ts). compatibilityFlags: ["nodejs_compat"], // The 5-minute maximum, for ranking a full week of geo rows. limits: { cpuMs: 300_000 }, exports: { // One instance per run: a plan step, then one step per stat period and retention cohort. - StatsSyncWorkflow: exports.workflow({ name }), + DataSyncWorkflow: exports.workflow({ name }), }, env: { - StatsSyncWorkflow: bindings.workflow({ name, worker: name, exportName: "StatsSyncWorkflow" }), - STAGE: bindings.text(stage), + DataSyncWorkflow: bindings.workflow({ name, worker: name, exportName: "DataSyncWorkflow" }), + STAGE: bindings.text(current), // Set from AWS Secrets Manager (opencode//*) by the deploy job. DATA_DATABASE_HOST: bindings.secret(), DATA_DATABASE_USERNAME: bindings.secret(),