diff --git a/packages/stats/app/src/lib/noindex.ts b/packages/stats/app/src/lib/noindex.ts new file mode 100644 index 00000000000..fcb86e4e094 --- /dev/null +++ b/packages/stats/app/src/lib/noindex.ts @@ -0,0 +1,27 @@ +import type { createMiddleware } from "@solidjs/start/middleware" +import { Resource } from "sst/resource" + +// Only production may be indexed: dev.opencode.ai and the personal stages run unreleased builds, and +// search engines must never list them in place of opencode.ai. Every response there asks them not +// to index it: the Worker's through this middleware, and the static assets the Worker never sees +// through the rule vite.config.ts adds to those stages' _headers. + +type Middleware = ReturnType[number] + +/** + * The headers every response from this stage carries: none in production, or outside SST (`vite dev` on its own), + * where no App is linked. SST links the App to the build as well as the Worker. + */ +export function stageHeaders(): Record { + const stage = "App" in Resource ? Resource.App.stage : "production" + return stage === "production" ? {} : { "X-Robots-Tag": "noindex, nofollow" } +} + +// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more. +export const noindexOutsideProduction: Middleware = (event, next) => { + Object.entries(stageHeaders()).forEach(([name, value]) => { + event.res.headers.set(name, value) + event.res.errHeaders.set(name, value) + }) + return next() +} diff --git a/packages/stats/app/src/middleware.ts b/packages/stats/app/src/middleware.ts index 682da302584..411c398ebe0 100644 --- a/packages/stats/app/src/middleware.ts +++ b/packages/stats/app/src/middleware.ts @@ -21,6 +21,7 @@ import { route, strip, } from "./lib/language" +import { noindexOutsideProduction } from "./lib/noindex" import { loadComparePage, loadHomePage, loadLabPage, loadModelPage } from "./lib/page-data" import { canonicalModelEntry, catalogModelPath, findModelCatalogLab, loadModelCatalog } from "./routes/model-catalog" import { statsPageCacheControl } from "./routes/stats-cache" @@ -47,7 +48,7 @@ const contentTypes = { text: "text/plain", } as const satisfies Record -export default createMiddleware({ +const pageMiddleware = createMiddleware({ onRequest: async (event) => { const request = event.request if (request.method !== "GET" && request.method !== "HEAD") return @@ -82,6 +83,9 @@ export default createMiddleware({ }, }) +// The noindex header outside production first (lib/noindex.ts), so every response carries it. +export default [noindexOutsideProduction, ...pageMiddleware] + // Unprefixed pages redirect to the visitor's language from their cookie or Accept-Language. /en/data stays put so // English remains reachable for visitors whose browser prefers another language. function localizedRedirect(request: Request, url: URL) { diff --git a/packages/stats/app/vite.config.ts b/packages/stats/app/vite.config.ts index 068d0d91886..d0c36d31ff7 100644 --- a/packages/stats/app/vite.config.ts +++ b/packages/stats/app/vite.config.ts @@ -3,6 +3,7 @@ import path from "node:path" import { solidStart } from "@solidjs/start/config" import { nitro } from "nitro/vite" import { defineConfig, type PluginOption } from "vite" +import { stageHeaders } from "./src/lib/noindex" export default defineConfig({ base: "/data/", @@ -25,9 +26,12 @@ export default defineConfig({ order: "post", async handler() { const source = path.join(import.meta.dirname, ".output/public/data/_headers") + // The stage's headers for the assets the Worker never sees (src/lib/noindex.ts). + const stage = Object.entries(stageHeaders()).map(([name, value]) => ` ${name}: ${value}\n`) await writeFile( path.join(import.meta.dirname, ".output/public/_headers"), - (await readFile(source, "utf8")).replace(/^\//gm, "/data/"), + (await readFile(source, "utf8")).replace(/^\//gm, "/data/") + + (stage.length ? `\n/data/*\n${stage.join("")}` : ""), ) await rm(source) }, diff --git a/services/files/src/index.ts b/services/files/src/index.ts index 3591ffdac4c..c9061479456 100644 --- a/services/files/src/index.ts +++ b/services/files/src/index.ts @@ -8,6 +8,8 @@ export default { "Range, If-Range, If-Match, If-None-Match, If-Modified-Since, If-Unmodified-Since", "Access-Control-Expose-Headers": "Accept-Ranges, Content-Length, Content-Range, Content-Disposition, ETag, Last-Modified", + // Like everything on dev.opencode.ai, dev's files ask search engines not to index them. + ...(env.STAGE === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {}), } if (!url.pathname.startsWith("/files/")) { @@ -153,7 +155,7 @@ export default { } return response }, -} satisfies ExportedHandler<{ FILES: R2Bucket }> +} satisfies ExportedHandler<{ FILES: R2Bucket; STAGE: "dev" | "production" }> function matches(value: string, etag: string, weak: boolean) { return value.split(",").some((part) => { diff --git a/services/files/wrangler.jsonc b/services/files/wrangler.jsonc index b623f5d293c..8ea83447e81 100644 --- a/services/files/wrangler.jsonc +++ b/services/files/wrangler.jsonc @@ -8,11 +8,13 @@ "env": { "dev": { "name": "opencode-dev-files", + "vars": { "STAGE": "dev" }, "routes": [{ "pattern": "dev.opencode.ai/files/*", "zone_name": "opencode.ai" }], "r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-dev-files" }], }, "production": { "name": "opencode-production-files", + "vars": { "STAGE": "production" }, "routes": [{ "pattern": "opencode.ai/files/*", "zone_name": "opencode.ai" }], "r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-production-files" }], }, diff --git a/services/site/public/_headers b/services/site/public/_headers index a915898be85..f2f67a020cf 100644 --- a/services/site/public/_headers +++ b/services/site/public/_headers @@ -1,6 +1,7 @@ # Cache headers for the static files in public/ (Cloudflare Workers static assets; script/build.ts ships # .output/public/_headers with the assets). Nitro appends its own /_build/assets/* rule (hashed -# build files, immutable). Rules must not overlap: Cloudflare joins repeated headers with a comma. +# build files, immutable), and dev builds a /* X-Robots-Tag rule (src/lib/noindex.ts). Rules must not +# overlap: Cloudflare joins repeated headers with a comma. # # Only files whose name changes with their content are immutable. The media below keep their URL # when re-rendered in place, so browsers and the edge may keep them for a day and serve a stale diff --git a/services/site/script/build.ts b/services/site/script/build.ts index 61302169e51..1dbd181a3b6 100644 --- a/services/site/script/build.ts +++ b/services/site/script/build.ts @@ -4,7 +4,7 @@ // .output/public as its assets. cf's own build step can't detect this project, so it deploys prebuilt. // // bun script/build.ts [--mode dev|production] -import { cp, mkdir, rm } from "node:fs/promises" +import { appendFile, cp, mkdir, rm } from "node:fs/promises" import path from "node:path" import { cleanBuildOutputDir, @@ -15,6 +15,7 @@ import { } from "@cloudflare/build-output-utils" import { resolveAndParseConfig } from "@cloudflare/config" import { mediaFile } from "../src/lib/media" +import { stageHeaders } from "../src/lib/noindex" const root = path.resolve(import.meta.dir, "..") const modeIndex = process.argv.indexOf("--mode") @@ -67,6 +68,9 @@ const fixed = ["config.json", "tui.json", "openapi.json", "v2/cli.json", "docs/o (file) => !shipped.includes(file), ) if (fixed.length) throw new Error(`fixed-URL files missing from Worker assets: ${fixed.join(", ")}`) +// The stage's headers for the assets the Worker never sees; its own responses get them from src/lib/noindex.ts. +const headers = Object.entries(stageHeaders(mode)).map(([name, value]) => ` ${name}: ${value}\n`) +if (headers.length) await appendFile(path.join(assets, "_headers"), `\n/*\n${headers.join("")}`) await writeAssets({ root, sourceDirectory: assets }) const bundle = getWorkerBundleDir(root) diff --git a/services/site/script/check-worker-start.ts b/services/site/script/check-worker-start.ts index 151a76d1946..9f16bd60f71 100644 --- a/services/site/script/check-worker-start.ts +++ b/services/site/script/check-worker-start.ts @@ -13,11 +13,14 @@ import path from "node:path" import { readBuildOutput } from "@cloudflare/build-output-utils" import { Miniflare, Response } from "miniflare" +import { stageHeaders } from "../src/lib/noindex" const root = path.resolve(import.meta.dir, "..") // Pages with a status must answer exactly that, others anything but a 5xx; a redirect must go to its // location. The docs page is asked for as a browser asks and, at a second URL, as an agent asks (its -// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect. +// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect. The +// last three are files the Worker serves, or the assets layer before it (_headers). Every answer +// carries the stage's headers (src/lib/noindex.ts): dev's noindex, none in production. const pages: { path: string; status?: number; accept?: string; location?: string }[] = [ { path: "/", status: 200 }, { path: "/ja", status: 200 }, @@ -27,6 +30,9 @@ const pages: { path: string; status?: number; accept?: string; location?: string { path: "/docs/config/", status: 200, accept: "text/markdown" }, { path: "/v2/docs/config/", status: 301, location: "/docs/config/" }, { path: "/docs/ja/", status: 302, location: "/v1/docs/ja/" }, + { path: "/robots.txt", status: 200 }, + { path: "/docs/llms.txt", status: 200 }, + { path: "/favicon.svg", status: 200 }, ] const started = performance.now() @@ -52,6 +58,9 @@ const env = Object.fromEntries( }), ) +const stage = env.STAGE?.type === "text" && env.STAGE.value === "production" ? "production" : "dev" +const robots = new Headers(stageHeaders(stage)).get("x-robots-tag") ?? undefined + const errors: string[] = [] let offline = 0 const miniflare = new Miniflare({ @@ -92,10 +101,14 @@ const results = await Promise.resolve() }) .then(async (response) => { await response.arrayBuffer() - return { status: response.status, location: response.headers.get("location") ?? undefined } + return { + status: response.status, + location: response.headers.get("location") ?? undefined, + robots: response.headers.get("x-robots-tag") ?? undefined, + } }) - .catch(() => ({ status: 0, location: undefined })) - answers.push({ ...page, actual: actual.status, actualLocation: actual.location }) + .catch(() => ({ status: 0, location: undefined, robots: undefined })) + answers.push({ ...page, actual: actual.status, actualLocation: actual.location, actualRobots: actual.robots }) } return answers }) @@ -110,12 +123,14 @@ const failures = results .filter( (result) => (result.status ? result.actual !== result.status : result.actual === 0 || result.actual >= 500) || - result.actualLocation !== result.location, + result.actualLocation !== result.location || + result.actualRobots !== robots, ) .map( (result) => `${result.path} answered ${result.actual || "nothing"}${result.status ? `, not ${result.status}` : ""}` + - (result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : ""), + (result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : "") + + (result.actualRobots !== robots ? ` with X-Robots-Tag ${result.actualRobots ?? "absent"} in ${stage}` : ""), ) if (failures.length || errors.length) { console.error(`check-worker-start: FAILED\n${[...errors, ...failures].map((line) => ` ${line}`).join("\n")}`) @@ -123,5 +138,5 @@ if (failures.length || errors.length) { } const answered = results.map((result) => `${result.path} ${result.actual}`).join(", ") console.log( - `check-worker-start: ${config.name} starts in workerd; ${answered}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`, + `check-worker-start: ${config.name} starts in workerd; ${answered}; X-Robots-Tag ${robots ?? "absent"}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`, ) diff --git a/services/site/src/docs/AGENTS.md b/services/site/src/docs/AGENTS.md index e50f4bd2f4a..88474914b0a 100644 --- a/services/site/src/docs/AGENTS.md +++ b/services/site/src/docs/AGENTS.md @@ -28,7 +28,7 @@ ## Build and serving -- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `--mode dev` makes `llms.txt` and the sitemap list dev.opencode.ai. +- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `llms.txt` and the sitemap list opencode.ai's URLs in every mode, so dev never advertises its own. - Pull requests that touch the site run the same build on its own (`.github/workflows/check-docs.yml`): `bun run generate`, then `bun src/docs/script/build.ts --mode dev --out `. - The Worker never renders docs. `middleware.ts` serves the prerendered files: docs URLs reach it first (`runWorkerFirst` in `cloudflare.config.ts`), so a page's URL answers `Accept: text/markdown` with its twin. `vite dev` has no prerendered files, so the development server renders on request. - `/docs` still receives the V1 docs' old links, which had more pages and other languages (`/docs/ja/...`). `middleware.ts` redirects any `/docs` path that isn't a page here (`virtual:docs-pages`, the same list the build prerenders) to the same path on the V1 docs with a 302, so adding a page here takes its URL over. The old `/v2` URLs redirect to `/docs` permanently (`src/lib/legacy.ts`). diff --git a/services/site/src/docs/script/build.ts b/services/site/src/docs/script/build.ts index 0ac02031b27..eda317b0394 100644 --- a/services/site/src/docs/script/build.ts +++ b/services/site/src/docs/script/build.ts @@ -34,8 +34,9 @@ const alias = { "~": path.join(root, "src") } const inputs = restyle ? { stylesheet: "src/docs/styles/docs.css", script: "src/docs/client.ts" } : { stylesheet: "src/docs/legacy/styles/global.css", script: "src/docs/legacy/client.ts" } -// llms.txt and the sitemap list the deployment's own URLs; pages keep production canonical links. -const site = new URL(mode === "dev" ? "https://dev.opencode.ai" : config.baseUrl) +// llms.txt and the sitemap list opencode.ai's URLs in every mode, as the pages' canonical links do: +// search engines must not take dev.opencode.ai's for the real ones (src/lib/noindex.ts). +const site = new URL(config.baseUrl) await rm(cache, { recursive: true, force: true }) diff --git a/services/site/src/lib/noindex.ts b/services/site/src/lib/noindex.ts new file mode 100644 index 00000000000..d8e8401f12f --- /dev/null +++ b/services/site/src/lib/noindex.ts @@ -0,0 +1,24 @@ +import type { createMiddleware } from "@solidjs/start/middleware" +import { Resource } from "#resource" + +// dev.opencode.ai runs every merge to dev, and search engines must never list it in place of +// opencode.ai. Every response there asks them not to index it: the Worker's (pages, Markdown, +// redirects, errors) through this middleware, and the static assets the Worker never sees through +// the rule script/build.ts adds to dev's _headers. robots.txt keeps allowing the crawl, or crawlers +// would never see the header and dev pages already in an index would stay there. + +type Middleware = ReturnType[number] + +/** The headers every response from a stage carries: none in production. */ +export function stageHeaders(stage: "dev" | "production"): Record { + return stage === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {} +} + +// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more. +export const noindexDev: Middleware = (event, next) => { + Object.entries(stageHeaders(Resource.App.stage)).forEach(([name, value]) => { + event.res.headers.set(name, value) + event.res.errHeaders.set(name, value) + }) + return next() +} diff --git a/services/site/src/middleware.ts b/services/site/src/middleware.ts index f8c831beec4..e7ba58cf223 100644 --- a/services/site/src/middleware.ts +++ b/services/site/src/middleware.ts @@ -24,13 +24,16 @@ import { mediaUrl } from "~/lib/media" import { legacyRedirect } from "~/lib/legacy" import { metaResponse } from "~/lib/meta" import { setSecurityHeaders } from "~/lib/security-headers" +import { noindexDev } from "~/lib/noindex" import { v1DocsOrigin } from "~/lib/docs" import { docsMiddleware } from "~/docs/middleware" -// The V2 docs (everything under /docs, docs/middleware.ts), then the page middleware, then the HTML -// cache headers and edge cache (lib/html-cache.ts), which see the request without its locale, then -// one stylesheet per page (lib/one-stylesheet.ts), inside the cache so cached pages carry it too. +// dev.opencode.ai's noindex header on every response (lib/noindex.ts), then the V2 docs (everything +// under /docs, docs/middleware.ts), then the page middleware, then the HTML cache headers and edge +// cache (lib/html-cache.ts), which see the request without its locale, then one stylesheet per page +// (lib/one-stylesheet.ts), inside the cache so cached pages carry it too. export default [ + noindexDev, ...docsMiddleware, ...createMiddleware({ async onRequest(event) { @@ -102,7 +105,10 @@ export default [ if (english || englishOnly || (slash && findPageMarkdown(path))) { // Pages lose a trailing slash; other apps' addresses (the docs) keep theirs. const target = findPageMarkdown(path) ? path : url.pathname - return new Response(null, { status: 301, headers: { Location: `${route(locale ?? "en", target)}${url.search}` } }) + return new Response(null, { + status: 301, + headers: { Location: `${route(locale ?? "en", target)}${url.search}` }, + }) } // One release as JSON, at a stable URL per version (for the apps' "What's new"). const release = path.match(/^\/changelog\/(v\d[\w.+-]*)\.json$/)?.[1] diff --git a/services/site/test/noindex.test.ts b/services/site/test/noindex.test.ts new file mode 100644 index 00000000000..be0f08fc166 --- /dev/null +++ b/services/site/test/noindex.test.ts @@ -0,0 +1,37 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { H3, HTTPError } from "nitro/h3" +import { noindexDev } from "../src/lib/noindex" + +// The middleware in an h3 app like the site's, answering the kinds of response its routes return. +const app = new H3() +app.use(noindexDev) +app.get("/page", () => "") +app.get("/markdown", () => new Response("# Page", { headers: { "Content-Type": "text/markdown" } })) +app.get("/moved", () => new Response(null, { status: 301, headers: { Location: "/page" } })) +app.get("/missing", () => new Response("Not found", { status: 404 })) +app.get("/broken", () => { + throw new HTTPError({ status: 500 }) +}) +const paths = ["/page", "/markdown", "/moved", "/missing", "/broken", "/unrouted"] + +const stage = process.env.STAGE +afterEach(() => { + if (stage === undefined) delete process.env.STAGE + else process.env.STAGE = stage +}) + +async function robots(stage: string) { + process.env.STAGE = stage + const responses = await Promise.all(paths.map((path) => app.fetch(new Request(`https://opencode.ai${path}`)))) + return responses.map((response) => response.headers.get("X-Robots-Tag")) +} + +describe("noindex on dev", () => { + test("every dev response asks search engines not to index it", async () => { + expect(await robots("dev")).toEqual(paths.map(() => "noindex, nofollow")) + }) + + test("production responses don't", async () => { + expect(await robots("production")).toEqual(paths.map(() => null)) + }) +})