fix(site): tell search engines not to index dev.opencode.ai (#62430)

This commit is contained in:
Adam
2026-11-06 15:16:39 -06:00
committed by GitHub
Unverified
parent cd47971fd6
commit 6e92017dfa
13 changed files with 146 additions and 19 deletions
+27
View File
@@ -0,0 +1,27 @@
import type { createMiddleware } from "@solidjs/start/middleware"
import { Resource } from "sst/resource"
// Only production may be indexed: dev.opencode.ai and the personal stages run unreleased builds, and
// search engines must never list them in place of opencode.ai. Every response there asks them not
// to index it: the Worker's through this middleware, and the static assets the Worker never sees
// through the rule vite.config.ts adds to those stages' _headers.
type Middleware = ReturnType<typeof createMiddleware>[number]
/**
* The headers every response from this stage carries: none in production, or outside SST (`vite dev` on its own),
* where no App is linked. SST links the App to the build as well as the Worker.
*/
export function stageHeaders(): Record<string, string> {
const stage = "App" in Resource ? Resource.App.stage : "production"
return stage === "production" ? {} : { "X-Robots-Tag": "noindex, nofollow" }
}
// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more.
export const noindexOutsideProduction: Middleware = (event, next) => {
Object.entries(stageHeaders()).forEach(([name, value]) => {
event.res.headers.set(name, value)
event.res.errHeaders.set(name, value)
})
return next()
}
+5 -1
View File
@@ -21,6 +21,7 @@ import {
route,
strip,
} from "./lib/language"
import { noindexOutsideProduction } from "./lib/noindex"
import { loadComparePage, loadHomePage, loadLabPage, loadModelPage } from "./lib/page-data"
import { canonicalModelEntry, catalogModelPath, findModelCatalogLab, loadModelCatalog } from "./routes/model-catalog"
import { statsPageCacheControl } from "./routes/stats-cache"
@@ -47,7 +48,7 @@ const contentTypes = {
text: "text/plain",
} as const satisfies Record<Format, string>
export default createMiddleware({
const pageMiddleware = createMiddleware({
onRequest: async (event) => {
const request = event.request
if (request.method !== "GET" && request.method !== "HEAD") return
@@ -82,6 +83,9 @@ export default createMiddleware({
},
})
// The noindex header outside production first (lib/noindex.ts), so every response carries it.
export default [noindexOutsideProduction, ...pageMiddleware]
// Unprefixed pages redirect to the visitor's language from their cookie or Accept-Language. /en/data stays put so
// English remains reachable for visitors whose browser prefers another language.
function localizedRedirect(request: Request, url: URL) {
+5 -1
View File
@@ -3,6 +3,7 @@ import path from "node:path"
import { solidStart } from "@solidjs/start/config"
import { nitro } from "nitro/vite"
import { defineConfig, type PluginOption } from "vite"
import { stageHeaders } from "./src/lib/noindex"
export default defineConfig({
base: "/data/",
@@ -25,9 +26,12 @@ export default defineConfig({
order: "post",
async handler() {
const source = path.join(import.meta.dirname, ".output/public/data/_headers")
// The stage's headers for the assets the Worker never sees (src/lib/noindex.ts).
const stage = Object.entries(stageHeaders()).map(([name, value]) => ` ${name}: ${value}\n`)
await writeFile(
path.join(import.meta.dirname, ".output/public/_headers"),
(await readFile(source, "utf8")).replace(/^\//gm, "/data/"),
(await readFile(source, "utf8")).replace(/^\//gm, "/data/") +
(stage.length ? `\n/data/*\n${stage.join("")}` : ""),
)
await rm(source)
},
+3 -1
View File
@@ -8,6 +8,8 @@ export default {
"Range, If-Range, If-Match, If-None-Match, If-Modified-Since, If-Unmodified-Since",
"Access-Control-Expose-Headers":
"Accept-Ranges, Content-Length, Content-Range, Content-Disposition, ETag, Last-Modified",
// Like everything on dev.opencode.ai, dev's files ask search engines not to index them.
...(env.STAGE === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {}),
}
if (!url.pathname.startsWith("/files/")) {
@@ -153,7 +155,7 @@ export default {
}
return response
},
} satisfies ExportedHandler<{ FILES: R2Bucket }>
} satisfies ExportedHandler<{ FILES: R2Bucket; STAGE: "dev" | "production" }>
function matches(value: string, etag: string, weak: boolean) {
return value.split(",").some((part) => {
+2
View File
@@ -8,11 +8,13 @@
"env": {
"dev": {
"name": "opencode-dev-files",
"vars": { "STAGE": "dev" },
"routes": [{ "pattern": "dev.opencode.ai/files/*", "zone_name": "opencode.ai" }],
"r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-dev-files" }],
},
"production": {
"name": "opencode-production-files",
"vars": { "STAGE": "production" },
"routes": [{ "pattern": "opencode.ai/files/*", "zone_name": "opencode.ai" }],
"r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-production-files" }],
},
+2 -1
View File
@@ -1,6 +1,7 @@
# Cache headers for the static files in public/ (Cloudflare Workers static assets; script/build.ts ships
# .output/public/_headers with the assets). Nitro appends its own /_build/assets/* rule (hashed
# build files, immutable). Rules must not overlap: Cloudflare joins repeated headers with a comma.
# build files, immutable), and dev builds a /* X-Robots-Tag rule (src/lib/noindex.ts). Rules must not
# overlap: Cloudflare joins repeated headers with a comma.
#
# Only files whose name changes with their content are immutable. The media below keep their URL
# when re-rendered in place, so browsers and the edge may keep them for a day and serve a stale
+5 -1
View File
@@ -4,7 +4,7 @@
// .output/public as its assets. cf's own build step can't detect this project, so it deploys prebuilt.
//
// bun script/build.ts [--mode dev|production]
import { cp, mkdir, rm } from "node:fs/promises"
import { appendFile, cp, mkdir, rm } from "node:fs/promises"
import path from "node:path"
import {
cleanBuildOutputDir,
@@ -15,6 +15,7 @@ import {
} from "@cloudflare/build-output-utils"
import { resolveAndParseConfig } from "@cloudflare/config"
import { mediaFile } from "../src/lib/media"
import { stageHeaders } from "../src/lib/noindex"
const root = path.resolve(import.meta.dir, "..")
const modeIndex = process.argv.indexOf("--mode")
@@ -67,6 +68,9 @@ const fixed = ["config.json", "tui.json", "openapi.json", "v2/cli.json", "docs/o
(file) => !shipped.includes(file),
)
if (fixed.length) throw new Error(`fixed-URL files missing from Worker assets: ${fixed.join(", ")}`)
// The stage's headers for the assets the Worker never sees; its own responses get them from src/lib/noindex.ts.
const headers = Object.entries(stageHeaders(mode)).map(([name, value]) => ` ${name}: ${value}\n`)
if (headers.length) await appendFile(path.join(assets, "_headers"), `\n/*\n${headers.join("")}`)
await writeAssets({ root, sourceDirectory: assets })
const bundle = getWorkerBundleDir(root)
+22 -7
View File
@@ -13,11 +13,14 @@
import path from "node:path"
import { readBuildOutput } from "@cloudflare/build-output-utils"
import { Miniflare, Response } from "miniflare"
import { stageHeaders } from "../src/lib/noindex"
const root = path.resolve(import.meta.dir, "..")
// Pages with a status must answer exactly that, others anything but a 5xx; a redirect must go to its
// location. The docs page is asked for as a browser asks and, at a second URL, as an agent asks (its
// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect.
// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect. The
// last three are files the Worker serves, or the assets layer before it (_headers). Every answer
// carries the stage's headers (src/lib/noindex.ts): dev's noindex, none in production.
const pages: { path: string; status?: number; accept?: string; location?: string }[] = [
{ path: "/", status: 200 },
{ path: "/ja", status: 200 },
@@ -27,6 +30,9 @@ const pages: { path: string; status?: number; accept?: string; location?: string
{ path: "/docs/config/", status: 200, accept: "text/markdown" },
{ path: "/v2/docs/config/", status: 301, location: "/docs/config/" },
{ path: "/docs/ja/", status: 302, location: "/v1/docs/ja/" },
{ path: "/robots.txt", status: 200 },
{ path: "/docs/llms.txt", status: 200 },
{ path: "/favicon.svg", status: 200 },
]
const started = performance.now()
@@ -52,6 +58,9 @@ const env = Object.fromEntries(
}),
)
const stage = env.STAGE?.type === "text" && env.STAGE.value === "production" ? "production" : "dev"
const robots = new Headers(stageHeaders(stage)).get("x-robots-tag") ?? undefined
const errors: string[] = []
let offline = 0
const miniflare = new Miniflare({
@@ -92,10 +101,14 @@ const results = await Promise.resolve()
})
.then(async (response) => {
await response.arrayBuffer()
return { status: response.status, location: response.headers.get("location") ?? undefined }
return {
status: response.status,
location: response.headers.get("location") ?? undefined,
robots: response.headers.get("x-robots-tag") ?? undefined,
}
})
.catch(() => ({ status: 0, location: undefined }))
answers.push({ ...page, actual: actual.status, actualLocation: actual.location })
.catch(() => ({ status: 0, location: undefined, robots: undefined }))
answers.push({ ...page, actual: actual.status, actualLocation: actual.location, actualRobots: actual.robots })
}
return answers
})
@@ -110,12 +123,14 @@ const failures = results
.filter(
(result) =>
(result.status ? result.actual !== result.status : result.actual === 0 || result.actual >= 500) ||
result.actualLocation !== result.location,
result.actualLocation !== result.location ||
result.actualRobots !== robots,
)
.map(
(result) =>
`${result.path} answered ${result.actual || "nothing"}${result.status ? `, not ${result.status}` : ""}` +
(result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : ""),
(result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : "") +
(result.actualRobots !== robots ? ` with X-Robots-Tag ${result.actualRobots ?? "absent"} in ${stage}` : ""),
)
if (failures.length || errors.length) {
console.error(`check-worker-start: FAILED\n${[...errors, ...failures].map((line) => ` ${line}`).join("\n")}`)
@@ -123,5 +138,5 @@ if (failures.length || errors.length) {
}
const answered = results.map((result) => `${result.path} ${result.actual}`).join(", ")
console.log(
`check-worker-start: ${config.name} starts in workerd; ${answered}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`,
`check-worker-start: ${config.name} starts in workerd; ${answered}; X-Robots-Tag ${robots ?? "absent"}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`,
)
+1 -1
View File
@@ -28,7 +28,7 @@
## Build and serving
- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `--mode dev` makes `llms.txt` and the sitemap list dev.opencode.ai.
- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `llms.txt` and the sitemap list opencode.ai's URLs in every mode, so dev never advertises its own.
- Pull requests that touch the site run the same build on its own (`.github/workflows/check-docs.yml`): `bun run generate`, then `bun src/docs/script/build.ts --mode dev --out <dir>`.
- The Worker never renders docs. `middleware.ts` serves the prerendered files: docs URLs reach it first (`runWorkerFirst` in `cloudflare.config.ts`), so a page's URL answers `Accept: text/markdown` with its twin. `vite dev` has no prerendered files, so the development server renders on request.
- `/docs` still receives the V1 docs' old links, which had more pages and other languages (`/docs/ja/...`). `middleware.ts` redirects any `/docs` path that isn't a page here (`virtual:docs-pages`, the same list the build prerenders) to the same path on the V1 docs with a 302, so adding a page here takes its URL over. The old `/v2` URLs redirect to `/docs` permanently (`src/lib/legacy.ts`).
+3 -2
View File
@@ -34,8 +34,9 @@ const alias = { "~": path.join(root, "src") }
const inputs = restyle
? { stylesheet: "src/docs/styles/docs.css", script: "src/docs/client.ts" }
: { stylesheet: "src/docs/legacy/styles/global.css", script: "src/docs/legacy/client.ts" }
// llms.txt and the sitemap list the deployment's own URLs; pages keep production canonical links.
const site = new URL(mode === "dev" ? "https://dev.opencode.ai" : config.baseUrl)
// llms.txt and the sitemap list opencode.ai's URLs in every mode, as the pages' canonical links do:
// search engines must not take dev.opencode.ai's for the real ones (src/lib/noindex.ts).
const site = new URL(config.baseUrl)
await rm(cache, { recursive: true, force: true })
+24
View File
@@ -0,0 +1,24 @@
import type { createMiddleware } from "@solidjs/start/middleware"
import { Resource } from "#resource"
// dev.opencode.ai runs every merge to dev, and search engines must never list it in place of
// opencode.ai. Every response there asks them not to index it: the Worker's (pages, Markdown,
// redirects, errors) through this middleware, and the static assets the Worker never sees through
// the rule script/build.ts adds to dev's _headers. robots.txt keeps allowing the crawl, or crawlers
// would never see the header and dev pages already in an index would stay there.
type Middleware = ReturnType<typeof createMiddleware>[number]
/** The headers every response from a stage carries: none in production. */
export function stageHeaders(stage: "dev" | "production"): Record<string, string> {
return stage === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {}
}
// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more.
export const noindexDev: Middleware = (event, next) => {
Object.entries(stageHeaders(Resource.App.stage)).forEach(([name, value]) => {
event.res.headers.set(name, value)
event.res.errHeaders.set(name, value)
})
return next()
}
+10 -4
View File
@@ -24,13 +24,16 @@ import { mediaUrl } from "~/lib/media"
import { legacyRedirect } from "~/lib/legacy"
import { metaResponse } from "~/lib/meta"
import { setSecurityHeaders } from "~/lib/security-headers"
import { noindexDev } from "~/lib/noindex"
import { v1DocsOrigin } from "~/lib/docs"
import { docsMiddleware } from "~/docs/middleware"
// The V2 docs (everything under /docs, docs/middleware.ts), then the page middleware, then the HTML
// cache headers and edge cache (lib/html-cache.ts), which see the request without its locale, then
// one stylesheet per page (lib/one-stylesheet.ts), inside the cache so cached pages carry it too.
// dev.opencode.ai's noindex header on every response (lib/noindex.ts), then the V2 docs (everything
// under /docs, docs/middleware.ts), then the page middleware, then the HTML cache headers and edge
// cache (lib/html-cache.ts), which see the request without its locale, then one stylesheet per page
// (lib/one-stylesheet.ts), inside the cache so cached pages carry it too.
export default [
noindexDev,
...docsMiddleware,
...createMiddleware({
async onRequest(event) {
@@ -102,7 +105,10 @@ export default [
if (english || englishOnly || (slash && findPageMarkdown(path))) {
// Pages lose a trailing slash; other apps' addresses (the docs) keep theirs.
const target = findPageMarkdown(path) ? path : url.pathname
return new Response(null, { status: 301, headers: { Location: `${route(locale ?? "en", target)}${url.search}` } })
return new Response(null, {
status: 301,
headers: { Location: `${route(locale ?? "en", target)}${url.search}` },
})
}
// One release as JSON, at a stable URL per version (for the apps' "What's new").
const release = path.match(/^\/changelog\/(v\d[\w.+-]*)\.json$/)?.[1]
+37
View File
@@ -0,0 +1,37 @@
import { afterEach, describe, expect, test } from "bun:test"
import { H3, HTTPError } from "nitro/h3"
import { noindexDev } from "../src/lib/noindex"
// The middleware in an h3 app like the site's, answering the kinds of response its routes return.
const app = new H3()
app.use(noindexDev)
app.get("/page", () => "<!doctype html>")
app.get("/markdown", () => new Response("# Page", { headers: { "Content-Type": "text/markdown" } }))
app.get("/moved", () => new Response(null, { status: 301, headers: { Location: "/page" } }))
app.get("/missing", () => new Response("Not found", { status: 404 }))
app.get("/broken", () => {
throw new HTTPError({ status: 500 })
})
const paths = ["/page", "/markdown", "/moved", "/missing", "/broken", "/unrouted"]
const stage = process.env.STAGE
afterEach(() => {
if (stage === undefined) delete process.env.STAGE
else process.env.STAGE = stage
})
async function robots(stage: string) {
process.env.STAGE = stage
const responses = await Promise.all(paths.map((path) => app.fetch(new Request(`https://opencode.ai${path}`))))
return responses.map((response) => response.headers.get("X-Robots-Tag"))
}
describe("noindex on dev", () => {
test("every dev response asks search engines not to index it", async () => {
expect(await robots("dev")).toEqual(paths.map(() => "noindex, nofollow"))
})
test("production responses don't", async () => {
expect(await robots("production")).toEqual(paths.map(() => null))
})
})