mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-21 04:12:08 +03:00
fix(site): tell search engines not to index dev.opencode.ai (#62430)
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
import type { createMiddleware } from "@solidjs/start/middleware"
|
||||
import { Resource } from "sst/resource"
|
||||
|
||||
// Only production may be indexed: dev.opencode.ai and the personal stages run unreleased builds, and
|
||||
// search engines must never list them in place of opencode.ai. Every response there asks them not
|
||||
// to index it: the Worker's through this middleware, and the static assets the Worker never sees
|
||||
// through the rule vite.config.ts adds to those stages' _headers.
|
||||
|
||||
type Middleware = ReturnType<typeof createMiddleware>[number]
|
||||
|
||||
/**
|
||||
* The headers every response from this stage carries: none in production, or outside SST (`vite dev` on its own),
|
||||
* where no App is linked. SST links the App to the build as well as the Worker.
|
||||
*/
|
||||
export function stageHeaders(): Record<string, string> {
|
||||
const stage = "App" in Resource ? Resource.App.stage : "production"
|
||||
return stage === "production" ? {} : { "X-Robots-Tag": "noindex, nofollow" }
|
||||
}
|
||||
|
||||
// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more.
|
||||
export const noindexOutsideProduction: Middleware = (event, next) => {
|
||||
Object.entries(stageHeaders()).forEach(([name, value]) => {
|
||||
event.res.headers.set(name, value)
|
||||
event.res.errHeaders.set(name, value)
|
||||
})
|
||||
return next()
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
route,
|
||||
strip,
|
||||
} from "./lib/language"
|
||||
import { noindexOutsideProduction } from "./lib/noindex"
|
||||
import { loadComparePage, loadHomePage, loadLabPage, loadModelPage } from "./lib/page-data"
|
||||
import { canonicalModelEntry, catalogModelPath, findModelCatalogLab, loadModelCatalog } from "./routes/model-catalog"
|
||||
import { statsPageCacheControl } from "./routes/stats-cache"
|
||||
@@ -47,7 +48,7 @@ const contentTypes = {
|
||||
text: "text/plain",
|
||||
} as const satisfies Record<Format, string>
|
||||
|
||||
export default createMiddleware({
|
||||
const pageMiddleware = createMiddleware({
|
||||
onRequest: async (event) => {
|
||||
const request = event.request
|
||||
if (request.method !== "GET" && request.method !== "HEAD") return
|
||||
@@ -82,6 +83,9 @@ export default createMiddleware({
|
||||
},
|
||||
})
|
||||
|
||||
// The noindex header outside production first (lib/noindex.ts), so every response carries it.
|
||||
export default [noindexOutsideProduction, ...pageMiddleware]
|
||||
|
||||
// Unprefixed pages redirect to the visitor's language from their cookie or Accept-Language. /en/data stays put so
|
||||
// English remains reachable for visitors whose browser prefers another language.
|
||||
function localizedRedirect(request: Request, url: URL) {
|
||||
|
||||
@@ -3,6 +3,7 @@ import path from "node:path"
|
||||
import { solidStart } from "@solidjs/start/config"
|
||||
import { nitro } from "nitro/vite"
|
||||
import { defineConfig, type PluginOption } from "vite"
|
||||
import { stageHeaders } from "./src/lib/noindex"
|
||||
|
||||
export default defineConfig({
|
||||
base: "/data/",
|
||||
@@ -25,9 +26,12 @@ export default defineConfig({
|
||||
order: "post",
|
||||
async handler() {
|
||||
const source = path.join(import.meta.dirname, ".output/public/data/_headers")
|
||||
// The stage's headers for the assets the Worker never sees (src/lib/noindex.ts).
|
||||
const stage = Object.entries(stageHeaders()).map(([name, value]) => ` ${name}: ${value}\n`)
|
||||
await writeFile(
|
||||
path.join(import.meta.dirname, ".output/public/_headers"),
|
||||
(await readFile(source, "utf8")).replace(/^\//gm, "/data/"),
|
||||
(await readFile(source, "utf8")).replace(/^\//gm, "/data/") +
|
||||
(stage.length ? `\n/data/*\n${stage.join("")}` : ""),
|
||||
)
|
||||
await rm(source)
|
||||
},
|
||||
|
||||
@@ -8,6 +8,8 @@ export default {
|
||||
"Range, If-Range, If-Match, If-None-Match, If-Modified-Since, If-Unmodified-Since",
|
||||
"Access-Control-Expose-Headers":
|
||||
"Accept-Ranges, Content-Length, Content-Range, Content-Disposition, ETag, Last-Modified",
|
||||
// Like everything on dev.opencode.ai, dev's files ask search engines not to index them.
|
||||
...(env.STAGE === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {}),
|
||||
}
|
||||
|
||||
if (!url.pathname.startsWith("/files/")) {
|
||||
@@ -153,7 +155,7 @@ export default {
|
||||
}
|
||||
return response
|
||||
},
|
||||
} satisfies ExportedHandler<{ FILES: R2Bucket }>
|
||||
} satisfies ExportedHandler<{ FILES: R2Bucket; STAGE: "dev" | "production" }>
|
||||
|
||||
function matches(value: string, etag: string, weak: boolean) {
|
||||
return value.split(",").some((part) => {
|
||||
|
||||
@@ -8,11 +8,13 @@
|
||||
"env": {
|
||||
"dev": {
|
||||
"name": "opencode-dev-files",
|
||||
"vars": { "STAGE": "dev" },
|
||||
"routes": [{ "pattern": "dev.opencode.ai/files/*", "zone_name": "opencode.ai" }],
|
||||
"r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-dev-files" }],
|
||||
},
|
||||
"production": {
|
||||
"name": "opencode-production-files",
|
||||
"vars": { "STAGE": "production" },
|
||||
"routes": [{ "pattern": "opencode.ai/files/*", "zone_name": "opencode.ai" }],
|
||||
"r2_buckets": [{ "binding": "FILES", "bucket_name": "opencode-production-files" }],
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Cache headers for the static files in public/ (Cloudflare Workers static assets; script/build.ts ships
|
||||
# .output/public/_headers with the assets). Nitro appends its own /_build/assets/* rule (hashed
|
||||
# build files, immutable). Rules must not overlap: Cloudflare joins repeated headers with a comma.
|
||||
# build files, immutable), and dev builds a /* X-Robots-Tag rule (src/lib/noindex.ts). Rules must not
|
||||
# overlap: Cloudflare joins repeated headers with a comma.
|
||||
#
|
||||
# Only files whose name changes with their content are immutable. The media below keep their URL
|
||||
# when re-rendered in place, so browsers and the edge may keep them for a day and serve a stale
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// .output/public as its assets. cf's own build step can't detect this project, so it deploys prebuilt.
|
||||
//
|
||||
// bun script/build.ts [--mode dev|production]
|
||||
import { cp, mkdir, rm } from "node:fs/promises"
|
||||
import { appendFile, cp, mkdir, rm } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import {
|
||||
cleanBuildOutputDir,
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "@cloudflare/build-output-utils"
|
||||
import { resolveAndParseConfig } from "@cloudflare/config"
|
||||
import { mediaFile } from "../src/lib/media"
|
||||
import { stageHeaders } from "../src/lib/noindex"
|
||||
|
||||
const root = path.resolve(import.meta.dir, "..")
|
||||
const modeIndex = process.argv.indexOf("--mode")
|
||||
@@ -67,6 +68,9 @@ const fixed = ["config.json", "tui.json", "openapi.json", "v2/cli.json", "docs/o
|
||||
(file) => !shipped.includes(file),
|
||||
)
|
||||
if (fixed.length) throw new Error(`fixed-URL files missing from Worker assets: ${fixed.join(", ")}`)
|
||||
// The stage's headers for the assets the Worker never sees; its own responses get them from src/lib/noindex.ts.
|
||||
const headers = Object.entries(stageHeaders(mode)).map(([name, value]) => ` ${name}: ${value}\n`)
|
||||
if (headers.length) await appendFile(path.join(assets, "_headers"), `\n/*\n${headers.join("")}`)
|
||||
await writeAssets({ root, sourceDirectory: assets })
|
||||
|
||||
const bundle = getWorkerBundleDir(root)
|
||||
|
||||
@@ -13,11 +13,14 @@
|
||||
import path from "node:path"
|
||||
import { readBuildOutput } from "@cloudflare/build-output-utils"
|
||||
import { Miniflare, Response } from "miniflare"
|
||||
import { stageHeaders } from "../src/lib/noindex"
|
||||
|
||||
const root = path.resolve(import.meta.dir, "..")
|
||||
// Pages with a status must answer exactly that, others anything but a 5xx; a redirect must go to its
|
||||
// location. The docs page is asked for as a browser asks and, at a second URL, as an agent asks (its
|
||||
// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect.
|
||||
// Markdown twin, src/docs/middleware.ts). The docs' old /v2 address and a V1 docs path redirect. The
|
||||
// last three are files the Worker serves, or the assets layer before it (_headers). Every answer
|
||||
// carries the stage's headers (src/lib/noindex.ts): dev's noindex, none in production.
|
||||
const pages: { path: string; status?: number; accept?: string; location?: string }[] = [
|
||||
{ path: "/", status: 200 },
|
||||
{ path: "/ja", status: 200 },
|
||||
@@ -27,6 +30,9 @@ const pages: { path: string; status?: number; accept?: string; location?: string
|
||||
{ path: "/docs/config/", status: 200, accept: "text/markdown" },
|
||||
{ path: "/v2/docs/config/", status: 301, location: "/docs/config/" },
|
||||
{ path: "/docs/ja/", status: 302, location: "/v1/docs/ja/" },
|
||||
{ path: "/robots.txt", status: 200 },
|
||||
{ path: "/docs/llms.txt", status: 200 },
|
||||
{ path: "/favicon.svg", status: 200 },
|
||||
]
|
||||
|
||||
const started = performance.now()
|
||||
@@ -52,6 +58,9 @@ const env = Object.fromEntries(
|
||||
}),
|
||||
)
|
||||
|
||||
const stage = env.STAGE?.type === "text" && env.STAGE.value === "production" ? "production" : "dev"
|
||||
const robots = new Headers(stageHeaders(stage)).get("x-robots-tag") ?? undefined
|
||||
|
||||
const errors: string[] = []
|
||||
let offline = 0
|
||||
const miniflare = new Miniflare({
|
||||
@@ -92,10 +101,14 @@ const results = await Promise.resolve()
|
||||
})
|
||||
.then(async (response) => {
|
||||
await response.arrayBuffer()
|
||||
return { status: response.status, location: response.headers.get("location") ?? undefined }
|
||||
return {
|
||||
status: response.status,
|
||||
location: response.headers.get("location") ?? undefined,
|
||||
robots: response.headers.get("x-robots-tag") ?? undefined,
|
||||
}
|
||||
})
|
||||
.catch(() => ({ status: 0, location: undefined }))
|
||||
answers.push({ ...page, actual: actual.status, actualLocation: actual.location })
|
||||
.catch(() => ({ status: 0, location: undefined, robots: undefined }))
|
||||
answers.push({ ...page, actual: actual.status, actualLocation: actual.location, actualRobots: actual.robots })
|
||||
}
|
||||
return answers
|
||||
})
|
||||
@@ -110,12 +123,14 @@ const failures = results
|
||||
.filter(
|
||||
(result) =>
|
||||
(result.status ? result.actual !== result.status : result.actual === 0 || result.actual >= 500) ||
|
||||
result.actualLocation !== result.location,
|
||||
result.actualLocation !== result.location ||
|
||||
result.actualRobots !== robots,
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.path} answered ${result.actual || "nothing"}${result.status ? `, not ${result.status}` : ""}` +
|
||||
(result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : ""),
|
||||
(result.actualLocation !== result.location ? ` to ${result.actualLocation}, not ${result.location}` : "") +
|
||||
(result.actualRobots !== robots ? ` with X-Robots-Tag ${result.actualRobots ?? "absent"} in ${stage}` : ""),
|
||||
)
|
||||
if (failures.length || errors.length) {
|
||||
console.error(`check-worker-start: FAILED\n${[...errors, ...failures].map((line) => ` ${line}`).join("\n")}`)
|
||||
@@ -123,5 +138,5 @@ if (failures.length || errors.length) {
|
||||
}
|
||||
const answered = results.map((result) => `${result.path} ${result.actual}`).join(", ")
|
||||
console.log(
|
||||
`check-worker-start: ${config.name} starts in workerd; ${answered}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`,
|
||||
`check-worker-start: ${config.name} starts in workerd; ${answered}; X-Robots-Tag ${robots ?? "absent"}; ${offline} outbound calls answered offline (${Math.round(performance.now() - started)} ms)`,
|
||||
)
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
|
||||
## Build and serving
|
||||
|
||||
- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `--mode dev` makes `llms.txt` and the sitemap list dev.opencode.ai.
|
||||
- `script/build.ts` runs after the site's `vite build` (from `script/build.ts` at the site root): it builds the docs' stylesheet and script, prerenders every page, every content page's Markdown twin (`/docs/config.md`; the intro's is `/docs/index.md`), `llms.txt` and the sitemap into `.output/public`, builds the Pagefind index, and checks every link, that nothing loads from outside `/docs` and that no page takes a reserved file name. `llms.txt` and the sitemap list opencode.ai's URLs in every mode, so dev never advertises its own.
|
||||
- Pull requests that touch the site run the same build on its own (`.github/workflows/check-docs.yml`): `bun run generate`, then `bun src/docs/script/build.ts --mode dev --out <dir>`.
|
||||
- The Worker never renders docs. `middleware.ts` serves the prerendered files: docs URLs reach it first (`runWorkerFirst` in `cloudflare.config.ts`), so a page's URL answers `Accept: text/markdown` with its twin. `vite dev` has no prerendered files, so the development server renders on request.
|
||||
- `/docs` still receives the V1 docs' old links, which had more pages and other languages (`/docs/ja/...`). `middleware.ts` redirects any `/docs` path that isn't a page here (`virtual:docs-pages`, the same list the build prerenders) to the same path on the V1 docs with a 302, so adding a page here takes its URL over. The old `/v2` URLs redirect to `/docs` permanently (`src/lib/legacy.ts`).
|
||||
|
||||
@@ -34,8 +34,9 @@ const alias = { "~": path.join(root, "src") }
|
||||
const inputs = restyle
|
||||
? { stylesheet: "src/docs/styles/docs.css", script: "src/docs/client.ts" }
|
||||
: { stylesheet: "src/docs/legacy/styles/global.css", script: "src/docs/legacy/client.ts" }
|
||||
// llms.txt and the sitemap list the deployment's own URLs; pages keep production canonical links.
|
||||
const site = new URL(mode === "dev" ? "https://dev.opencode.ai" : config.baseUrl)
|
||||
// llms.txt and the sitemap list opencode.ai's URLs in every mode, as the pages' canonical links do:
|
||||
// search engines must not take dev.opencode.ai's for the real ones (src/lib/noindex.ts).
|
||||
const site = new URL(config.baseUrl)
|
||||
|
||||
await rm(cache, { recursive: true, force: true })
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { createMiddleware } from "@solidjs/start/middleware"
|
||||
import { Resource } from "#resource"
|
||||
|
||||
// dev.opencode.ai runs every merge to dev, and search engines must never list it in place of
|
||||
// opencode.ai. Every response there asks them not to index it: the Worker's (pages, Markdown,
|
||||
// redirects, errors) through this middleware, and the static assets the Worker never sees through
|
||||
// the rule script/build.ts adds to dev's _headers. robots.txt keeps allowing the crawl, or crawlers
|
||||
// would never see the header and dev pages already in an index would stay there.
|
||||
|
||||
type Middleware = ReturnType<typeof createMiddleware>[number]
|
||||
|
||||
/** The headers every response from a stage carries: none in production. */
|
||||
export function stageHeaders(stage: "dev" | "production"): Record<string, string> {
|
||||
return stage === "dev" ? { "X-Robots-Tag": "noindex, nofollow" } : {}
|
||||
}
|
||||
|
||||
// h3 adds the prepared headers to whatever a route returns, and the error headers to a status of 400 or more.
|
||||
export const noindexDev: Middleware = (event, next) => {
|
||||
Object.entries(stageHeaders(Resource.App.stage)).forEach(([name, value]) => {
|
||||
event.res.headers.set(name, value)
|
||||
event.res.errHeaders.set(name, value)
|
||||
})
|
||||
return next()
|
||||
}
|
||||
@@ -24,13 +24,16 @@ import { mediaUrl } from "~/lib/media"
|
||||
import { legacyRedirect } from "~/lib/legacy"
|
||||
import { metaResponse } from "~/lib/meta"
|
||||
import { setSecurityHeaders } from "~/lib/security-headers"
|
||||
import { noindexDev } from "~/lib/noindex"
|
||||
import { v1DocsOrigin } from "~/lib/docs"
|
||||
import { docsMiddleware } from "~/docs/middleware"
|
||||
|
||||
// The V2 docs (everything under /docs, docs/middleware.ts), then the page middleware, then the HTML
|
||||
// cache headers and edge cache (lib/html-cache.ts), which see the request without its locale, then
|
||||
// one stylesheet per page (lib/one-stylesheet.ts), inside the cache so cached pages carry it too.
|
||||
// dev.opencode.ai's noindex header on every response (lib/noindex.ts), then the V2 docs (everything
|
||||
// under /docs, docs/middleware.ts), then the page middleware, then the HTML cache headers and edge
|
||||
// cache (lib/html-cache.ts), which see the request without its locale, then one stylesheet per page
|
||||
// (lib/one-stylesheet.ts), inside the cache so cached pages carry it too.
|
||||
export default [
|
||||
noindexDev,
|
||||
...docsMiddleware,
|
||||
...createMiddleware({
|
||||
async onRequest(event) {
|
||||
@@ -102,7 +105,10 @@ export default [
|
||||
if (english || englishOnly || (slash && findPageMarkdown(path))) {
|
||||
// Pages lose a trailing slash; other apps' addresses (the docs) keep theirs.
|
||||
const target = findPageMarkdown(path) ? path : url.pathname
|
||||
return new Response(null, { status: 301, headers: { Location: `${route(locale ?? "en", target)}${url.search}` } })
|
||||
return new Response(null, {
|
||||
status: 301,
|
||||
headers: { Location: `${route(locale ?? "en", target)}${url.search}` },
|
||||
})
|
||||
}
|
||||
// One release as JSON, at a stable URL per version (for the apps' "What's new").
|
||||
const release = path.match(/^\/changelog\/(v\d[\w.+-]*)\.json$/)?.[1]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test"
|
||||
import { H3, HTTPError } from "nitro/h3"
|
||||
import { noindexDev } from "../src/lib/noindex"
|
||||
|
||||
// The middleware in an h3 app like the site's, answering the kinds of response its routes return.
|
||||
const app = new H3()
|
||||
app.use(noindexDev)
|
||||
app.get("/page", () => "<!doctype html>")
|
||||
app.get("/markdown", () => new Response("# Page", { headers: { "Content-Type": "text/markdown" } }))
|
||||
app.get("/moved", () => new Response(null, { status: 301, headers: { Location: "/page" } }))
|
||||
app.get("/missing", () => new Response("Not found", { status: 404 }))
|
||||
app.get("/broken", () => {
|
||||
throw new HTTPError({ status: 500 })
|
||||
})
|
||||
const paths = ["/page", "/markdown", "/moved", "/missing", "/broken", "/unrouted"]
|
||||
|
||||
const stage = process.env.STAGE
|
||||
afterEach(() => {
|
||||
if (stage === undefined) delete process.env.STAGE
|
||||
else process.env.STAGE = stage
|
||||
})
|
||||
|
||||
async function robots(stage: string) {
|
||||
process.env.STAGE = stage
|
||||
const responses = await Promise.all(paths.map((path) => app.fetch(new Request(`https://opencode.ai${path}`))))
|
||||
return responses.map((response) => response.headers.get("X-Robots-Tag"))
|
||||
}
|
||||
|
||||
describe("noindex on dev", () => {
|
||||
test("every dev response asks search engines not to index it", async () => {
|
||||
expect(await robots("dev")).toEqual(paths.map(() => "noindex, nofollow"))
|
||||
})
|
||||
|
||||
test("production responses don't", async () => {
|
||||
expect(await robots("production")).toEqual(paths.map(() => null))
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user