#!/usr/bin/env bun // Reads the git history of each legal document's source files and writes the versions whose text // changed to src/lib/legal/history.ts, which the History list, "What changed" and the legal feed read. // Commits that only move or reformat the text (no sentence added or removed) are left out. // Needs the full history (not a shallow clone). Run from packages/console/app after a legal change // is committed: bun script/legal-history.ts import { createHash } from "crypto" import { dirname, join } from "path" import { fileURLToPath } from "url" const app = join(dirname(fileURLToPath(import.meta.url)), "..") const repo = "https://github.com/anomalyco/opencode" // Every file that has held a document's text, newest first. The text moved from the route files to // src/lib/legal, so a version is the text of all of them at that commit. const documents = { "/legal/privacy-policy": [ "packages/console/app/src/lib/legal/privacy-policy.ts", "packages/console/app/src/routes/legal/privacy-policy/index.tsx", ], "/legal/terms-of-service": [ "packages/console/app/src/lib/legal/terms-of-service.ts", "packages/console/app/src/routes/legal/terms-of-service/index.tsx", ], } // At most this many changed sentences are kept per version; the diff link has the rest. const limit = 12 if ((await Bun.$`git rev-parse --is-shallow-repository`.cwd(app).text()).trim() === "true") throw new Error("The history needs a full clone: run git fetch --unshallow first.") const root = (await Bun.$`git rev-parse --show-toplevel`.cwd(app).text()).trim() const history = Object.fromEntries( await Promise.all( Object.entries(documents).map(async ([path, files]) => { const log = (await Bun.$`git log --format=%H%x09%aI%x09%s -- ${files}`.cwd(root).text()) .trim() .split("\n") .filter(Boolean) .map((line) => { const [sha, date, ...subject] = line.split("\t") return { sha, date: date.slice(0, 10), subject: subject.join("\t") } }) .reverse() const texts = await Promise.all(log.map((commit) => source(commit.sha, files))) const versions = log.flatMap((commit, index) => { const before = index === 0 ? [] : sentences(texts[index - 1]) const after = sentences(texts[index]) const added = changed(after, before) const removed = changed(before, after) if (index > 0 && added.length === 0 && removed.length === 0) return [] const file = files.find((file) => texts[index].files.includes(file)) ?? files[0] return [ { sha: commit.sha, date: commit.date, subject: commit.subject, effective: effective(texts[index].text), first: index === 0, added: index === 0 ? [] : added.slice(0, limit), removed: index === 0 ? [] : removed.slice(0, limit), addedCount: added.length, removedCount: removed.length, source: `${repo}/blob/${commit.sha}/${file}`, diff: `${repo}/commit/${commit.sha}#diff-${createHash("sha256").update(file).digest("hex")}`, }, ] }) return [path, versions.reverse()] as const }), ), ) await Bun.write( join(app, "src/lib/legal/history.ts"), [ "// Generated by script/legal-history.ts from the git history of the legal documents. Do not edit.", "", `export const legalHistory = ${JSON.stringify(history, null, 2)} satisfies Record`, "", "/** A version of a legal document: a commit that changed its text, newest first. */", "export type LegalVersion = {", " sha: string", " /** The commit date, YYYY-MM-DD. */", " date: string", " subject: string", " /** The effective date the document stated in this version. */", " effective: string | null", " /** The version that first published the document. */", " first: boolean", " /** Sentences this version added and removed, up to " + limit + " of each. */", " added: string[]", " removed: string[]", " addedCount: number", " removedCount: number", " /** The document's source file at this version, on GitHub. */", " source: string", " /** The commit's diff of that file, on GitHub. */", " diff: string", "}", "", ].join("\n"), ) await Bun.$`bunx prettier --write src/lib/legal/history.ts`.cwd(app).quiet() console.log( Object.entries(history) .map(([path, versions]) => `${path}: ${versions.length} versions`) .join("\n"), ) /** The document's source files at a commit, joined, and which of them existed. */ async function source(sha: string, files: string[]) { const contents = await Promise.all( files.map(async (file) => { const result = await Bun.$`git show ${sha}:${file}`.cwd(root).quiet().nothrow() return result.exitCode === 0 ? { file, text: result.stdout.toString() } : undefined }), ) const found = contents.filter((content) => content !== undefined) return { files: found.map((content) => content.file), text: found.map((content) => content.text).join("\n"), found } } /** * The document's prose as sentences, split at block elements (paragraphs, headings, list items, * table cells) and then at sentence ends. The same text in the old JSX and the newer node-tree * source gives the same sentences, so moving the text between them is not a change. */ function sentences(input: { found: { file: string; text: string }[] }) { return input.found.flatMap((content) => fileSentences(content.text, content.file.endsWith(".tsx"))) } /** * One source file's prose. Old route files are JSX: the text between tags, from the effective date * on (the page title and meta tags come before it). Newer files are a node tree: every string that * is a node's child (not a tag name or an attribute value), plus the effective date. */ function fileSentences(text: string, jsx: boolean) { const stripped = text .replace(/^import .*$/gm, "") .replace(/\{\/\*[\s\S]*?\*\/\}/g, "") .replace(/\/\*[\s\S]*?\*\//g, "") .replace(/^\s*\/\/.*$/gm, "") const start = jsx ? stripped.indexOf("effective-date") : stripped.indexOf("body:") const code = start === -1 ? stripped : stripped.slice(start) const blocks = "p|li|h[1-6]|td|th|tr|ul|ol|div|table|thead|tbody|header|section|article" const runs = jsx ? jsxRuns(code, blocks) : treeRuns(code, blocks) const date = jsx ? [] : [text.match(/effective:\s*"([^"]*)"/)?.[1] ?? ""] return [...date, ...runs.join(" ").split("\u0001")] .map((block) => block.replace(/\s+/g, " ").trim()) .flatMap((block) => block.split(/(?<=[.!?])\s+(?=[A-Z"(])/)) .map((sentence) => sentence.replace(/\s+([.,;:)])/g, "$1").trim()) .filter((sentence) => /[A-Za-z]{2,}/.test(sentence)) } function jsxRuns(code: string, blocks: string) { const pattern = new RegExp( `<\\/?(?:${blocks})\\b|"((?:[^"\\\\\\n]|\\\\.)*)"|'((?:[^'\\\\\\n]|\\\\.)*)'|[>}]([^<>{}]+)(?=[<{])`, "g", ) return [...code.matchAll(pattern)].map((match) => { if (match[1] === undefined && match[2] === undefined && match[3] === undefined) return "\u0001" if (match[3] !== undefined) return clean(match[3]) const value = match[1] ?? match[2] // Prose strings have a word next to a space; ids, class names and CSS declarations don't. if (/^[a-z-]+:\s/.test(value)) return "" return /[A-Za-z]{2,}\s|\s[A-Za-z]{2,}/.test(value) ? clean(value) : "" }) } // In the node tree, a string after "[" is a tag name and one after "key:" is an attribute value; // every other string (except a quoted key) is a child, so prose. function treeRuns(code: string, blocks: string) { const block = new RegExp(`^(?:${blocks})$`) return [...code.matchAll(/"((?:[^"\\\n]|\\.)*)"|'((?:[^'\\\n]|\\.)*)'/g)].map((match) => { const value = match[1] ?? match[2] const before = code.slice(0, match.index).trimEnd().at(-1) if (before === "[") return block.test(value) ? "\u0001" : "" if (before === ":") return "" // A quoted attribute name, like "aria-label": if ( code .slice(match.index + match[0].length) .trimStart() .startsWith(":") ) return "" return clean(value) }) } /** The sentences that occur more often in `next` than in `previous`, each once, in order. */ function changed(next: string[], previous: string[]) { const count = (list: string[]) => list.reduce((counts, sentence) => counts.set(sentence, (counts.get(sentence) ?? 0) + 1), new Map()) const nextCounts = count(next) const previousCounts = count(previous) return [...nextCounts.keys()].filter((sentence) => nextCounts.get(sentence)! > (previousCounts.get(sentence) ?? 0)) } function clean(run: string) { return run .replace(/\\(["'\\])/g, "$1") .replace(/'|'|’|‘/g, "'") .replace(/"|“|”/g, '"') .replace(/&/g, "&") .replace(/ /g, " ") .replace(/[\u2018\u2019]/g, "'") .replace(/[\u201c\u201d]/g, '"') } function effective(text: string) { return text.match(/Effective(?: date)?:?\s*([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})/)?.[1] ?? null }