Files
opencode/services/site/script/legal-history.ts

215 lines
9.0 KiB
TypeScript

#!/usr/bin/env bun
// Reads the git history of each legal document's source files and writes the versions whose text
// changed to src/lib/legal/history.ts, which the History list, "What changed" and the legal feed read.
// Commits that only move or reformat the text (no sentence added or removed) are left out.
// Needs the full history (not a shallow clone). Run from packages/console/app after a legal change
// is committed: bun script/legal-history.ts
import { createHash } from "crypto"
import { dirname, join } from "path"
import { fileURLToPath } from "url"
const app = join(dirname(fileURLToPath(import.meta.url)), "..")
const repo = "https://github.com/anomalyco/opencode"
// Every file that has held a document's text, newest first. The text moved from the route files to
// src/lib/legal, so a version is the text of all of them at that commit.
const documents = {
"/legal/privacy-policy": [
"packages/console/app/src/lib/legal/privacy-policy.ts",
"packages/console/app/src/routes/legal/privacy-policy/index.tsx",
],
"/legal/terms-of-service": [
"packages/console/app/src/lib/legal/terms-of-service.ts",
"packages/console/app/src/routes/legal/terms-of-service/index.tsx",
],
}
// At most this many changed sentences are kept per version; the diff link has the rest.
const limit = 12
if ((await Bun.$`git rev-parse --is-shallow-repository`.cwd(app).text()).trim() === "true")
throw new Error("The history needs a full clone: run git fetch --unshallow first.")
const root = (await Bun.$`git rev-parse --show-toplevel`.cwd(app).text()).trim()
const history = Object.fromEntries(
await Promise.all(
Object.entries(documents).map(async ([path, files]) => {
const log = (await Bun.$`git log --format=%H%x09%aI%x09%s -- ${files}`.cwd(root).text())
.trim()
.split("\n")
.filter(Boolean)
.map((line) => {
const [sha, date, ...subject] = line.split("\t")
return { sha, date: date.slice(0, 10), subject: subject.join("\t") }
})
.reverse()
const texts = await Promise.all(log.map((commit) => source(commit.sha, files)))
const versions = log.flatMap((commit, index) => {
const before = index === 0 ? [] : sentences(texts[index - 1])
const after = sentences(texts[index])
const added = changed(after, before)
const removed = changed(before, after)
if (index > 0 && added.length === 0 && removed.length === 0) return []
const file = files.find((file) => texts[index].files.includes(file)) ?? files[0]
return [
{
sha: commit.sha,
date: commit.date,
subject: commit.subject,
effective: effective(texts[index].text),
first: index === 0,
added: index === 0 ? [] : added.slice(0, limit),
removed: index === 0 ? [] : removed.slice(0, limit),
addedCount: added.length,
removedCount: removed.length,
source: `${repo}/blob/${commit.sha}/${file}`,
diff: `${repo}/commit/${commit.sha}#diff-${createHash("sha256").update(file).digest("hex")}`,
},
]
})
return [path, versions.reverse()] as const
}),
),
)
await Bun.write(
join(app, "src/lib/legal/history.ts"),
[
"// Generated by script/legal-history.ts from the git history of the legal documents. Do not edit.",
"",
`export const legalHistory = ${JSON.stringify(history, null, 2)} satisfies Record<string, LegalVersion[]>`,
"",
"/** A version of a legal document: a commit that changed its text, newest first. */",
"export type LegalVersion = {",
" sha: string",
" /** The commit date, YYYY-MM-DD. */",
" date: string",
" subject: string",
" /** The effective date the document stated in this version. */",
" effective: string | null",
" /** The version that first published the document. */",
" first: boolean",
" /** Sentences this version added and removed, up to " + limit + " of each. */",
" added: string[]",
" removed: string[]",
" addedCount: number",
" removedCount: number",
" /** The document's source file at this version, on GitHub. */",
" source: string",
" /** The commit's diff of that file, on GitHub. */",
" diff: string",
"}",
"",
].join("\n"),
)
await Bun.$`bunx prettier --write src/lib/legal/history.ts`.cwd(app).quiet()
console.log(
Object.entries(history)
.map(([path, versions]) => `${path}: ${versions.length} versions`)
.join("\n"),
)
/** The document's source files at a commit, joined, and which of them existed. */
async function source(sha: string, files: string[]) {
const contents = await Promise.all(
files.map(async (file) => {
const result = await Bun.$`git show ${sha}:${file}`.cwd(root).quiet().nothrow()
return result.exitCode === 0 ? { file, text: result.stdout.toString() } : undefined
}),
)
const found = contents.filter((content) => content !== undefined)
return { files: found.map((content) => content.file), text: found.map((content) => content.text).join("\n"), found }
}
/**
* The document's prose as sentences, split at block elements (paragraphs, headings, list items,
* table cells) and then at sentence ends. The same text in the old JSX and the newer node-tree
* source gives the same sentences, so moving the text between them is not a change.
*/
function sentences(input: { found: { file: string; text: string }[] }) {
return input.found.flatMap((content) => fileSentences(content.text, content.file.endsWith(".tsx")))
}
/**
* One source file's prose. Old route files are JSX: the text between tags, from the effective date
* on (the page title and meta tags come before it). Newer files are a node tree: every string that
* is a node's child (not a tag name or an attribute value), plus the effective date.
*/
function fileSentences(text: string, jsx: boolean) {
const stripped = text
.replace(/^import .*$/gm, "")
.replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
.replace(/\/\*[\s\S]*?\*\//g, "")
.replace(/^\s*\/\/.*$/gm, "")
const start = jsx ? stripped.indexOf("effective-date") : stripped.indexOf("body:")
const code = start === -1 ? stripped : stripped.slice(start)
const blocks = "p|li|h[1-6]|td|th|tr|ul|ol|div|table|thead|tbody|header|section|article"
const runs = jsx ? jsxRuns(code, blocks) : treeRuns(code, blocks)
const date = jsx ? [] : [text.match(/effective:\s*"([^"]*)"/)?.[1] ?? ""]
return [...date, ...runs.join(" ").split("\u0001")]
.map((block) => block.replace(/\s+/g, " ").trim())
.flatMap((block) => block.split(/(?<=[.!?])\s+(?=[A-Z"(])/))
.map((sentence) => sentence.replace(/\s+([.,;:)])/g, "$1").trim())
.filter((sentence) => /[A-Za-z]{2,}/.test(sentence))
}
function jsxRuns(code: string, blocks: string) {
const pattern = new RegExp(
`<\\/?(?:${blocks})\\b|"((?:[^"\\\\\\n]|\\\\.)*)"|'((?:[^'\\\\\\n]|\\\\.)*)'|[>}]([^<>{}]+)(?=[<{])`,
"g",
)
return [...code.matchAll(pattern)].map((match) => {
if (match[1] === undefined && match[2] === undefined && match[3] === undefined) return "\u0001"
if (match[3] !== undefined) return clean(match[3])
const value = match[1] ?? match[2]
// Prose strings have a word next to a space; ids, class names and CSS declarations don't.
if (/^[a-z-]+:\s/.test(value)) return ""
return /[A-Za-z]{2,}\s|\s[A-Za-z]{2,}/.test(value) ? clean(value) : ""
})
}
// In the node tree, a string after "[" is a tag name and one after "key:" is an attribute value;
// every other string (except a quoted key) is a child, so prose.
function treeRuns(code: string, blocks: string) {
const block = new RegExp(`^(?:${blocks})$`)
return [...code.matchAll(/"((?:[^"\\\n]|\\.)*)"|'((?:[^'\\\n]|\\.)*)'/g)].map((match) => {
const value = match[1] ?? match[2]
const before = code.slice(0, match.index).trimEnd().at(-1)
if (before === "[") return block.test(value) ? "\u0001" : ""
if (before === ":") return ""
// A quoted attribute name, like "aria-label":
if (
code
.slice(match.index + match[0].length)
.trimStart()
.startsWith(":")
)
return ""
return clean(value)
})
}
/** The sentences that occur more often in `next` than in `previous`, each once, in order. */
function changed(next: string[], previous: string[]) {
const count = (list: string[]) =>
list.reduce((counts, sentence) => counts.set(sentence, (counts.get(sentence) ?? 0) + 1), new Map<string, number>())
const nextCounts = count(next)
const previousCounts = count(previous)
return [...nextCounts.keys()].filter((sentence) => nextCounts.get(sentence)! > (previousCounts.get(sentence) ?? 0))
}
function clean(run: string) {
return run
.replace(/\\(["'\\])/g, "$1")
.replace(/&apos;|&#39;|&rsquo;|&lsquo;/g, "'")
.replace(/&quot;|&ldquo;|&rdquo;/g, '"')
.replace(/&amp;/g, "&")
.replace(/&nbsp;/g, " ")
.replace(/[\u2018\u2019]/g, "'")
.replace(/[\u201c\u201d]/g, '"')
}
function effective(text: string) {
return text.match(/Effective(?: date)?:?\s*([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})/)?.[1] ?? null
}