mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 17:01:37 +03:00
215 lines
9.0 KiB
TypeScript
215 lines
9.0 KiB
TypeScript
#!/usr/bin/env bun
|
|
// Reads the git history of each legal document's source files and writes the versions whose text
|
|
// changed to src/lib/legal/history.ts, which the History list, "What changed" and the legal feed read.
|
|
// Commits that only move or reformat the text (no sentence added or removed) are left out.
|
|
// Needs the full history (not a shallow clone). Run from packages/console/app after a legal change
|
|
// is committed: bun script/legal-history.ts
|
|
import { createHash } from "crypto"
|
|
import { dirname, join } from "path"
|
|
import { fileURLToPath } from "url"
|
|
|
|
const app = join(dirname(fileURLToPath(import.meta.url)), "..")
|
|
const repo = "https://github.com/anomalyco/opencode"
|
|
|
|
// Every file that has held a document's text, newest first. The text moved from the route files to
|
|
// src/lib/legal, so a version is the text of all of them at that commit.
|
|
const documents = {
|
|
"/legal/privacy-policy": [
|
|
"packages/console/app/src/lib/legal/privacy-policy.ts",
|
|
"packages/console/app/src/routes/legal/privacy-policy/index.tsx",
|
|
],
|
|
"/legal/terms-of-service": [
|
|
"packages/console/app/src/lib/legal/terms-of-service.ts",
|
|
"packages/console/app/src/routes/legal/terms-of-service/index.tsx",
|
|
],
|
|
}
|
|
|
|
// At most this many changed sentences are kept per version; the diff link has the rest.
|
|
const limit = 12
|
|
|
|
if ((await Bun.$`git rev-parse --is-shallow-repository`.cwd(app).text()).trim() === "true")
|
|
throw new Error("The history needs a full clone: run git fetch --unshallow first.")
|
|
const root = (await Bun.$`git rev-parse --show-toplevel`.cwd(app).text()).trim()
|
|
|
|
const history = Object.fromEntries(
|
|
await Promise.all(
|
|
Object.entries(documents).map(async ([path, files]) => {
|
|
const log = (await Bun.$`git log --format=%H%x09%aI%x09%s -- ${files}`.cwd(root).text())
|
|
.trim()
|
|
.split("\n")
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
const [sha, date, ...subject] = line.split("\t")
|
|
return { sha, date: date.slice(0, 10), subject: subject.join("\t") }
|
|
})
|
|
.reverse()
|
|
const texts = await Promise.all(log.map((commit) => source(commit.sha, files)))
|
|
const versions = log.flatMap((commit, index) => {
|
|
const before = index === 0 ? [] : sentences(texts[index - 1])
|
|
const after = sentences(texts[index])
|
|
const added = changed(after, before)
|
|
const removed = changed(before, after)
|
|
if (index > 0 && added.length === 0 && removed.length === 0) return []
|
|
const file = files.find((file) => texts[index].files.includes(file)) ?? files[0]
|
|
return [
|
|
{
|
|
sha: commit.sha,
|
|
date: commit.date,
|
|
subject: commit.subject,
|
|
effective: effective(texts[index].text),
|
|
first: index === 0,
|
|
added: index === 0 ? [] : added.slice(0, limit),
|
|
removed: index === 0 ? [] : removed.slice(0, limit),
|
|
addedCount: added.length,
|
|
removedCount: removed.length,
|
|
source: `${repo}/blob/${commit.sha}/${file}`,
|
|
diff: `${repo}/commit/${commit.sha}#diff-${createHash("sha256").update(file).digest("hex")}`,
|
|
},
|
|
]
|
|
})
|
|
return [path, versions.reverse()] as const
|
|
}),
|
|
),
|
|
)
|
|
|
|
await Bun.write(
|
|
join(app, "src/lib/legal/history.ts"),
|
|
[
|
|
"// Generated by script/legal-history.ts from the git history of the legal documents. Do not edit.",
|
|
"",
|
|
`export const legalHistory = ${JSON.stringify(history, null, 2)} satisfies Record<string, LegalVersion[]>`,
|
|
"",
|
|
"/** A version of a legal document: a commit that changed its text, newest first. */",
|
|
"export type LegalVersion = {",
|
|
" sha: string",
|
|
" /** The commit date, YYYY-MM-DD. */",
|
|
" date: string",
|
|
" subject: string",
|
|
" /** The effective date the document stated in this version. */",
|
|
" effective: string | null",
|
|
" /** The version that first published the document. */",
|
|
" first: boolean",
|
|
" /** Sentences this version added and removed, up to " + limit + " of each. */",
|
|
" added: string[]",
|
|
" removed: string[]",
|
|
" addedCount: number",
|
|
" removedCount: number",
|
|
" /** The document's source file at this version, on GitHub. */",
|
|
" source: string",
|
|
" /** The commit's diff of that file, on GitHub. */",
|
|
" diff: string",
|
|
"}",
|
|
"",
|
|
].join("\n"),
|
|
)
|
|
await Bun.$`bunx prettier --write src/lib/legal/history.ts`.cwd(app).quiet()
|
|
console.log(
|
|
Object.entries(history)
|
|
.map(([path, versions]) => `${path}: ${versions.length} versions`)
|
|
.join("\n"),
|
|
)
|
|
|
|
/** The document's source files at a commit, joined, and which of them existed. */
|
|
async function source(sha: string, files: string[]) {
|
|
const contents = await Promise.all(
|
|
files.map(async (file) => {
|
|
const result = await Bun.$`git show ${sha}:${file}`.cwd(root).quiet().nothrow()
|
|
return result.exitCode === 0 ? { file, text: result.stdout.toString() } : undefined
|
|
}),
|
|
)
|
|
const found = contents.filter((content) => content !== undefined)
|
|
return { files: found.map((content) => content.file), text: found.map((content) => content.text).join("\n"), found }
|
|
}
|
|
|
|
/**
|
|
* The document's prose as sentences, split at block elements (paragraphs, headings, list items,
|
|
* table cells) and then at sentence ends. The same text in the old JSX and the newer node-tree
|
|
* source gives the same sentences, so moving the text between them is not a change.
|
|
*/
|
|
function sentences(input: { found: { file: string; text: string }[] }) {
|
|
return input.found.flatMap((content) => fileSentences(content.text, content.file.endsWith(".tsx")))
|
|
}
|
|
|
|
/**
|
|
* One source file's prose. Old route files are JSX: the text between tags, from the effective date
|
|
* on (the page title and meta tags come before it). Newer files are a node tree: every string that
|
|
* is a node's child (not a tag name or an attribute value), plus the effective date.
|
|
*/
|
|
function fileSentences(text: string, jsx: boolean) {
|
|
const stripped = text
|
|
.replace(/^import .*$/gm, "")
|
|
.replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
|
|
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
.replace(/^\s*\/\/.*$/gm, "")
|
|
const start = jsx ? stripped.indexOf("effective-date") : stripped.indexOf("body:")
|
|
const code = start === -1 ? stripped : stripped.slice(start)
|
|
const blocks = "p|li|h[1-6]|td|th|tr|ul|ol|div|table|thead|tbody|header|section|article"
|
|
const runs = jsx ? jsxRuns(code, blocks) : treeRuns(code, blocks)
|
|
const date = jsx ? [] : [text.match(/effective:\s*"([^"]*)"/)?.[1] ?? ""]
|
|
return [...date, ...runs.join(" ").split("\u0001")]
|
|
.map((block) => block.replace(/\s+/g, " ").trim())
|
|
.flatMap((block) => block.split(/(?<=[.!?])\s+(?=[A-Z"(])/))
|
|
.map((sentence) => sentence.replace(/\s+([.,;:)])/g, "$1").trim())
|
|
.filter((sentence) => /[A-Za-z]{2,}/.test(sentence))
|
|
}
|
|
|
|
function jsxRuns(code: string, blocks: string) {
|
|
const pattern = new RegExp(
|
|
`<\\/?(?:${blocks})\\b|"((?:[^"\\\\\\n]|\\\\.)*)"|'((?:[^'\\\\\\n]|\\\\.)*)'|[>}]([^<>{}]+)(?=[<{])`,
|
|
"g",
|
|
)
|
|
return [...code.matchAll(pattern)].map((match) => {
|
|
if (match[1] === undefined && match[2] === undefined && match[3] === undefined) return "\u0001"
|
|
if (match[3] !== undefined) return clean(match[3])
|
|
const value = match[1] ?? match[2]
|
|
// Prose strings have a word next to a space; ids, class names and CSS declarations don't.
|
|
if (/^[a-z-]+:\s/.test(value)) return ""
|
|
return /[A-Za-z]{2,}\s|\s[A-Za-z]{2,}/.test(value) ? clean(value) : ""
|
|
})
|
|
}
|
|
|
|
// In the node tree, a string after "[" is a tag name and one after "key:" is an attribute value;
|
|
// every other string (except a quoted key) is a child, so prose.
|
|
function treeRuns(code: string, blocks: string) {
|
|
const block = new RegExp(`^(?:${blocks})$`)
|
|
return [...code.matchAll(/"((?:[^"\\\n]|\\.)*)"|'((?:[^'\\\n]|\\.)*)'/g)].map((match) => {
|
|
const value = match[1] ?? match[2]
|
|
const before = code.slice(0, match.index).trimEnd().at(-1)
|
|
if (before === "[") return block.test(value) ? "\u0001" : ""
|
|
if (before === ":") return ""
|
|
// A quoted attribute name, like "aria-label":
|
|
if (
|
|
code
|
|
.slice(match.index + match[0].length)
|
|
.trimStart()
|
|
.startsWith(":")
|
|
)
|
|
return ""
|
|
return clean(value)
|
|
})
|
|
}
|
|
|
|
/** The sentences that occur more often in `next` than in `previous`, each once, in order. */
|
|
function changed(next: string[], previous: string[]) {
|
|
const count = (list: string[]) =>
|
|
list.reduce((counts, sentence) => counts.set(sentence, (counts.get(sentence) ?? 0) + 1), new Map<string, number>())
|
|
const nextCounts = count(next)
|
|
const previousCounts = count(previous)
|
|
return [...nextCounts.keys()].filter((sentence) => nextCounts.get(sentence)! > (previousCounts.get(sentence) ?? 0))
|
|
}
|
|
|
|
function clean(run: string) {
|
|
return run
|
|
.replace(/\\(["'\\])/g, "$1")
|
|
.replace(/'|'|’|‘/g, "'")
|
|
.replace(/"|“|”/g, '"')
|
|
.replace(/&/g, "&")
|
|
.replace(/ /g, " ")
|
|
.replace(/[\u2018\u2019]/g, "'")
|
|
.replace(/[\u201c\u201d]/g, '"')
|
|
}
|
|
|
|
function effective(text: string) {
|
|
return text.match(/Effective(?: date)?:?\s*([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})/)?.[1] ?? null
|
|
}
|