diff --git a/packages/cli-rs/crates/local/tests/shared_files.rs b/packages/cli-rs/crates/local/tests/shared_files.rs index f3b96f78021..582183cb9df 100644 --- a/packages/cli-rs/crates/local/tests/shared_files.rs +++ b/packages/cli-rs/crates/local/tests/shared_files.rs @@ -1,5 +1,6 @@ -//! P-FILE under contention for the state files the TypeScript TUI shares (MS1 review, carbon-7): -//! `session.json` and `prompt-history.jsonl`, written at the same time by this crate and by +//! P-FILE under contention for the state files the TypeScript TUI shares (MS1 review carbon-7, MS3/MS4 +//! review carbon-3): `session.json`, `prompt-history.jsonl`, `frecency.jsonl` and `prompt-stash.jsonl`, +//! written at the same time by this crate and by //! `parity/oracle/MS1-files.ts`, which writes with the TypeScript TUI's own `util/persistence` //! functions. A reader thread checks every state of the files meanwhile. //! @@ -232,6 +233,127 @@ fn prompt_history_under_contention() { ); } +/// `frecency.jsonl` at the limit: the TypeScript TUI opens new files while Rust does. Every state a reader +/// sees is whole lines, no visit either side wrote is lost, and the newest 1000 paths both TUIs read are +/// the same. +#[test] +fn frecency_at_the_limit_under_contention() { + if !available() { + return; + } + let state = tempfile::tempdir().unwrap(); + let dir = state.path().to_owned(); + let file = dir.join("frecency.jsonl"); + let full: String = (0..jsonl::MAX_FRECENCY_ENTRIES) + .map(|n| format!("{{\"path\":\"/w/old-{n}\",\"frequency\":1,\"lastOpen\":{n}}}\n")) + .collect(); + std::fs::write(&file, full).unwrap(); + let mut frecency = jsonl::Frecency::load(&dir, Path::new("/w")); + let stop = Arc::new(AtomicBool::new(false)); + let reader = watch(file.clone(), stop.clone(), whole_lines); + let mut ts = typescript(&["frecency", dir.to_str().unwrap(), "ts", &WRITES.to_string()]); + ts.go(); + for i in 0..WRITES { + frecency.update(&format!("rs-{i}"), jsonl::now_ms()); + thread::sleep(Duration::from_millis(1)); + } + ts.finish(); + // Rust reopens one of TypeScript's files: it counts from TypeScript's line. + frecency.update("ts-0", jsonl::now_ms()); + stop.store(true, Ordering::SeqCst); + let reads = reader.join().unwrap(); + + let typescript_read = read(&dir)["frecency"].clone(); + for side in ["ts", "rs"] { + for i in 0..WRITES { + let path = format!("/w/{side}-{i}"); + assert!(typescript_read.get(&path).is_some(), "{path} was lost"); + } + } + assert_eq!(typescript_read["/w/ts-0"], 2); + let lines = std::fs::read_to_string(&file).unwrap().lines().count(); + let loaded = jsonl::Frecency::load(&dir, Path::new("/w")); + assert_eq!(loaded.data().len(), jsonl::MAX_FRECENCY_ENTRIES); + let rust_read: serde_json::Map = loaded + .data() + .iter() + .map(|(path, visit)| (path.clone(), json!(visit.frequency as i64))) + .collect(); + assert_eq!(Value::Object(rust_read), typescript_read); + assert!(leftovers(&dir).is_empty(), "{:?}", leftovers(&dir)); + eprintln!( + "frecency.jsonl: {reads} reads, all whole; {lines} lines before the load trimmed it to 1000" + ); +} + +/// `prompt-stash.jsonl`: the TypeScript TUI stashes while Rust does, and then Rust pops its own entries. +/// Every state a reader sees is whole lines, no entry either side stashed is lost, and a pop drops only +/// its own line. +/// +/// The pops run after TypeScript stopped: a pop rewrites the file, and TypeScript appends without the +/// lock, so an append that opened the file just before the rename and writes just after it is lost (see +/// `steps/rs-ms34-files.md`). +#[test] +fn prompt_stash_under_contention() { + if !available() { + return; + } + let state = tempfile::tempdir().unwrap(); + let dir = state.path().to_owned(); + let stop = Arc::new(AtomicBool::new(false)); + let reader = watch(dir.join("prompt-stash.jsonl"), stop.clone(), whole_lines); + let mut stash = jsonl::Stash::load(&dir); + let mut ts = typescript(&["stash", dir.to_str().unwrap(), "ts", &WRITES.to_string()]); + ts.go(); + for i in 0..WRITES { + let prompt = json!({ "text": format!("rs-{i}"), "pasted": [] }); + stash.push(prompt, jsonl::now_ms()); + thread::sleep(Duration::from_millis(1)); + } + ts.finish(); + for _ in 0..WRITES / 2 { + stash.pop(); + } + stop.store(true, Ordering::SeqCst); + let reads = reader.join().unwrap(); + + let texts: HashSet = read(&dir)["stash"] + .as_array() + .unwrap() + .iter() + .filter_map(|text| text.as_str().map(str::to_owned)) + .collect(); + for i in 0..WRITES { + assert!( + texts.contains(&format!("ts-{i}")), + "ts-{i} was lost: {texts:?}" + ); + assert_eq!( + texts.contains(&format!("rs-{i}")), + i < WRITES / 2, + "rs-{i}: {texts:?}" + ); + } + assert!(leftovers(&dir).is_empty(), "{:?}", leftovers(&dir)); + eprintln!( + "prompt-stash.jsonl: {reads} reads, all whole; {} entries", + texts.len() + ); +} + +fn whole_lines(text: &str) -> Result<(), String> { + if !text.is_empty() && !text.ends_with('\n') { + return Err("a partial line".into()); + } + match text + .lines() + .find(|line| serde_json::from_str::(line).is_err()) + { + Some(line) => Err(format!("a broken line {line:?}")), + None => Ok(()), + } +} + /// TypeScript and Rust each write [`WRITES`] entries `-ts-` and `-rs-` at once. fn race(dir: &Path, round: &str, minimum: usize) -> usize { let stop = Arc::new(AtomicBool::new(false)); diff --git a/packages/cli-rs/parity/oracle/MS1-files.ts b/packages/cli-rs/parity/oracle/MS1-files.ts index 682693d0f3d..1fc4a7d0671 100644 --- a/packages/cli-rs/parity/oracle/MS1-files.ts +++ b/packages/cli-rs/parity/oracle/MS1-files.ts @@ -1,4 +1,4 @@ -// P-FILE for the shared state files of the MS1 review (carbon-7), driven by +// P-FILE for the shared state files of the MS1 review (carbon-7) and the MS3/MS4 review (carbon-3), driven by // crates/local/tests/shared_files.rs. Run it from /packages/tui with OpenTUI's Solid preload, as // E.2-history.ts runs. Each writing command prints `ready` and waits for one line on stdin, so the Rust // side can start writing at the same moment, then writes the way the TypeScript TUI does, with its own @@ -9,9 +9,14 @@ // writeJsonAtomic // bun MS1-files.ts history prompt-history.jsonl: `append` below the // limit, one appendText per entry -// bun MS1-files.ts read what the TypeScript TUI reads of both +// bun MS1-files.ts frecency frecency.jsonl: `updateFrecency` of a new +// path /w/-, one appendText each +// bun MS1-files.ts stash prompt-stash.jsonl: `push` below the limit, +// one appendText per entry +// bun MS1-files.ts read what the TypeScript TUI reads of them all // -// Prints one JSON value: `{pinned, history}` as `readJson` and `parsePromptHistory` read them. +// Prints one JSON value: `{pinned, history, frecency, stash}` as `readJson`, `parsePromptHistory`, +// `parseFrecency` (paths and frequencies) and `parsePromptStash` (texts) read them. import path from "node:path" const [root, command, state, prefix, count] = process.argv.slice(2) @@ -20,13 +25,26 @@ if (root && command && state) { const load = (spec: string) => import(Bun.resolveSync(spec, `${root}/packages/tui/src`)) const { appendText, readJson, readText, writeJsonAtomic } = await load("./util/persistence") const { parsePromptHistory } = await load("./prompt/history") + const { parseFrecency } = await load("./prompt/frecency") + const { parsePromptStash } = await load("./prompt/stash") const pins = path.join(state, "session.json") const history = path.join(state, "prompt-history.jsonl") + const frecency = path.join(state, "frecency.jsonl") + const stash = path.join(state, "prompt-stash.jsonl") const read = async () => ({ pinned: await readJson(pins) .then((value: { pinned: string[] }) => value.pinned) .catch(() => null), history: parsePromptHistory(await readText(history).catch(() => "")).map((entry: { text: string }) => entry.text), + frecency: Object.fromEntries( + parseFrecency(await readText(frecency).catch(() => "")).map((entry: { path: string; frequency: number }) => [ + entry.path, + entry.frequency, + ]), + ), + stash: parsePromptStash(await readText(stash).catch(() => "")).map( + (entry: { prompt: { text: string } }) => entry.prompt.text, + ), }) if (command !== "read") { console.log("ready") @@ -46,5 +64,19 @@ if (root && command && state) { await Bun.sleep(1) } } + if (command === "frecency") { + for (let i = 0; i < Number(count); i++) { + const entry = { path: `/w/${prefix}-${i}`, frequency: 1, lastOpen: Date.now() } + await appendText(frecency, JSON.stringify(entry) + "\n") + await Bun.sleep(1) + } + } + if (command === "stash") { + for (let i = 0; i < Number(count); i++) { + const prompt = { text: `${prefix}-${i}`, files: [], agents: [], skills: [], pasted: [] } + await appendText(stash, JSON.stringify({ prompt, timestamp: Date.now() }) + "\n") + await Bun.sleep(1) + } + } console.log(JSON.stringify(await read())) }