diff --git a/packages/cli-rs/crates/ui/src/fuzzy.rs b/packages/cli-rs/crates/ui/src/fuzzy.rs index 27f7465920b..8dd63454d82 100644 --- a/packages/cli-rs/crates/ui/src/fuzzy.rs +++ b/packages/cli-rs/crates/ui/src/fuzzy.rs @@ -1,6 +1,8 @@ //! A port of `fuzzysort` 3.1's `go` with `keys`, `scoreFn`, and `threshold`, as `DialogSelect` calls it: the same -//! scores and the same order, ties included. Strings are compared in UTF-16 code units, as JavaScript does. -//! Accents are not removed from Latin letters (TS `remove_accents`). +//! scores and the same order, ties included. Strings are compared in UTF-16 code units, as JavaScript does, after +//! Latin letters lose their accents (`remove_accents`). + +mod accents; const NEG: f64 = f64::NEG_INFINITY; @@ -152,7 +154,7 @@ fn denormalize(score: f64) -> f64 { } fn lower_info(text: &str) -> (Vec, u32, bool) { - let lower: Vec = text.to_lowercase().encode_utf16().collect(); + let lower: Vec = remove_accents(text).to_lowercase().encode_utf16().collect(); let mut bitflags = 0u32; let mut space = false; for &code in &lower { @@ -193,8 +195,23 @@ fn prepare(target: &str) -> Prepared { Prepared { lower, bitflags, next: next_beginnings(target) } } +/// `remove_accents`: Latin letters decomposed, and the marks U+0300 to U+036F dropped. +fn remove_accents(text: &str) -> std::borrow::Cow<'_, str> { + if text.is_ascii() { + return text.into(); + } + let folded = text.chars().fold(String::with_capacity(text.len()), |mut out, char| { + match accents::FOLD.binary_search_by_key(&char, |(from, _)| *from) { + Ok(index) => out.push_str(accents::FOLD[index].1), + Err(_) => out.push(char), + } + out + }); + folded.into() +} + fn next_beginnings(target: &str) -> Vec { - let codes: Vec = target.encode_utf16().collect(); + let codes: Vec = remove_accents(target).encode_utf16().collect(); let mut beginnings = Vec::new(); let (mut was_upper, mut was_alnum) = (false, false); for (i, &code) in codes.iter().enumerate() { @@ -476,3 +493,34 @@ impl Heap { self.items[a] = v; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accents_fold_before_matching_as_remove_accents_does() { + let score = |search: &str, target: &str| { + let seen = std::cell::Cell::new(None); + let keep = |_: &[f64; 1], score, _| { + seen.set(Some(score)); + score + }; + go_scored(search, &[target], |target| [Some(*target)], keep, 0.0, 1); + seen.get() + }; + // An accented letter, a decomposed one, and an accented search score as their plain forms. + for (search, target, plain) in [ + ("cafe", "Café crème", "Cafe creme"), + ("ang", "Ångström", "Angstrom"), + ("cafe", "Cafe\u{301}", "Cafe"), + ("crème", "cream creme", "cream creme"), + ] { + assert_eq!(score(search, target), score(search, plain), "{search:?} in {target:?}"); + assert!(score(search, target).is_some(), "{search:?} in {target:?}"); + } + // Letters without a decomposition and other scripts stay. + assert_eq!(score("o", "Ø"), None); + assert_eq!(score("α", "ά"), None); + } +} diff --git a/packages/cli-rs/crates/ui/src/fuzzy/accents.rs b/packages/cli-rs/crates/ui/src/fuzzy/accents.rs new file mode 100644 index 00000000000..10b25675e63 --- /dev/null +++ b/packages/cli-rs/crates/ui/src/fuzzy/accents.rs @@ -0,0 +1,616 @@ +// Generated by script/gen/accents.ts from fuzzysort's `remove_accents`. Do not edit. + +/// `(character, folded)`, sorted by character. +pub const FOLD: &[(char, &str)] = &[ + ('\u{c0}', "A"), + ('\u{c1}', "A"), + ('\u{c2}', "A"), + ('\u{c3}', "A"), + ('\u{c4}', "A"), + ('\u{c5}', "A"), + ('\u{c7}', "C"), + ('\u{c8}', "E"), + ('\u{c9}', "E"), + ('\u{ca}', "E"), + ('\u{cb}', "E"), + ('\u{cc}', "I"), + ('\u{cd}', "I"), + ('\u{ce}', "I"), + ('\u{cf}', "I"), + ('\u{d1}', "N"), + ('\u{d2}', "O"), + ('\u{d3}', "O"), + ('\u{d4}', "O"), + ('\u{d5}', "O"), + ('\u{d6}', "O"), + ('\u{d9}', "U"), + ('\u{da}', "U"), + ('\u{db}', "U"), + ('\u{dc}', "U"), + ('\u{dd}', "Y"), + ('\u{e0}', "a"), + ('\u{e1}', "a"), + ('\u{e2}', "a"), + ('\u{e3}', "a"), + ('\u{e4}', "a"), + ('\u{e5}', "a"), + ('\u{e7}', "c"), + ('\u{e8}', "e"), + ('\u{e9}', "e"), + ('\u{ea}', "e"), + ('\u{eb}', "e"), + ('\u{ec}', "i"), + ('\u{ed}', "i"), + ('\u{ee}', "i"), + ('\u{ef}', "i"), + ('\u{f1}', "n"), + ('\u{f2}', "o"), + ('\u{f3}', "o"), + ('\u{f4}', "o"), + ('\u{f5}', "o"), + ('\u{f6}', "o"), + ('\u{f9}', "u"), + ('\u{fa}', "u"), + ('\u{fb}', "u"), + ('\u{fc}', "u"), + ('\u{fd}', "y"), + ('\u{ff}', "y"), + ('\u{100}', "A"), + ('\u{101}', "a"), + ('\u{102}', "A"), + ('\u{103}', "a"), + ('\u{104}', "A"), + ('\u{105}', "a"), + ('\u{106}', "C"), + ('\u{107}', "c"), + ('\u{108}', "C"), + ('\u{109}', "c"), + ('\u{10a}', "C"), + ('\u{10b}', "c"), + ('\u{10c}', "C"), + ('\u{10d}', "c"), + ('\u{10e}', "D"), + ('\u{10f}', "d"), + ('\u{112}', "E"), + ('\u{113}', "e"), + ('\u{114}', "E"), + ('\u{115}', "e"), + ('\u{116}', "E"), + ('\u{117}', "e"), + ('\u{118}', "E"), + ('\u{119}', "e"), + ('\u{11a}', "E"), + ('\u{11b}', "e"), + ('\u{11c}', "G"), + ('\u{11d}', "g"), + ('\u{11e}', "G"), + ('\u{11f}', "g"), + ('\u{120}', "G"), + ('\u{121}', "g"), + ('\u{122}', "G"), + ('\u{123}', "g"), + ('\u{124}', "H"), + ('\u{125}', "h"), + ('\u{128}', "I"), + ('\u{129}', "i"), + ('\u{12a}', "I"), + ('\u{12b}', "i"), + ('\u{12c}', "I"), + ('\u{12d}', "i"), + ('\u{12e}', "I"), + ('\u{12f}', "i"), + ('\u{130}', "I"), + ('\u{134}', "J"), + ('\u{135}', "j"), + ('\u{136}', "K"), + ('\u{137}', "k"), + ('\u{139}', "L"), + ('\u{13a}', "l"), + ('\u{13b}', "L"), + ('\u{13c}', "l"), + ('\u{13d}', "L"), + ('\u{13e}', "l"), + ('\u{143}', "N"), + ('\u{144}', "n"), + ('\u{145}', "N"), + ('\u{146}', "n"), + ('\u{147}', "N"), + ('\u{148}', "n"), + ('\u{14c}', "O"), + ('\u{14d}', "o"), + ('\u{14e}', "O"), + ('\u{14f}', "o"), + ('\u{150}', "O"), + ('\u{151}', "o"), + ('\u{154}', "R"), + ('\u{155}', "r"), + ('\u{156}', "R"), + ('\u{157}', "r"), + ('\u{158}', "R"), + ('\u{159}', "r"), + ('\u{15a}', "S"), + ('\u{15b}', "s"), + ('\u{15c}', "S"), + ('\u{15d}', "s"), + ('\u{15e}', "S"), + ('\u{15f}', "s"), + ('\u{160}', "S"), + ('\u{161}', "s"), + ('\u{162}', "T"), + ('\u{163}', "t"), + ('\u{164}', "T"), + ('\u{165}', "t"), + ('\u{168}', "U"), + ('\u{169}', "u"), + ('\u{16a}', "U"), + ('\u{16b}', "u"), + ('\u{16c}', "U"), + ('\u{16d}', "u"), + ('\u{16e}', "U"), + ('\u{16f}', "u"), + ('\u{170}', "U"), + ('\u{171}', "u"), + ('\u{172}', "U"), + ('\u{173}', "u"), + ('\u{174}', "W"), + ('\u{175}', "w"), + ('\u{176}', "Y"), + ('\u{177}', "y"), + ('\u{178}', "Y"), + ('\u{179}', "Z"), + ('\u{17a}', "z"), + ('\u{17b}', "Z"), + ('\u{17c}', "z"), + ('\u{17d}', "Z"), + ('\u{17e}', "z"), + ('\u{1a0}', "O"), + ('\u{1a1}', "o"), + ('\u{1af}', "U"), + ('\u{1b0}', "u"), + ('\u{1cd}', "A"), + ('\u{1ce}', "a"), + ('\u{1cf}', "I"), + ('\u{1d0}', "i"), + ('\u{1d1}', "O"), + ('\u{1d2}', "o"), + ('\u{1d3}', "U"), + ('\u{1d4}', "u"), + ('\u{1d5}', "U"), + ('\u{1d6}', "u"), + ('\u{1d7}', "U"), + ('\u{1d8}', "u"), + ('\u{1d9}', "U"), + ('\u{1da}', "u"), + ('\u{1db}', "U"), + ('\u{1dc}', "u"), + ('\u{1de}', "A"), + ('\u{1df}', "a"), + ('\u{1e0}', "A"), + ('\u{1e1}', "a"), + ('\u{1e2}', "\u{c6}"), + ('\u{1e3}', "\u{e6}"), + ('\u{1e6}', "G"), + ('\u{1e7}', "g"), + ('\u{1e8}', "K"), + ('\u{1e9}', "k"), + ('\u{1ea}', "O"), + ('\u{1eb}', "o"), + ('\u{1ec}', "O"), + ('\u{1ed}', "o"), + ('\u{1ee}', "\u{1b7}"), + ('\u{1ef}', "\u{292}"), + ('\u{1f0}', "j"), + ('\u{1f4}', "G"), + ('\u{1f5}', "g"), + ('\u{1f8}', "N"), + ('\u{1f9}', "n"), + ('\u{1fa}', "A"), + ('\u{1fb}', "a"), + ('\u{1fc}', "\u{c6}"), + ('\u{1fd}', "\u{e6}"), + ('\u{1fe}', "\u{d8}"), + ('\u{1ff}', "\u{f8}"), + ('\u{200}', "A"), + ('\u{201}', "a"), + ('\u{202}', "A"), + ('\u{203}', "a"), + ('\u{204}', "E"), + ('\u{205}', "e"), + ('\u{206}', "E"), + ('\u{207}', "e"), + ('\u{208}', "I"), + ('\u{209}', "i"), + ('\u{20a}', "I"), + ('\u{20b}', "i"), + ('\u{20c}', "O"), + ('\u{20d}', "o"), + ('\u{20e}', "O"), + ('\u{20f}', "o"), + ('\u{210}', "R"), + ('\u{211}', "r"), + ('\u{212}', "R"), + ('\u{213}', "r"), + ('\u{214}', "U"), + ('\u{215}', "u"), + ('\u{216}', "U"), + ('\u{217}', "u"), + ('\u{218}', "S"), + ('\u{219}', "s"), + ('\u{21a}', "T"), + ('\u{21b}', "t"), + ('\u{21e}', "H"), + ('\u{21f}', "h"), + ('\u{226}', "A"), + ('\u{227}', "a"), + ('\u{228}', "E"), + ('\u{229}', "e"), + ('\u{22a}', "O"), + ('\u{22b}', "o"), + ('\u{22c}', "O"), + ('\u{22d}', "o"), + ('\u{22e}', "O"), + ('\u{22f}', "o"), + ('\u{230}', "O"), + ('\u{231}', "o"), + ('\u{232}', "Y"), + ('\u{233}', "y"), + ('\u{300}', ""), + ('\u{301}', ""), + ('\u{302}', ""), + ('\u{303}', ""), + ('\u{304}', ""), + ('\u{305}', ""), + ('\u{306}', ""), + ('\u{307}', ""), + ('\u{308}', ""), + ('\u{309}', ""), + ('\u{30a}', ""), + ('\u{30b}', ""), + ('\u{30c}', ""), + ('\u{30d}', ""), + ('\u{30e}', ""), + ('\u{30f}', ""), + ('\u{310}', ""), + ('\u{311}', ""), + ('\u{312}', ""), + ('\u{313}', ""), + ('\u{314}', ""), + ('\u{315}', ""), + ('\u{316}', ""), + ('\u{317}', ""), + ('\u{318}', ""), + ('\u{319}', ""), + ('\u{31a}', ""), + ('\u{31b}', ""), + ('\u{31c}', ""), + ('\u{31d}', ""), + ('\u{31e}', ""), + ('\u{31f}', ""), + ('\u{320}', ""), + ('\u{321}', ""), + ('\u{322}', ""), + ('\u{323}', ""), + ('\u{324}', ""), + ('\u{325}', ""), + ('\u{326}', ""), + ('\u{327}', ""), + ('\u{328}', ""), + ('\u{329}', ""), + ('\u{32a}', ""), + ('\u{32b}', ""), + ('\u{32c}', ""), + ('\u{32d}', ""), + ('\u{32e}', ""), + ('\u{32f}', ""), + ('\u{330}', ""), + ('\u{331}', ""), + ('\u{332}', ""), + ('\u{333}', ""), + ('\u{334}', ""), + ('\u{335}', ""), + ('\u{336}', ""), + ('\u{337}', ""), + ('\u{338}', ""), + ('\u{339}', ""), + ('\u{33a}', ""), + ('\u{33b}', ""), + ('\u{33c}', ""), + ('\u{33d}', ""), + ('\u{33e}', ""), + ('\u{33f}', ""), + ('\u{340}', ""), + ('\u{341}', ""), + ('\u{342}', ""), + ('\u{343}', ""), + ('\u{344}', ""), + ('\u{345}', ""), + ('\u{346}', ""), + ('\u{347}', ""), + ('\u{348}', ""), + ('\u{349}', ""), + ('\u{34a}', ""), + ('\u{34b}', ""), + ('\u{34c}', ""), + ('\u{34d}', ""), + ('\u{34e}', ""), + ('\u{34f}', ""), + ('\u{350}', ""), + ('\u{351}', ""), + ('\u{352}', ""), + ('\u{353}', ""), + ('\u{354}', ""), + ('\u{355}', ""), + ('\u{356}', ""), + ('\u{357}', ""), + ('\u{358}', ""), + ('\u{359}', ""), + ('\u{35a}', ""), + ('\u{35b}', ""), + ('\u{35c}', ""), + ('\u{35d}', ""), + ('\u{35e}', ""), + ('\u{35f}', ""), + ('\u{360}', ""), + ('\u{361}', ""), + ('\u{362}', ""), + ('\u{363}', ""), + ('\u{364}', ""), + ('\u{365}', ""), + ('\u{366}', ""), + ('\u{367}', ""), + ('\u{368}', ""), + ('\u{369}', ""), + ('\u{36a}', ""), + ('\u{36b}', ""), + ('\u{36c}', ""), + ('\u{36d}', ""), + ('\u{36e}', ""), + ('\u{36f}', ""), + ('\u{1e00}', "A"), + ('\u{1e01}', "a"), + ('\u{1e02}', "B"), + ('\u{1e03}', "b"), + ('\u{1e04}', "B"), + ('\u{1e05}', "b"), + ('\u{1e06}', "B"), + ('\u{1e07}', "b"), + ('\u{1e08}', "C"), + ('\u{1e09}', "c"), + ('\u{1e0a}', "D"), + ('\u{1e0b}', "d"), + ('\u{1e0c}', "D"), + ('\u{1e0d}', "d"), + ('\u{1e0e}', "D"), + ('\u{1e0f}', "d"), + ('\u{1e10}', "D"), + ('\u{1e11}', "d"), + ('\u{1e12}', "D"), + ('\u{1e13}', "d"), + ('\u{1e14}', "E"), + ('\u{1e15}', "e"), + ('\u{1e16}', "E"), + ('\u{1e17}', "e"), + ('\u{1e18}', "E"), + ('\u{1e19}', "e"), + ('\u{1e1a}', "E"), + ('\u{1e1b}', "e"), + ('\u{1e1c}', "E"), + ('\u{1e1d}', "e"), + ('\u{1e1e}', "F"), + ('\u{1e1f}', "f"), + ('\u{1e20}', "G"), + ('\u{1e21}', "g"), + ('\u{1e22}', "H"), + ('\u{1e23}', "h"), + ('\u{1e24}', "H"), + ('\u{1e25}', "h"), + ('\u{1e26}', "H"), + ('\u{1e27}', "h"), + ('\u{1e28}', "H"), + ('\u{1e29}', "h"), + ('\u{1e2a}', "H"), + ('\u{1e2b}', "h"), + ('\u{1e2c}', "I"), + ('\u{1e2d}', "i"), + ('\u{1e2e}', "I"), + ('\u{1e2f}', "i"), + ('\u{1e30}', "K"), + ('\u{1e31}', "k"), + ('\u{1e32}', "K"), + ('\u{1e33}', "k"), + ('\u{1e34}', "K"), + ('\u{1e35}', "k"), + ('\u{1e36}', "L"), + ('\u{1e37}', "l"), + ('\u{1e38}', "L"), + ('\u{1e39}', "l"), + ('\u{1e3a}', "L"), + ('\u{1e3b}', "l"), + ('\u{1e3c}', "L"), + ('\u{1e3d}', "l"), + ('\u{1e3e}', "M"), + ('\u{1e3f}', "m"), + ('\u{1e40}', "M"), + ('\u{1e41}', "m"), + ('\u{1e42}', "M"), + ('\u{1e43}', "m"), + ('\u{1e44}', "N"), + ('\u{1e45}', "n"), + ('\u{1e46}', "N"), + ('\u{1e47}', "n"), + ('\u{1e48}', "N"), + ('\u{1e49}', "n"), + ('\u{1e4a}', "N"), + ('\u{1e4b}', "n"), + ('\u{1e4c}', "O"), + ('\u{1e4d}', "o"), + ('\u{1e4e}', "O"), + ('\u{1e4f}', "o"), + ('\u{1e50}', "O"), + ('\u{1e51}', "o"), + ('\u{1e52}', "O"), + ('\u{1e53}', "o"), + ('\u{1e54}', "P"), + ('\u{1e55}', "p"), + ('\u{1e56}', "P"), + ('\u{1e57}', "p"), + ('\u{1e58}', "R"), + ('\u{1e59}', "r"), + ('\u{1e5a}', "R"), + ('\u{1e5b}', "r"), + ('\u{1e5c}', "R"), + ('\u{1e5d}', "r"), + ('\u{1e5e}', "R"), + ('\u{1e5f}', "r"), + ('\u{1e60}', "S"), + ('\u{1e61}', "s"), + ('\u{1e62}', "S"), + ('\u{1e63}', "s"), + ('\u{1e64}', "S"), + ('\u{1e65}', "s"), + ('\u{1e66}', "S"), + ('\u{1e67}', "s"), + ('\u{1e68}', "S"), + ('\u{1e69}', "s"), + ('\u{1e6a}', "T"), + ('\u{1e6b}', "t"), + ('\u{1e6c}', "T"), + ('\u{1e6d}', "t"), + ('\u{1e6e}', "T"), + ('\u{1e6f}', "t"), + ('\u{1e70}', "T"), + ('\u{1e71}', "t"), + ('\u{1e72}', "U"), + ('\u{1e73}', "u"), + ('\u{1e74}', "U"), + ('\u{1e75}', "u"), + ('\u{1e76}', "U"), + ('\u{1e77}', "u"), + ('\u{1e78}', "U"), + ('\u{1e79}', "u"), + ('\u{1e7a}', "U"), + ('\u{1e7b}', "u"), + ('\u{1e7c}', "V"), + ('\u{1e7d}', "v"), + ('\u{1e7e}', "V"), + ('\u{1e7f}', "v"), + ('\u{1e80}', "W"), + ('\u{1e81}', "w"), + ('\u{1e82}', "W"), + ('\u{1e83}', "w"), + ('\u{1e84}', "W"), + ('\u{1e85}', "w"), + ('\u{1e86}', "W"), + ('\u{1e87}', "w"), + ('\u{1e88}', "W"), + ('\u{1e89}', "w"), + ('\u{1e8a}', "X"), + ('\u{1e8b}', "x"), + ('\u{1e8c}', "X"), + ('\u{1e8d}', "x"), + ('\u{1e8e}', "Y"), + ('\u{1e8f}', "y"), + ('\u{1e90}', "Z"), + ('\u{1e91}', "z"), + ('\u{1e92}', "Z"), + ('\u{1e93}', "z"), + ('\u{1e94}', "Z"), + ('\u{1e95}', "z"), + ('\u{1e96}', "h"), + ('\u{1e97}', "t"), + ('\u{1e98}', "w"), + ('\u{1e99}', "y"), + ('\u{1e9b}', "\u{17f}"), + ('\u{1ea0}', "A"), + ('\u{1ea1}', "a"), + ('\u{1ea2}', "A"), + ('\u{1ea3}', "a"), + ('\u{1ea4}', "A"), + ('\u{1ea5}', "a"), + ('\u{1ea6}', "A"), + ('\u{1ea7}', "a"), + ('\u{1ea8}', "A"), + ('\u{1ea9}', "a"), + ('\u{1eaa}', "A"), + ('\u{1eab}', "a"), + ('\u{1eac}', "A"), + ('\u{1ead}', "a"), + ('\u{1eae}', "A"), + ('\u{1eaf}', "a"), + ('\u{1eb0}', "A"), + ('\u{1eb1}', "a"), + ('\u{1eb2}', "A"), + ('\u{1eb3}', "a"), + ('\u{1eb4}', "A"), + ('\u{1eb5}', "a"), + ('\u{1eb6}', "A"), + ('\u{1eb7}', "a"), + ('\u{1eb8}', "E"), + ('\u{1eb9}', "e"), + ('\u{1eba}', "E"), + ('\u{1ebb}', "e"), + ('\u{1ebc}', "E"), + ('\u{1ebd}', "e"), + ('\u{1ebe}', "E"), + ('\u{1ebf}', "e"), + ('\u{1ec0}', "E"), + ('\u{1ec1}', "e"), + ('\u{1ec2}', "E"), + ('\u{1ec3}', "e"), + ('\u{1ec4}', "E"), + ('\u{1ec5}', "e"), + ('\u{1ec6}', "E"), + ('\u{1ec7}', "e"), + ('\u{1ec8}', "I"), + ('\u{1ec9}', "i"), + ('\u{1eca}', "I"), + ('\u{1ecb}', "i"), + ('\u{1ecc}', "O"), + ('\u{1ecd}', "o"), + ('\u{1ece}', "O"), + ('\u{1ecf}', "o"), + ('\u{1ed0}', "O"), + ('\u{1ed1}', "o"), + ('\u{1ed2}', "O"), + ('\u{1ed3}', "o"), + ('\u{1ed4}', "O"), + ('\u{1ed5}', "o"), + ('\u{1ed6}', "O"), + ('\u{1ed7}', "o"), + ('\u{1ed8}', "O"), + ('\u{1ed9}', "o"), + ('\u{1eda}', "O"), + ('\u{1edb}', "o"), + ('\u{1edc}', "O"), + ('\u{1edd}', "o"), + ('\u{1ede}', "O"), + ('\u{1edf}', "o"), + ('\u{1ee0}', "O"), + ('\u{1ee1}', "o"), + ('\u{1ee2}', "O"), + ('\u{1ee3}', "o"), + ('\u{1ee4}', "U"), + ('\u{1ee5}', "u"), + ('\u{1ee6}', "U"), + ('\u{1ee7}', "u"), + ('\u{1ee8}', "U"), + ('\u{1ee9}', "u"), + ('\u{1eea}', "U"), + ('\u{1eeb}', "u"), + ('\u{1eec}', "U"), + ('\u{1eed}', "u"), + ('\u{1eee}', "U"), + ('\u{1eef}', "u"), + ('\u{1ef0}', "U"), + ('\u{1ef1}', "u"), + ('\u{1ef2}', "Y"), + ('\u{1ef3}', "y"), + ('\u{1ef4}', "Y"), + ('\u{1ef5}', "y"), + ('\u{1ef6}', "Y"), + ('\u{1ef7}', "y"), + ('\u{1ef8}', "Y"), + ('\u{1ef9}', "y"), + ('\u{212a}', "K"), + ('\u{212b}', "A"), +]; diff --git a/packages/cli-rs/parity/deviations/DLG35-select-text-units.md b/packages/cli-rs/parity/deviations/DLG35-select-text-units.md index 286f27af71d..6ff4fe2b240 100644 --- a/packages/cli-rs/parity/deviations/DLG35-select-text-units.md +++ b/packages/cli-rs/parity/deviations/DLG35-select-text-units.md @@ -1,20 +1,17 @@ -# DLG35: select filtering and truncation on accented and non-BMP text +# DLG35: select truncation on non-BMP text ## TypeScript -`DialogSelect` filters with `fuzzysort` (`packages/tui/src/ui/dialog-select.tsx:223-229`). Before it compares, -`fuzzysort` decomposes Latin letters and drops their accents (`remove_accents`), so `é` matches `e`. Titles, -details, and paths are cut with `Locale.truncate`, `truncateLeft`, and `truncateMiddle` +`DialogSelect` cuts titles, details, and paths with `Locale.truncate`, `truncateLeft`, and `truncateMiddle` (`packages/tui/src/util/locale.ts:31-79`), which count UTF-16 code units. ## Rust -`select::fuzzy` compares UTF-16 code units as `fuzzysort` does, but keeps accents: `é` does not match `e`. The -truncation helpers count Unicode scalar values, so a title with characters outside the Basic Multilingual Plane, -such as emoji, keeps one more character before the `…` than in TypeScript. +The truncation helpers count Unicode scalar values, so a title with characters outside the Basic Multilingual Plane, +such as emoji, keeps one more character before the `…` than in TypeScript. Filtering matches: `fuzzy` compares +UTF-16 code units and drops accents from Latin letters as `fuzzysort`'s `remove_accents` does, from the table +`script/gen/accents.ts` writes. ## Reason -Accent folding needs Unicode normalization, which the workspace does not depend on yet. Counting code units -would cut a surrogate pair in half, which TypeScript does and shows as a broken character. P-ORACLE -`U.3/fuzzy` (48 queries over the 244 keybind descriptions, all ASCII) gives the same order on both sides. +Counting code units would cut a surrogate pair in half, which TypeScript does and shows as a broken character. diff --git a/packages/cli-rs/script/gen/accents.ts b/packages/cli-rs/script/gen/accents.ts new file mode 100644 index 00000000000..34de8702316 --- /dev/null +++ b/packages/cli-rs/script/gen/accents.ts @@ -0,0 +1,27 @@ +// Writes crates/ui/src/fuzzy/accents.rs: what fuzzysort's `remove_accents` makes of each character it changes, from +// this runtime's Unicode data. It decomposes Latin letters (NFD) and drops the marks U+0300 to U+036F. Marks are not +// Latin, so a run of Latin letters decomposes letter by letter. Sorted by character for a binary search. +const removeAccents = (text: string) => + text.replace(/\p{Script=Latin}+/gu, (match) => match.normalize("NFD")).replace(/[\u0300-\u036f]/g, "") + +const entries = Array.from({ length: 0x110000 }, (_, code) => code) + .filter((code) => code < 0xd800 || code > 0xdfff) + .map((code) => [String.fromCodePoint(code), removeAccents(String.fromCodePoint(code))] as const) + .filter(([char, folded]) => char !== folded) +const escape = (text: string) => + Array.from(text, (char) => + /[ -~]/.test(char) && !`'"\\`.includes(char) ? char : `\\u{${char.codePointAt(0)!.toString(16)}}`, + ).join("") +const lines = [ + "// Generated by script/gen/accents.ts from fuzzysort's `remove_accents`. Do not edit.", + "", + "/// `(character, folded)`, sorted by character.", + "pub const FOLD: &[(char, &str)] = &[", + ...entries.map(([char, folded]) => ` ('${escape(char)}', "${escape(folded)}"),`), + "];", + "", +] + +const file = new URL("../../crates/ui/src/fuzzy/accents.rs", import.meta.url).pathname +await Bun.write(file, lines.join("\n")) +console.log(`accents: ${entries.length} characters`)