fix(cli-rs): fold Latin accents before fuzzy matching

This commit is contained in:
Simon Klee
2026-11-06 23:50:39 +01:00
Unverified
parent ea0a143574
commit c70bf1d019
4 changed files with 702 additions and 14 deletions
+52 -4
View File
@@ -1,6 +1,8 @@
//! A port of `fuzzysort` 3.1's `go` with `keys`, `scoreFn`, and `threshold`, as `DialogSelect` calls it: the same
//! scores and the same order, ties included. Strings are compared in UTF-16 code units, as JavaScript does.
//! Accents are not removed from Latin letters (TS `remove_accents`).
//! scores and the same order, ties included. Strings are compared in UTF-16 code units, as JavaScript does, after
//! Latin letters lose their accents (`remove_accents`).
mod accents;
const NEG: f64 = f64::NEG_INFINITY;
@@ -152,7 +154,7 @@ fn denormalize(score: f64) -> f64 {
}
fn lower_info(text: &str) -> (Vec<u16>, u32, bool) {
let lower: Vec<u16> = text.to_lowercase().encode_utf16().collect();
let lower: Vec<u16> = remove_accents(text).to_lowercase().encode_utf16().collect();
let mut bitflags = 0u32;
let mut space = false;
for &code in &lower {
@@ -193,8 +195,23 @@ fn prepare(target: &str) -> Prepared {
Prepared { lower, bitflags, next: next_beginnings(target) }
}
/// `remove_accents`: Latin letters decomposed, and the marks U+0300 to U+036F dropped.
fn remove_accents(text: &str) -> std::borrow::Cow<'_, str> {
if text.is_ascii() {
return text.into();
}
let folded = text.chars().fold(String::with_capacity(text.len()), |mut out, char| {
match accents::FOLD.binary_search_by_key(&char, |(from, _)| *from) {
Ok(index) => out.push_str(accents::FOLD[index].1),
Err(_) => out.push(char),
}
out
});
folded.into()
}
fn next_beginnings(target: &str) -> Vec<usize> {
let codes: Vec<u16> = target.encode_utf16().collect();
let codes: Vec<u16> = remove_accents(target).encode_utf16().collect();
let mut beginnings = Vec::new();
let (mut was_upper, mut was_alnum) = (false, false);
for (i, &code) in codes.iter().enumerate() {
@@ -476,3 +493,34 @@ impl Heap {
self.items[a] = v;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accents_fold_before_matching_as_remove_accents_does() {
let score = |search: &str, target: &str| {
let seen = std::cell::Cell::new(None);
let keep = |_: &[f64; 1], score, _| {
seen.set(Some(score));
score
};
go_scored(search, &[target], |target| [Some(*target)], keep, 0.0, 1);
seen.get()
};
// An accented letter, a decomposed one, and an accented search score as their plain forms.
for (search, target, plain) in [
("cafe", "Café crème", "Cafe creme"),
("ang", "Ångström", "Angstrom"),
("cafe", "Cafe\u{301}", "Cafe"),
("crème", "cream creme", "cream creme"),
] {
assert_eq!(score(search, target), score(search, plain), "{search:?} in {target:?}");
assert!(score(search, target).is_some(), "{search:?} in {target:?}");
}
// Letters without a decomposition and other scripts stay.
assert_eq!(score("o", "Ø"), None);
assert_eq!(score("α", "ά"), None);
}
}
@@ -0,0 +1,616 @@
// Generated by script/gen/accents.ts from fuzzysort's `remove_accents`. Do not edit.
/// `(character, folded)`, sorted by character.
pub const FOLD: &[(char, &str)] = &[
('\u{c0}', "A"),
('\u{c1}', "A"),
('\u{c2}', "A"),
('\u{c3}', "A"),
('\u{c4}', "A"),
('\u{c5}', "A"),
('\u{c7}', "C"),
('\u{c8}', "E"),
('\u{c9}', "E"),
('\u{ca}', "E"),
('\u{cb}', "E"),
('\u{cc}', "I"),
('\u{cd}', "I"),
('\u{ce}', "I"),
('\u{cf}', "I"),
('\u{d1}', "N"),
('\u{d2}', "O"),
('\u{d3}', "O"),
('\u{d4}', "O"),
('\u{d5}', "O"),
('\u{d6}', "O"),
('\u{d9}', "U"),
('\u{da}', "U"),
('\u{db}', "U"),
('\u{dc}', "U"),
('\u{dd}', "Y"),
('\u{e0}', "a"),
('\u{e1}', "a"),
('\u{e2}', "a"),
('\u{e3}', "a"),
('\u{e4}', "a"),
('\u{e5}', "a"),
('\u{e7}', "c"),
('\u{e8}', "e"),
('\u{e9}', "e"),
('\u{ea}', "e"),
('\u{eb}', "e"),
('\u{ec}', "i"),
('\u{ed}', "i"),
('\u{ee}', "i"),
('\u{ef}', "i"),
('\u{f1}', "n"),
('\u{f2}', "o"),
('\u{f3}', "o"),
('\u{f4}', "o"),
('\u{f5}', "o"),
('\u{f6}', "o"),
('\u{f9}', "u"),
('\u{fa}', "u"),
('\u{fb}', "u"),
('\u{fc}', "u"),
('\u{fd}', "y"),
('\u{ff}', "y"),
('\u{100}', "A"),
('\u{101}', "a"),
('\u{102}', "A"),
('\u{103}', "a"),
('\u{104}', "A"),
('\u{105}', "a"),
('\u{106}', "C"),
('\u{107}', "c"),
('\u{108}', "C"),
('\u{109}', "c"),
('\u{10a}', "C"),
('\u{10b}', "c"),
('\u{10c}', "C"),
('\u{10d}', "c"),
('\u{10e}', "D"),
('\u{10f}', "d"),
('\u{112}', "E"),
('\u{113}', "e"),
('\u{114}', "E"),
('\u{115}', "e"),
('\u{116}', "E"),
('\u{117}', "e"),
('\u{118}', "E"),
('\u{119}', "e"),
('\u{11a}', "E"),
('\u{11b}', "e"),
('\u{11c}', "G"),
('\u{11d}', "g"),
('\u{11e}', "G"),
('\u{11f}', "g"),
('\u{120}', "G"),
('\u{121}', "g"),
('\u{122}', "G"),
('\u{123}', "g"),
('\u{124}', "H"),
('\u{125}', "h"),
('\u{128}', "I"),
('\u{129}', "i"),
('\u{12a}', "I"),
('\u{12b}', "i"),
('\u{12c}', "I"),
('\u{12d}', "i"),
('\u{12e}', "I"),
('\u{12f}', "i"),
('\u{130}', "I"),
('\u{134}', "J"),
('\u{135}', "j"),
('\u{136}', "K"),
('\u{137}', "k"),
('\u{139}', "L"),
('\u{13a}', "l"),
('\u{13b}', "L"),
('\u{13c}', "l"),
('\u{13d}', "L"),
('\u{13e}', "l"),
('\u{143}', "N"),
('\u{144}', "n"),
('\u{145}', "N"),
('\u{146}', "n"),
('\u{147}', "N"),
('\u{148}', "n"),
('\u{14c}', "O"),
('\u{14d}', "o"),
('\u{14e}', "O"),
('\u{14f}', "o"),
('\u{150}', "O"),
('\u{151}', "o"),
('\u{154}', "R"),
('\u{155}', "r"),
('\u{156}', "R"),
('\u{157}', "r"),
('\u{158}', "R"),
('\u{159}', "r"),
('\u{15a}', "S"),
('\u{15b}', "s"),
('\u{15c}', "S"),
('\u{15d}', "s"),
('\u{15e}', "S"),
('\u{15f}', "s"),
('\u{160}', "S"),
('\u{161}', "s"),
('\u{162}', "T"),
('\u{163}', "t"),
('\u{164}', "T"),
('\u{165}', "t"),
('\u{168}', "U"),
('\u{169}', "u"),
('\u{16a}', "U"),
('\u{16b}', "u"),
('\u{16c}', "U"),
('\u{16d}', "u"),
('\u{16e}', "U"),
('\u{16f}', "u"),
('\u{170}', "U"),
('\u{171}', "u"),
('\u{172}', "U"),
('\u{173}', "u"),
('\u{174}', "W"),
('\u{175}', "w"),
('\u{176}', "Y"),
('\u{177}', "y"),
('\u{178}', "Y"),
('\u{179}', "Z"),
('\u{17a}', "z"),
('\u{17b}', "Z"),
('\u{17c}', "z"),
('\u{17d}', "Z"),
('\u{17e}', "z"),
('\u{1a0}', "O"),
('\u{1a1}', "o"),
('\u{1af}', "U"),
('\u{1b0}', "u"),
('\u{1cd}', "A"),
('\u{1ce}', "a"),
('\u{1cf}', "I"),
('\u{1d0}', "i"),
('\u{1d1}', "O"),
('\u{1d2}', "o"),
('\u{1d3}', "U"),
('\u{1d4}', "u"),
('\u{1d5}', "U"),
('\u{1d6}', "u"),
('\u{1d7}', "U"),
('\u{1d8}', "u"),
('\u{1d9}', "U"),
('\u{1da}', "u"),
('\u{1db}', "U"),
('\u{1dc}', "u"),
('\u{1de}', "A"),
('\u{1df}', "a"),
('\u{1e0}', "A"),
('\u{1e1}', "a"),
('\u{1e2}', "\u{c6}"),
('\u{1e3}', "\u{e6}"),
('\u{1e6}', "G"),
('\u{1e7}', "g"),
('\u{1e8}', "K"),
('\u{1e9}', "k"),
('\u{1ea}', "O"),
('\u{1eb}', "o"),
('\u{1ec}', "O"),
('\u{1ed}', "o"),
('\u{1ee}', "\u{1b7}"),
('\u{1ef}', "\u{292}"),
('\u{1f0}', "j"),
('\u{1f4}', "G"),
('\u{1f5}', "g"),
('\u{1f8}', "N"),
('\u{1f9}', "n"),
('\u{1fa}', "A"),
('\u{1fb}', "a"),
('\u{1fc}', "\u{c6}"),
('\u{1fd}', "\u{e6}"),
('\u{1fe}', "\u{d8}"),
('\u{1ff}', "\u{f8}"),
('\u{200}', "A"),
('\u{201}', "a"),
('\u{202}', "A"),
('\u{203}', "a"),
('\u{204}', "E"),
('\u{205}', "e"),
('\u{206}', "E"),
('\u{207}', "e"),
('\u{208}', "I"),
('\u{209}', "i"),
('\u{20a}', "I"),
('\u{20b}', "i"),
('\u{20c}', "O"),
('\u{20d}', "o"),
('\u{20e}', "O"),
('\u{20f}', "o"),
('\u{210}', "R"),
('\u{211}', "r"),
('\u{212}', "R"),
('\u{213}', "r"),
('\u{214}', "U"),
('\u{215}', "u"),
('\u{216}', "U"),
('\u{217}', "u"),
('\u{218}', "S"),
('\u{219}', "s"),
('\u{21a}', "T"),
('\u{21b}', "t"),
('\u{21e}', "H"),
('\u{21f}', "h"),
('\u{226}', "A"),
('\u{227}', "a"),
('\u{228}', "E"),
('\u{229}', "e"),
('\u{22a}', "O"),
('\u{22b}', "o"),
('\u{22c}', "O"),
('\u{22d}', "o"),
('\u{22e}', "O"),
('\u{22f}', "o"),
('\u{230}', "O"),
('\u{231}', "o"),
('\u{232}', "Y"),
('\u{233}', "y"),
('\u{300}', ""),
('\u{301}', ""),
('\u{302}', ""),
('\u{303}', ""),
('\u{304}', ""),
('\u{305}', ""),
('\u{306}', ""),
('\u{307}', ""),
('\u{308}', ""),
('\u{309}', ""),
('\u{30a}', ""),
('\u{30b}', ""),
('\u{30c}', ""),
('\u{30d}', ""),
('\u{30e}', ""),
('\u{30f}', ""),
('\u{310}', ""),
('\u{311}', ""),
('\u{312}', ""),
('\u{313}', ""),
('\u{314}', ""),
('\u{315}', ""),
('\u{316}', ""),
('\u{317}', ""),
('\u{318}', ""),
('\u{319}', ""),
('\u{31a}', ""),
('\u{31b}', ""),
('\u{31c}', ""),
('\u{31d}', ""),
('\u{31e}', ""),
('\u{31f}', ""),
('\u{320}', ""),
('\u{321}', ""),
('\u{322}', ""),
('\u{323}', ""),
('\u{324}', ""),
('\u{325}', ""),
('\u{326}', ""),
('\u{327}', ""),
('\u{328}', ""),
('\u{329}', ""),
('\u{32a}', ""),
('\u{32b}', ""),
('\u{32c}', ""),
('\u{32d}', ""),
('\u{32e}', ""),
('\u{32f}', ""),
('\u{330}', ""),
('\u{331}', ""),
('\u{332}', ""),
('\u{333}', ""),
('\u{334}', ""),
('\u{335}', ""),
('\u{336}', ""),
('\u{337}', ""),
('\u{338}', ""),
('\u{339}', ""),
('\u{33a}', ""),
('\u{33b}', ""),
('\u{33c}', ""),
('\u{33d}', ""),
('\u{33e}', ""),
('\u{33f}', ""),
('\u{340}', ""),
('\u{341}', ""),
('\u{342}', ""),
('\u{343}', ""),
('\u{344}', ""),
('\u{345}', ""),
('\u{346}', ""),
('\u{347}', ""),
('\u{348}', ""),
('\u{349}', ""),
('\u{34a}', ""),
('\u{34b}', ""),
('\u{34c}', ""),
('\u{34d}', ""),
('\u{34e}', ""),
('\u{34f}', ""),
('\u{350}', ""),
('\u{351}', ""),
('\u{352}', ""),
('\u{353}', ""),
('\u{354}', ""),
('\u{355}', ""),
('\u{356}', ""),
('\u{357}', ""),
('\u{358}', ""),
('\u{359}', ""),
('\u{35a}', ""),
('\u{35b}', ""),
('\u{35c}', ""),
('\u{35d}', ""),
('\u{35e}', ""),
('\u{35f}', ""),
('\u{360}', ""),
('\u{361}', ""),
('\u{362}', ""),
('\u{363}', ""),
('\u{364}', ""),
('\u{365}', ""),
('\u{366}', ""),
('\u{367}', ""),
('\u{368}', ""),
('\u{369}', ""),
('\u{36a}', ""),
('\u{36b}', ""),
('\u{36c}', ""),
('\u{36d}', ""),
('\u{36e}', ""),
('\u{36f}', ""),
('\u{1e00}', "A"),
('\u{1e01}', "a"),
('\u{1e02}', "B"),
('\u{1e03}', "b"),
('\u{1e04}', "B"),
('\u{1e05}', "b"),
('\u{1e06}', "B"),
('\u{1e07}', "b"),
('\u{1e08}', "C"),
('\u{1e09}', "c"),
('\u{1e0a}', "D"),
('\u{1e0b}', "d"),
('\u{1e0c}', "D"),
('\u{1e0d}', "d"),
('\u{1e0e}', "D"),
('\u{1e0f}', "d"),
('\u{1e10}', "D"),
('\u{1e11}', "d"),
('\u{1e12}', "D"),
('\u{1e13}', "d"),
('\u{1e14}', "E"),
('\u{1e15}', "e"),
('\u{1e16}', "E"),
('\u{1e17}', "e"),
('\u{1e18}', "E"),
('\u{1e19}', "e"),
('\u{1e1a}', "E"),
('\u{1e1b}', "e"),
('\u{1e1c}', "E"),
('\u{1e1d}', "e"),
('\u{1e1e}', "F"),
('\u{1e1f}', "f"),
('\u{1e20}', "G"),
('\u{1e21}', "g"),
('\u{1e22}', "H"),
('\u{1e23}', "h"),
('\u{1e24}', "H"),
('\u{1e25}', "h"),
('\u{1e26}', "H"),
('\u{1e27}', "h"),
('\u{1e28}', "H"),
('\u{1e29}', "h"),
('\u{1e2a}', "H"),
('\u{1e2b}', "h"),
('\u{1e2c}', "I"),
('\u{1e2d}', "i"),
('\u{1e2e}', "I"),
('\u{1e2f}', "i"),
('\u{1e30}', "K"),
('\u{1e31}', "k"),
('\u{1e32}', "K"),
('\u{1e33}', "k"),
('\u{1e34}', "K"),
('\u{1e35}', "k"),
('\u{1e36}', "L"),
('\u{1e37}', "l"),
('\u{1e38}', "L"),
('\u{1e39}', "l"),
('\u{1e3a}', "L"),
('\u{1e3b}', "l"),
('\u{1e3c}', "L"),
('\u{1e3d}', "l"),
('\u{1e3e}', "M"),
('\u{1e3f}', "m"),
('\u{1e40}', "M"),
('\u{1e41}', "m"),
('\u{1e42}', "M"),
('\u{1e43}', "m"),
('\u{1e44}', "N"),
('\u{1e45}', "n"),
('\u{1e46}', "N"),
('\u{1e47}', "n"),
('\u{1e48}', "N"),
('\u{1e49}', "n"),
('\u{1e4a}', "N"),
('\u{1e4b}', "n"),
('\u{1e4c}', "O"),
('\u{1e4d}', "o"),
('\u{1e4e}', "O"),
('\u{1e4f}', "o"),
('\u{1e50}', "O"),
('\u{1e51}', "o"),
('\u{1e52}', "O"),
('\u{1e53}', "o"),
('\u{1e54}', "P"),
('\u{1e55}', "p"),
('\u{1e56}', "P"),
('\u{1e57}', "p"),
('\u{1e58}', "R"),
('\u{1e59}', "r"),
('\u{1e5a}', "R"),
('\u{1e5b}', "r"),
('\u{1e5c}', "R"),
('\u{1e5d}', "r"),
('\u{1e5e}', "R"),
('\u{1e5f}', "r"),
('\u{1e60}', "S"),
('\u{1e61}', "s"),
('\u{1e62}', "S"),
('\u{1e63}', "s"),
('\u{1e64}', "S"),
('\u{1e65}', "s"),
('\u{1e66}', "S"),
('\u{1e67}', "s"),
('\u{1e68}', "S"),
('\u{1e69}', "s"),
('\u{1e6a}', "T"),
('\u{1e6b}', "t"),
('\u{1e6c}', "T"),
('\u{1e6d}', "t"),
('\u{1e6e}', "T"),
('\u{1e6f}', "t"),
('\u{1e70}', "T"),
('\u{1e71}', "t"),
('\u{1e72}', "U"),
('\u{1e73}', "u"),
('\u{1e74}', "U"),
('\u{1e75}', "u"),
('\u{1e76}', "U"),
('\u{1e77}', "u"),
('\u{1e78}', "U"),
('\u{1e79}', "u"),
('\u{1e7a}', "U"),
('\u{1e7b}', "u"),
('\u{1e7c}', "V"),
('\u{1e7d}', "v"),
('\u{1e7e}', "V"),
('\u{1e7f}', "v"),
('\u{1e80}', "W"),
('\u{1e81}', "w"),
('\u{1e82}', "W"),
('\u{1e83}', "w"),
('\u{1e84}', "W"),
('\u{1e85}', "w"),
('\u{1e86}', "W"),
('\u{1e87}', "w"),
('\u{1e88}', "W"),
('\u{1e89}', "w"),
('\u{1e8a}', "X"),
('\u{1e8b}', "x"),
('\u{1e8c}', "X"),
('\u{1e8d}', "x"),
('\u{1e8e}', "Y"),
('\u{1e8f}', "y"),
('\u{1e90}', "Z"),
('\u{1e91}', "z"),
('\u{1e92}', "Z"),
('\u{1e93}', "z"),
('\u{1e94}', "Z"),
('\u{1e95}', "z"),
('\u{1e96}', "h"),
('\u{1e97}', "t"),
('\u{1e98}', "w"),
('\u{1e99}', "y"),
('\u{1e9b}', "\u{17f}"),
('\u{1ea0}', "A"),
('\u{1ea1}', "a"),
('\u{1ea2}', "A"),
('\u{1ea3}', "a"),
('\u{1ea4}', "A"),
('\u{1ea5}', "a"),
('\u{1ea6}', "A"),
('\u{1ea7}', "a"),
('\u{1ea8}', "A"),
('\u{1ea9}', "a"),
('\u{1eaa}', "A"),
('\u{1eab}', "a"),
('\u{1eac}', "A"),
('\u{1ead}', "a"),
('\u{1eae}', "A"),
('\u{1eaf}', "a"),
('\u{1eb0}', "A"),
('\u{1eb1}', "a"),
('\u{1eb2}', "A"),
('\u{1eb3}', "a"),
('\u{1eb4}', "A"),
('\u{1eb5}', "a"),
('\u{1eb6}', "A"),
('\u{1eb7}', "a"),
('\u{1eb8}', "E"),
('\u{1eb9}', "e"),
('\u{1eba}', "E"),
('\u{1ebb}', "e"),
('\u{1ebc}', "E"),
('\u{1ebd}', "e"),
('\u{1ebe}', "E"),
('\u{1ebf}', "e"),
('\u{1ec0}', "E"),
('\u{1ec1}', "e"),
('\u{1ec2}', "E"),
('\u{1ec3}', "e"),
('\u{1ec4}', "E"),
('\u{1ec5}', "e"),
('\u{1ec6}', "E"),
('\u{1ec7}', "e"),
('\u{1ec8}', "I"),
('\u{1ec9}', "i"),
('\u{1eca}', "I"),
('\u{1ecb}', "i"),
('\u{1ecc}', "O"),
('\u{1ecd}', "o"),
('\u{1ece}', "O"),
('\u{1ecf}', "o"),
('\u{1ed0}', "O"),
('\u{1ed1}', "o"),
('\u{1ed2}', "O"),
('\u{1ed3}', "o"),
('\u{1ed4}', "O"),
('\u{1ed5}', "o"),
('\u{1ed6}', "O"),
('\u{1ed7}', "o"),
('\u{1ed8}', "O"),
('\u{1ed9}', "o"),
('\u{1eda}', "O"),
('\u{1edb}', "o"),
('\u{1edc}', "O"),
('\u{1edd}', "o"),
('\u{1ede}', "O"),
('\u{1edf}', "o"),
('\u{1ee0}', "O"),
('\u{1ee1}', "o"),
('\u{1ee2}', "O"),
('\u{1ee3}', "o"),
('\u{1ee4}', "U"),
('\u{1ee5}', "u"),
('\u{1ee6}', "U"),
('\u{1ee7}', "u"),
('\u{1ee8}', "U"),
('\u{1ee9}', "u"),
('\u{1eea}', "U"),
('\u{1eeb}', "u"),
('\u{1eec}', "U"),
('\u{1eed}', "u"),
('\u{1eee}', "U"),
('\u{1eef}', "u"),
('\u{1ef0}', "U"),
('\u{1ef1}', "u"),
('\u{1ef2}', "Y"),
('\u{1ef3}', "y"),
('\u{1ef4}', "Y"),
('\u{1ef5}', "y"),
('\u{1ef6}', "Y"),
('\u{1ef7}', "y"),
('\u{1ef8}', "Y"),
('\u{1ef9}', "y"),
('\u{212a}', "K"),
('\u{212b}', "A"),
];
@@ -1,20 +1,17 @@
# DLG35: select filtering and truncation on accented and non-BMP text
# DLG35: select truncation on non-BMP text
## TypeScript
`DialogSelect` filters with `fuzzysort` (`packages/tui/src/ui/dialog-select.tsx:223-229`). Before it compares,
`fuzzysort` decomposes Latin letters and drops their accents (`remove_accents`), so `é` matches `e`. Titles,
details, and paths are cut with `Locale.truncate`, `truncateLeft`, and `truncateMiddle`
`DialogSelect` cuts titles, details, and paths with `Locale.truncate`, `truncateLeft`, and `truncateMiddle`
(`packages/tui/src/util/locale.ts:31-79`), which count UTF-16 code units.
## Rust
`select::fuzzy` compares UTF-16 code units as `fuzzysort` does, but keeps accents: `é` does not match `e`. The
truncation helpers count Unicode scalar values, so a title with characters outside the Basic Multilingual Plane,
such as emoji, keeps one more character before the `…` than in TypeScript.
The truncation helpers count Unicode scalar values, so a title with characters outside the Basic Multilingual Plane,
such as emoji, keeps one more character before the `…` than in TypeScript. Filtering matches: `fuzzy` compares
UTF-16 code units and drops accents from Latin letters as `fuzzysort`'s `remove_accents` does, from the table
`script/gen/accents.ts` writes.
## Reason
Accent folding needs Unicode normalization, which the workspace does not depend on yet. Counting code units
would cut a surrogate pair in half, which TypeScript does and shows as a broken character. P-ORACLE
`U.3/fuzzy` (48 queries over the 244 keybind descriptions, all ASCII) gives the same order on both sides.
Counting code units would cut a surrogate pair in half, which TypeScript does and shows as a broken character.
+27
View File
@@ -0,0 +1,27 @@
// Writes crates/ui/src/fuzzy/accents.rs: what fuzzysort's `remove_accents` makes of each character it changes, from
// this runtime's Unicode data. It decomposes Latin letters (NFD) and drops the marks U+0300 to U+036F. Marks are not
// Latin, so a run of Latin letters decomposes letter by letter. Sorted by character for a binary search.
const removeAccents = (text: string) =>
text.replace(/\p{Script=Latin}+/gu, (match) => match.normalize("NFD")).replace(/[\u0300-\u036f]/g, "")
const entries = Array.from({ length: 0x110000 }, (_, code) => code)
.filter((code) => code < 0xd800 || code > 0xdfff)
.map((code) => [String.fromCodePoint(code), removeAccents(String.fromCodePoint(code))] as const)
.filter(([char, folded]) => char !== folded)
const escape = (text: string) =>
Array.from(text, (char) =>
/[ -~]/.test(char) && !`'"\\`.includes(char) ? char : `\\u{${char.codePointAt(0)!.toString(16)}}`,
).join("")
const lines = [
"// Generated by script/gen/accents.ts from fuzzysort's `remove_accents`. Do not edit.",
"",
"/// `(character, folded)`, sorted by character.",
"pub const FOLD: &[(char, &str)] = &[",
...entries.map(([char, folded]) => ` ('${escape(char)}', "${escape(folded)}"),`),
"];",
"",
]
const file = new URL("../../crates/ui/src/fuzzy/accents.rs", import.meta.url).pathname
await Bun.write(file, lines.join("\n"))
console.log(`accents: ${entries.length} characters`)