From 861fba5a81ccc2452d0fba3310ffcc5714564565 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Wed, 30 Sep 2026 01:50:49 +0200 Subject: [PATCH] fix: width-fold fullwidth ASCII deterministically --- crates/john-slop-core/assets/unicode.toml | 935 ++++++++++++------ .../examples/generate_unicode_catalog.rs | 74 +- crates/john-slop-core/src/config.rs | 31 +- crates/john-slop-core/src/replacement.rs | 31 +- docs/configuration.md | 12 +- docs/unicode-confusables.md | 11 +- examples/README.md | 19 +- examples/ascii_replacements.rs | 51 +- tasks.md | 23 + tests/lsp.rs | 46 + 10 files changed, 868 insertions(+), 365 deletions(-) diff --git a/crates/john-slop-core/assets/unicode.toml b/crates/john-slop-core/assets/unicode.toml index 45985b5..0a7a835 100644 --- a/crates/john-slop-core/assets/unicode.toml +++ b/crates/john-slop-core/assets/unicode.toml @@ -1,5 +1,5 @@ # Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`. -# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation. +# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding. # Do not edit generated mappings by hand; edit this generator and regenerate. schema_version = 1 @@ -69,11 +69,669 @@ replacement = "-" rationale = "Replace the small hyphen-minus with reviewed ASCII `-`." enabled = true +[[replacements]] +id = "ideographic-space" +source = " " +replacement = " " +rationale = "Width-fold the ideographic space to ASCII space." +enabled = true + +[[replacements]] +id = "uts39-uff01" +source = "!" +replacement = "!" +rationale = 'Width-fold Unicode fullwidth form U+FF01 to ASCII "!".' +enabled = true + +[[replacements]] +id = "uts39-uff02" +source = """ +replacement = '"' +rationale = 'Width-fold Unicode fullwidth form U+FF02 to ASCII "\"".' +enabled = true + +[[replacements]] +id = "fullwidth-uff03" +source = "#" +replacement = "#" +rationale = 'Width-fold Unicode fullwidth form U+FF03 to ASCII "#".' +enabled = true + +[[replacements]] +id = "fullwidth-uff04" +source = "$" +replacement = "$" +rationale = 'Width-fold Unicode fullwidth form U+FF04 to ASCII "$".' +enabled = true + +[[replacements]] +id = "fullwidth-uff05" +source = "%" +replacement = "%" +rationale = 'Width-fold Unicode fullwidth form U+FF05 to ASCII "%".' +enabled = true + +[[replacements]] +id = "fullwidth-uff06" +source = "&" +replacement = "&" +rationale = 'Width-fold Unicode fullwidth form U+FF06 to ASCII "&".' +enabled = true + +[[replacements]] +id = "uts39-uff07" +source = "'" +replacement = "'" +rationale = """Width-fold Unicode fullwidth form U+FF07 to ASCII "'".""" +enabled = true + +[[replacements]] +id = "fullwidth-uff08" +source = "(" +replacement = "(" +rationale = 'Width-fold Unicode fullwidth form U+FF08 to ASCII "(".' +enabled = true + +[[replacements]] +id = "fullwidth-uff09" +source = ")" +replacement = ")" +rationale = 'Width-fold Unicode fullwidth form U+FF09 to ASCII ")".' +enabled = true + +[[replacements]] +id = "fullwidth-uff0a" +source = "*" +replacement = "*" +rationale = 'Width-fold Unicode fullwidth form U+FF0A to ASCII "*".' +enabled = true + +[[replacements]] +id = "fullwidth-uff0b" +source = "+" +replacement = "+" +rationale = 'Width-fold Unicode fullwidth form U+FF0B to ASCII "+".' +enabled = true + +[[replacements]] +id = "fullwidth-uff0c" +source = "," +replacement = "," +rationale = 'Width-fold Unicode fullwidth form U+FF0C to ASCII ",".' +enabled = true + [[replacements]] id = "fullwidth-hyphen-minus" source = "-" replacement = "-" -rationale = "Replace the fullwidth hyphen-minus with reviewed ASCII `-`." +rationale = 'Width-fold Unicode fullwidth form U+FF0D to ASCII "-".' +enabled = true + +[[replacements]] +id = "fullwidth-uff0e" +source = "." +replacement = "." +rationale = 'Width-fold Unicode fullwidth form U+FF0E to ASCII ".".' +enabled = true + +[[replacements]] +id = "fullwidth-uff0f" +source = "/" +replacement = "/" +rationale = 'Width-fold Unicode fullwidth form U+FF0F to ASCII "/".' +enabled = true + +[[replacements]] +id = "fullwidth-uff10" +source = "0" +replacement = "0" +rationale = 'Width-fold Unicode fullwidth form U+FF10 to ASCII "0".' +enabled = true + +[[replacements]] +id = "fullwidth-uff11" +source = "1" +replacement = "1" +rationale = 'Width-fold Unicode fullwidth form U+FF11 to ASCII "1".' +enabled = true + +[[replacements]] +id = "fullwidth-uff12" +source = "2" +replacement = "2" +rationale = 'Width-fold Unicode fullwidth form U+FF12 to ASCII "2".' +enabled = true + +[[replacements]] +id = "fullwidth-uff13" +source = "3" +replacement = "3" +rationale = 'Width-fold Unicode fullwidth form U+FF13 to ASCII "3".' +enabled = true + +[[replacements]] +id = "fullwidth-uff14" +source = "4" +replacement = "4" +rationale = 'Width-fold Unicode fullwidth form U+FF14 to ASCII "4".' +enabled = true + +[[replacements]] +id = "fullwidth-uff15" +source = "5" +replacement = "5" +rationale = 'Width-fold Unicode fullwidth form U+FF15 to ASCII "5".' +enabled = true + +[[replacements]] +id = "fullwidth-uff16" +source = "6" +replacement = "6" +rationale = 'Width-fold Unicode fullwidth form U+FF16 to ASCII "6".' +enabled = true + +[[replacements]] +id = "fullwidth-uff17" +source = "7" +replacement = "7" +rationale = 'Width-fold Unicode fullwidth form U+FF17 to ASCII "7".' +enabled = true + +[[replacements]] +id = "fullwidth-uff18" +source = "8" +replacement = "8" +rationale = 'Width-fold Unicode fullwidth form U+FF18 to ASCII "8".' +enabled = true + +[[replacements]] +id = "fullwidth-uff19" +source = "9" +replacement = "9" +rationale = 'Width-fold Unicode fullwidth form U+FF19 to ASCII "9".' +enabled = true + +[[replacements]] +id = "uts39-uff1a" +source = ":" +replacement = ":" +rationale = 'Width-fold Unicode fullwidth form U+FF1A to ASCII ":".' +enabled = true + +[[replacements]] +id = "fullwidth-uff1b" +source = ";" +replacement = ";" +rationale = 'Width-fold Unicode fullwidth form U+FF1B to ASCII ";".' +enabled = true + +[[replacements]] +id = "fullwidth-uff1c" +source = "<" +replacement = "<" +rationale = 'Width-fold Unicode fullwidth form U+FF1C to ASCII "<".' +enabled = true + +[[replacements]] +id = "fullwidth-uff1d" +source = "=" +replacement = "=" +rationale = 'Width-fold Unicode fullwidth form U+FF1D to ASCII "=".' +enabled = true + +[[replacements]] +id = "fullwidth-uff1e" +source = ">" +replacement = ">" +rationale = 'Width-fold Unicode fullwidth form U+FF1E to ASCII ">".' +enabled = true + +[[replacements]] +id = "fullwidth-uff1f" +source = "?" +replacement = "?" +rationale = 'Width-fold Unicode fullwidth form U+FF1F to ASCII "?".' +enabled = true + +[[replacements]] +id = "fullwidth-uff20" +source = "@" +replacement = "@" +rationale = 'Width-fold Unicode fullwidth form U+FF20 to ASCII "@".' +enabled = true + +[[replacements]] +id = "uts39-uff21" +source = "A" +replacement = "A" +rationale = 'Width-fold Unicode fullwidth form U+FF21 to ASCII "A".' +enabled = true + +[[replacements]] +id = "uts39-uff22" +source = "B" +replacement = "B" +rationale = 'Width-fold Unicode fullwidth form U+FF22 to ASCII "B".' +enabled = true + +[[replacements]] +id = "uts39-uff23" +source = "C" +replacement = "C" +rationale = 'Width-fold Unicode fullwidth form U+FF23 to ASCII "C".' +enabled = true + +[[replacements]] +id = "fullwidth-uff24" +source = "D" +replacement = "D" +rationale = 'Width-fold Unicode fullwidth form U+FF24 to ASCII "D".' +enabled = true + +[[replacements]] +id = "uts39-uff25" +source = "E" +replacement = "E" +rationale = 'Width-fold Unicode fullwidth form U+FF25 to ASCII "E".' +enabled = true + +[[replacements]] +id = "fullwidth-uff26" +source = "F" +replacement = "F" +rationale = 'Width-fold Unicode fullwidth form U+FF26 to ASCII "F".' +enabled = true + +[[replacements]] +id = "fullwidth-uff27" +source = "G" +replacement = "G" +rationale = 'Width-fold Unicode fullwidth form U+FF27 to ASCII "G".' +enabled = true + +[[replacements]] +id = "uts39-uff28" +source = "H" +replacement = "H" +rationale = 'Width-fold Unicode fullwidth form U+FF28 to ASCII "H".' +enabled = true + +[[replacements]] +id = "uts39-uff29" +source = "I" +replacement = "I" +rationale = 'Width-fold Unicode fullwidth form U+FF29 to ASCII "I".' +enabled = true + +[[replacements]] +id = "uts39-uff2a" +source = "J" +replacement = "J" +rationale = 'Width-fold Unicode fullwidth form U+FF2A to ASCII "J".' +enabled = true + +[[replacements]] +id = "uts39-uff2b" +source = "K" +replacement = "K" +rationale = 'Width-fold Unicode fullwidth form U+FF2B to ASCII "K".' +enabled = true + +[[replacements]] +id = "fullwidth-uff2c" +source = "L" +replacement = "L" +rationale = 'Width-fold Unicode fullwidth form U+FF2C to ASCII "L".' +enabled = true + +[[replacements]] +id = "uts39-uff2d" +source = "M" +replacement = "M" +rationale = 'Width-fold Unicode fullwidth form U+FF2D to ASCII "M".' +enabled = true + +[[replacements]] +id = "uts39-uff2e" +source = "N" +replacement = "N" +rationale = 'Width-fold Unicode fullwidth form U+FF2E to ASCII "N".' +enabled = true + +[[replacements]] +id = "uts39-uff2f" +source = "O" +replacement = "O" +rationale = 'Width-fold Unicode fullwidth form U+FF2F to ASCII "O".' +enabled = true + +[[replacements]] +id = "uts39-uff30" +source = "P" +replacement = "P" +rationale = 'Width-fold Unicode fullwidth form U+FF30 to ASCII "P".' +enabled = true + +[[replacements]] +id = "fullwidth-uff31" +source = "Q" +replacement = "Q" +rationale = 'Width-fold Unicode fullwidth form U+FF31 to ASCII "Q".' +enabled = true + +[[replacements]] +id = "fullwidth-uff32" +source = "R" +replacement = "R" +rationale = 'Width-fold Unicode fullwidth form U+FF32 to ASCII "R".' +enabled = true + +[[replacements]] +id = "uts39-uff33" +source = "S" +replacement = "S" +rationale = 'Width-fold Unicode fullwidth form U+FF33 to ASCII "S".' +enabled = true + +[[replacements]] +id = "uts39-uff34" +source = "T" +replacement = "T" +rationale = 'Width-fold Unicode fullwidth form U+FF34 to ASCII "T".' +enabled = true + +[[replacements]] +id = "fullwidth-uff35" +source = "U" +replacement = "U" +rationale = 'Width-fold Unicode fullwidth form U+FF35 to ASCII "U".' +enabled = true + +[[replacements]] +id = "fullwidth-uff36" +source = "V" +replacement = "V" +rationale = 'Width-fold Unicode fullwidth form U+FF36 to ASCII "V".' +enabled = true + +[[replacements]] +id = "fullwidth-uff37" +source = "W" +replacement = "W" +rationale = 'Width-fold Unicode fullwidth form U+FF37 to ASCII "W".' +enabled = true + +[[replacements]] +id = "uts39-uff38" +source = "X" +replacement = "X" +rationale = 'Width-fold Unicode fullwidth form U+FF38 to ASCII "X".' +enabled = true + +[[replacements]] +id = "uts39-uff39" +source = "Y" +replacement = "Y" +rationale = 'Width-fold Unicode fullwidth form U+FF39 to ASCII "Y".' +enabled = true + +[[replacements]] +id = "uts39-uff3a" +source = "Z" +replacement = "Z" +rationale = 'Width-fold Unicode fullwidth form U+FF3A to ASCII "Z".' +enabled = true + +[[replacements]] +id = "uts39-uff3b" +source = "[" +replacement = "[" +rationale = 'Width-fold Unicode fullwidth form U+FF3B to ASCII "[".' +enabled = true + +[[replacements]] +id = "uts39-uff3c" +source = "\" +replacement = '\' +rationale = 'Width-fold Unicode fullwidth form U+FF3C to ASCII "\\".' +enabled = true + +[[replacements]] +id = "uts39-uff3d" +source = "]" +replacement = "]" +rationale = 'Width-fold Unicode fullwidth form U+FF3D to ASCII "]".' +enabled = true + +[[replacements]] +id = "fullwidth-uff3e" +source = "^" +replacement = "^" +rationale = 'Width-fold Unicode fullwidth form U+FF3E to ASCII "^".' +enabled = true + +[[replacements]] +id = "fullwidth-uff3f" +source = "_" +replacement = "_" +rationale = 'Width-fold Unicode fullwidth form U+FF3F to ASCII "_".' +enabled = true + +[[replacements]] +id = "uts39-uff40" +source = "`" +replacement = "`" +rationale = 'Width-fold Unicode fullwidth form U+FF40 to ASCII "`".' +enabled = true + +[[replacements]] +id = "uts39-uff41" +source = "a" +replacement = "a" +rationale = 'Width-fold Unicode fullwidth form U+FF41 to ASCII "a".' +enabled = true + +[[replacements]] +id = "fullwidth-uff42" +source = "b" +replacement = "b" +rationale = 'Width-fold Unicode fullwidth form U+FF42 to ASCII "b".' +enabled = true + +[[replacements]] +id = "uts39-uff43" +source = "c" +replacement = "c" +rationale = 'Width-fold Unicode fullwidth form U+FF43 to ASCII "c".' +enabled = true + +[[replacements]] +id = "fullwidth-uff44" +source = "d" +replacement = "d" +rationale = 'Width-fold Unicode fullwidth form U+FF44 to ASCII "d".' +enabled = true + +[[replacements]] +id = "uts39-uff45" +source = "e" +replacement = "e" +rationale = 'Width-fold Unicode fullwidth form U+FF45 to ASCII "e".' +enabled = true + +[[replacements]] +id = "fullwidth-uff46" +source = "f" +replacement = "f" +rationale = 'Width-fold Unicode fullwidth form U+FF46 to ASCII "f".' +enabled = true + +[[replacements]] +id = "uts39-uff47" +source = "g" +replacement = "g" +rationale = 'Width-fold Unicode fullwidth form U+FF47 to ASCII "g".' +enabled = true + +[[replacements]] +id = "uts39-uff48" +source = "h" +replacement = "h" +rationale = 'Width-fold Unicode fullwidth form U+FF48 to ASCII "h".' +enabled = true + +[[replacements]] +id = "uts39-uff49" +source = "i" +replacement = "i" +rationale = 'Width-fold Unicode fullwidth form U+FF49 to ASCII "i".' +enabled = true + +[[replacements]] +id = "uts39-uff4a" +source = "j" +replacement = "j" +rationale = 'Width-fold Unicode fullwidth form U+FF4A to ASCII "j".' +enabled = true + +[[replacements]] +id = "fullwidth-uff4b" +source = "k" +replacement = "k" +rationale = 'Width-fold Unicode fullwidth form U+FF4B to ASCII "k".' +enabled = true + +[[replacements]] +id = "uts39-uff4c" +source = "l" +replacement = "l" +rationale = 'Width-fold Unicode fullwidth form U+FF4C to ASCII "l".' +enabled = true + +[[replacements]] +id = "fullwidth-uff4d" +source = "m" +replacement = "m" +rationale = 'Width-fold Unicode fullwidth form U+FF4D to ASCII "m".' +enabled = true + +[[replacements]] +id = "fullwidth-uff4e" +source = "n" +replacement = "n" +rationale = 'Width-fold Unicode fullwidth form U+FF4E to ASCII "n".' +enabled = true + +[[replacements]] +id = "uts39-uff4f" +source = "o" +replacement = "o" +rationale = 'Width-fold Unicode fullwidth form U+FF4F to ASCII "o".' +enabled = true + +[[replacements]] +id = "uts39-uff50" +source = "p" +replacement = "p" +rationale = 'Width-fold Unicode fullwidth form U+FF50 to ASCII "p".' +enabled = true + +[[replacements]] +id = "fullwidth-uff51" +source = "q" +replacement = "q" +rationale = 'Width-fold Unicode fullwidth form U+FF51 to ASCII "q".' +enabled = true + +[[replacements]] +id = "fullwidth-uff52" +source = "r" +replacement = "r" +rationale = 'Width-fold Unicode fullwidth form U+FF52 to ASCII "r".' +enabled = true + +[[replacements]] +id = "uts39-uff53" +source = "s" +replacement = "s" +rationale = 'Width-fold Unicode fullwidth form U+FF53 to ASCII "s".' +enabled = true + +[[replacements]] +id = "fullwidth-uff54" +source = "t" +replacement = "t" +rationale = 'Width-fold Unicode fullwidth form U+FF54 to ASCII "t".' +enabled = true + +[[replacements]] +id = "fullwidth-uff55" +source = "u" +replacement = "u" +rationale = 'Width-fold Unicode fullwidth form U+FF55 to ASCII "u".' +enabled = true + +[[replacements]] +id = "uts39-uff56" +source = "v" +replacement = "v" +rationale = 'Width-fold Unicode fullwidth form U+FF56 to ASCII "v".' +enabled = true + +[[replacements]] +id = "fullwidth-uff57" +source = "w" +replacement = "w" +rationale = 'Width-fold Unicode fullwidth form U+FF57 to ASCII "w".' +enabled = true + +[[replacements]] +id = "uts39-uff58" +source = "x" +replacement = "x" +rationale = 'Width-fold Unicode fullwidth form U+FF58 to ASCII "x".' +enabled = true + +[[replacements]] +id = "uts39-uff59" +source = "y" +replacement = "y" +rationale = 'Width-fold Unicode fullwidth form U+FF59 to ASCII "y".' +enabled = true + +[[replacements]] +id = "fullwidth-uff5a" +source = "z" +replacement = "z" +rationale = 'Width-fold Unicode fullwidth form U+FF5A to ASCII "z".' +enabled = true + +[[replacements]] +id = "fullwidth-uff5b" +source = "{" +replacement = "{" +rationale = 'Width-fold Unicode fullwidth form U+FF5B to ASCII "{".' +enabled = true + +[[replacements]] +id = "fullwidth-uff5c" +source = "|" +replacement = "|" +rationale = 'Width-fold Unicode fullwidth form U+FF5C to ASCII "|".' +enabled = true + +[[replacements]] +id = "fullwidth-uff5d" +source = "}" +replacement = "}" +rationale = 'Width-fold Unicode fullwidth form U+FF5D to ASCII "}".' +enabled = true + +[[replacements]] +id = "fullwidth-uff5e" +source = "~" +replacement = "~" +rationale = 'Width-fold Unicode fullwidth form U+FF5E to ASCII "~".' enabled = true [[replacements]] @@ -5263,279 +5921,6 @@ replacement = "o" rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FEEC with its printable ASCII skeleton "o".' enabled = true -[[replacements]] -id = "uts39-uff01" -source = "!" -replacement = "!" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF01 with its printable ASCII skeleton "!".' -enabled = true - -[[replacements]] -id = "uts39-uff02" -source = """ -replacement = "''" -rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF02 with its printable ASCII skeleton "''".""" -enabled = true - -[[replacements]] -id = "uts39-uff07" -source = "'" -replacement = "'" -rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF07 with its printable ASCII skeleton "'".""" -enabled = true - -[[replacements]] -id = "uts39-uff1a" -source = ":" -replacement = ":" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF1A with its printable ASCII skeleton ":".' -enabled = true - -[[replacements]] -id = "uts39-uff21" -source = "A" -replacement = "A" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF21 with its printable ASCII skeleton "A".' -enabled = true - -[[replacements]] -id = "uts39-uff22" -source = "B" -replacement = "B" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF22 with its printable ASCII skeleton "B".' -enabled = true - -[[replacements]] -id = "uts39-uff23" -source = "C" -replacement = "C" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF23 with its printable ASCII skeleton "C".' -enabled = true - -[[replacements]] -id = "uts39-uff25" -source = "E" -replacement = "E" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF25 with its printable ASCII skeleton "E".' -enabled = true - -[[replacements]] -id = "uts39-uff28" -source = "H" -replacement = "H" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF28 with its printable ASCII skeleton "H".' -enabled = true - -[[replacements]] -id = "uts39-uff29" -source = "I" -replacement = "l" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF29 with its printable ASCII skeleton "l".' -enabled = true - -[[replacements]] -id = "uts39-uff2a" -source = "J" -replacement = "J" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2A with its printable ASCII skeleton "J".' -enabled = true - -[[replacements]] -id = "uts39-uff2b" -source = "K" -replacement = "K" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2B with its printable ASCII skeleton "K".' -enabled = true - -[[replacements]] -id = "uts39-uff2d" -source = "M" -replacement = "M" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2D with its printable ASCII skeleton "M".' -enabled = true - -[[replacements]] -id = "uts39-uff2e" -source = "N" -replacement = "N" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2E with its printable ASCII skeleton "N".' -enabled = true - -[[replacements]] -id = "uts39-uff2f" -source = "O" -replacement = "O" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2F with its printable ASCII skeleton "O".' -enabled = true - -[[replacements]] -id = "uts39-uff30" -source = "P" -replacement = "P" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF30 with its printable ASCII skeleton "P".' -enabled = true - -[[replacements]] -id = "uts39-uff33" -source = "S" -replacement = "S" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF33 with its printable ASCII skeleton "S".' -enabled = true - -[[replacements]] -id = "uts39-uff34" -source = "T" -replacement = "T" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF34 with its printable ASCII skeleton "T".' -enabled = true - -[[replacements]] -id = "uts39-uff38" -source = "X" -replacement = "X" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF38 with its printable ASCII skeleton "X".' -enabled = true - -[[replacements]] -id = "uts39-uff39" -source = "Y" -replacement = "Y" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF39 with its printable ASCII skeleton "Y".' -enabled = true - -[[replacements]] -id = "uts39-uff3a" -source = "Z" -replacement = "Z" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3A with its printable ASCII skeleton "Z".' -enabled = true - -[[replacements]] -id = "uts39-uff3b" -source = "[" -replacement = "(" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3B with its printable ASCII skeleton "(".' -enabled = true - -[[replacements]] -id = "uts39-uff3c" -source = "\" -replacement = '\' -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3C with its printable ASCII skeleton "\\".' -enabled = true - -[[replacements]] -id = "uts39-uff3d" -source = "]" -replacement = ")" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3D with its printable ASCII skeleton ")".' -enabled = true - -[[replacements]] -id = "uts39-uff40" -source = "`" -replacement = "'" -rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF40 with its printable ASCII skeleton "'".""" -enabled = true - -[[replacements]] -id = "uts39-uff41" -source = "a" -replacement = "a" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF41 with its printable ASCII skeleton "a".' -enabled = true - -[[replacements]] -id = "uts39-uff43" -source = "c" -replacement = "c" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF43 with its printable ASCII skeleton "c".' -enabled = true - -[[replacements]] -id = "uts39-uff45" -source = "e" -replacement = "e" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF45 with its printable ASCII skeleton "e".' -enabled = true - -[[replacements]] -id = "uts39-uff47" -source = "g" -replacement = "g" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF47 with its printable ASCII skeleton "g".' -enabled = true - -[[replacements]] -id = "uts39-uff48" -source = "h" -replacement = "h" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF48 with its printable ASCII skeleton "h".' -enabled = true - -[[replacements]] -id = "uts39-uff49" -source = "i" -replacement = "i" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF49 with its printable ASCII skeleton "i".' -enabled = true - -[[replacements]] -id = "uts39-uff4a" -source = "j" -replacement = "j" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4A with its printable ASCII skeleton "j".' -enabled = true - -[[replacements]] -id = "uts39-uff4c" -source = "l" -replacement = "l" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4C with its printable ASCII skeleton "l".' -enabled = true - -[[replacements]] -id = "uts39-uff4f" -source = "o" -replacement = "o" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4F with its printable ASCII skeleton "o".' -enabled = true - -[[replacements]] -id = "uts39-uff50" -source = "p" -replacement = "p" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF50 with its printable ASCII skeleton "p".' -enabled = true - -[[replacements]] -id = "uts39-uff53" -source = "s" -replacement = "s" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF53 with its printable ASCII skeleton "s".' -enabled = true - -[[replacements]] -id = "uts39-uff56" -source = "v" -replacement = "v" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF56 with its printable ASCII skeleton "v".' -enabled = true - -[[replacements]] -id = "uts39-uff58" -source = "x" -replacement = "x" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF58 with its printable ASCII skeleton "x".' -enabled = true - -[[replacements]] -id = "uts39-uff59" -source = "y" -replacement = "y" -rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF59 with its printable ASCII skeleton "y".' -enabled = true - [[replacements]] id = "uts39-uffe8" source = "│" diff --git a/crates/john-slop-core/examples/generate_unicode_catalog.rs b/crates/john-slop-core/examples/generate_unicode_catalog.rs index 7964e77..3a51070 100644 --- a/crates/john-slop-core/examples/generate_unicode_catalog.rs +++ b/crates/john-slop-core/examples/generate_unicode_catalog.rs @@ -1,4 +1,6 @@ -//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables. +//! Regenerate `assets/unicode.toml` from reviewed mappings and Unicode Security. + +use std::collections::BTreeSet; use serde::Serialize; @@ -12,12 +14,6 @@ const CURATED: &[(&str, &str, &str, &str)] = &[ ("minus-sign", "−", "-", "mathematical minus sign"), ("small-em-dash", "﹘", "-", "small em dash"), ("small-hyphen-minus", "﹣", "-", "small hyphen-minus"), - ( - "fullwidth-hyphen-minus", - "-", - "-", - "fullwidth hyphen-minus", - ), ]; #[derive(Serialize)] @@ -75,25 +71,56 @@ fn main() { }) .collect::>(); + replacements.push(Replacement { + id: "ideographic-space".to_owned(), + source: '\u{3000}'.to_string(), + replacement: " ".to_owned(), + rationale: "Width-fold the ideographic space to ASCII space.".to_owned(), + enabled: true, + }); + for ascii in b'!'..=b'~' { + // U+FF01..=U+FF5E is a contiguous block corresponding to ASCII !..=~. + let value = u32::from(ascii) + 0xfee0; + let Some(fullwidth) = char::from_u32(value) else { + continue; + }; + let id = if ascii == b'-' { + // Preserve the public ID shipped before the complete width-folding set. + "fullwidth-hyphen-minus".to_owned() + } else if printable_ascii_skeleton(fullwidth).is_some() { + // Preserve IDs already shipped as generated UTS #39 mappings. + format!("uts39-u{value:04x}") + } else { + format!("fullwidth-u{value:04x}") + }; + let replacement = char::from(ascii).to_string(); + replacements.push(Replacement { + id, + source: fullwidth.to_string(), + rationale: format!( + "Width-fold Unicode fullwidth form U+{value:04X} to ASCII {replacement:?}." + ), + replacement, + enabled: true, + }); + } + + let reviewed_sources = replacements + .iter() + .filter_map(|rule| rule.source.chars().next()) + .collect::>(); + for value in 0..=char::MAX as u32 { let Some(character) = char::from_u32(value) else { continue; }; - if character.is_ascii() - || CURATED - .iter() - .any(|(_, source, _, _)| source.starts_with(character)) - { + if character.is_ascii() || reviewed_sources.contains(&character) { continue; } let source = character.to_string(); - let skeleton = unicode_security::skeleton(&source).collect::(); - if skeleton.is_empty() - || !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte)) - || skeleton == source - { + let Some(skeleton) = printable_ascii_skeleton(character) else { continue; - } + }; replacements.push(Replacement { id: format!("uts39-u{value:04x}"), source, @@ -149,8 +176,17 @@ fn main() { println!( "# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\ - # Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\ + # Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.\n\ # Do not edit generated mappings by hand; edit this generator and regenerate.\n{}", toml::to_string_pretty(&catalog).expect("the catalog model must serialize") ); } + +fn printable_ascii_skeleton(character: char) -> Option { + let source = character.to_string(); + let skeleton = unicode_security::skeleton(&source).collect::(); + (!skeleton.is_empty() + && skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte)) + && skeleton != source) + .then_some(skeleton) +} diff --git a/crates/john-slop-core/src/config.rs b/crates/john-slop-core/src/config.rs index 7479a97..0c9c4ac 100644 --- a/crates/john-slop-core/src/config.rs +++ b/crates/john-slop-core/src/config.rs @@ -1383,7 +1383,7 @@ mod tests { .map(|rule| rule.source.as_str()) .collect::>(); - assert_eq!(config.replacements.len(), 1_829); + assert_eq!(config.replacements.len(), 1_884); assert_eq!(config.schema_version, 1); assert_eq!(ids.len(), config.replacements.len()); assert_eq!(sources.len(), config.replacements.len()); @@ -1411,6 +1411,11 @@ mod tests { continue; } let source = character.to_string(); + let width_folded = match value { + 0x3000 => Some(" ".to_owned()), + 0xff01..=0xff5e => char::from_u32(value - 0xfee0).map(|ascii| ascii.to_string()), + _ => None, + }; let skeleton = unicode_security::skeleton(&source).collect::(); if !skeleton.is_empty() && skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte)) @@ -1418,13 +1423,33 @@ mod tests { { assert_eq!( replacements.get(source.as_str()), - Some(&skeleton.as_str()), + Some(&width_folded.as_deref().unwrap_or(skeleton.as_str())), "catalog drift for U+{value:04X}" ); } } } + #[test] + fn embedded_width_folds_preserve_existing_ids_and_cover_new_sources() { + let config = Config::embedded(); + let fullwidth_i = config + .replacements + .iter() + .find(|rule| rule.source == "I") + .unwrap(); + let fullwidth_number_sign = config + .replacements + .iter() + .find(|rule| rule.source == "#") + .unwrap(); + + assert_eq!(fullwidth_i.id, "uts39-uff29"); + assert_eq!(fullwidth_i.replacement, "I"); + assert_eq!(fullwidth_number_sign.id, "fullwidth-uff03"); + assert_eq!(fullwidth_number_sign.replacement, "#"); + } + #[test] fn default_input_limits_match_the_accepted_policy() { let limits = InputLimits::default(); @@ -1745,7 +1770,7 @@ weight = 9.5 let config = ConfigLoader::from_paths(paths).load().unwrap(); - assert_eq!(config.replacements.len(), 1_830); + assert_eq!(config.replacements.len(), 1_885); let replacement = config .replacements .iter() diff --git a/crates/john-slop-core/src/replacement.rs b/crates/john-slop-core/src/replacement.rs index 8f2e009..d94f143 100644 --- a/crates/john-slop-core/src/replacement.rs +++ b/crates/john-slop-core/src/replacement.rs @@ -38,7 +38,7 @@ pub enum ReplacementError { pub fn plan(config: &Config, source: &str) -> Vec { plan_matching(config, source, |rule| { rule.enabled - && (!rule.id.starts_with("uts39-") + && (!is_bulk_confusable(rule) || config.unicode.confusables == ConfusablePolicy::Replace) }) } @@ -48,10 +48,17 @@ pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec< return Vec::new(); } plan_matching(config, source, |rule| { - rule.enabled && rule.id.starts_with("uts39-") + rule.enabled && is_bulk_confusable(rule) }) } +fn is_bulk_confusable(rule: &ReplacementRule) -> bool { + let mut characters = rule.source.chars(); + let is_fullwidth_scalar = + matches!(characters.next(), Some('\u{ff01}'..='\u{ff5e}')) && characters.next().is_none(); + rule.id.starts_with("uts39-") && !is_fullwidth_scalar +} + fn plan_matching( config: &Config, source: &str, @@ -196,16 +203,28 @@ mod tests { assert_eq!(edits[4].replacement, "rn"); } + #[test] + fn generated_catalog_width_folds_every_printable_fullwidth_ascii_character() { + let source = std::iter::once('\u{3000}') + .chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0))) + .collect::(); + let expected = (b' '..=b'~').map(char::from).collect::(); + let edits = plan(&Config::embedded(), &source); + + assert_eq!(edits.len(), 95); + assert_eq!(apply(&source, &edits).unwrap(), expected); + } + #[test] fn bulk_confusable_policy_preserves_curated_replacements() { let mut config = Config::embedded(); config.unicode.confusables = ConfusablePolicy::Review; - assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-"); - assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1); + assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I "); + assert_eq!(plan_confusables_for_review(&config, "а—I ").len(), 1); config.unicode.confusables = ConfusablePolicy::Off; - assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-"); - assert!(plan_confusables_for_review(&config, "а—").is_empty()); + assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I "); + assert!(plan_confusables_for_review(&config, "а—I ").is_empty()); } #[test] diff --git a/docs/configuration.md b/docs/configuration.md index b849e60..fdfaeb7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -158,8 +158,9 @@ confusables = "review" # replace (default), review, or off ``` `replace` enables diagnostics and edits, `review` retains diagnostics without -formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated -punctuation replacements and invisible/control diagnostics are unaffected. +formatter/LSP edits, and `off` suppresses generated lookalikes. The reviewed +punctuation and width-folding replacements and invisible/control diagnostics +are unaffected. Set `enabled = false` on a known rule or replacement ID to disable it. Because records merge by ID, do not copy the full embedded table merely to change one @@ -171,8 +172,11 @@ must be unique. When enabled sources share a prefix, the longest matching source wins at each byte offset, with stable rule ID ordering as the deterministic tie-break. Edits always refer to the original UTF-8 snapshot. -Generated mapping IDs use `uts39-u`, so an individual -entry can still be disabled or overridden through normal merge-by-ID behavior. +Generated confusable IDs use `uts39-u`. New fullwidth +mappings use `fullwidth-u`; mappings shipped previously +retain their `uts39-*` or `fullwidth-hyphen-minus` IDs for compatibility, while +ideographic space uses `ideographic-space`. Every entry can be disabled or +overridden through normal merge-by-ID behavior. Maintain the embedded list by editing `crates/john-slop-core/examples/generate_unicode_catalog.rs` and running: diff --git a/docs/unicode-confusables.md b/docs/unicode-confusables.md index c1b011f..054ccde 100644 --- a/docs/unicode-confusables.md +++ b/docs/unicode-confusables.md @@ -31,8 +31,11 @@ locale-aware lookalike table. John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII -scalar whose single-character skeleton is non-empty printable ASCII, plus ten -reviewed punctuation mappings. That is 1,829 mappings in the current profile. +scalar whose single-character skeleton is non-empty printable ASCII, plus nine +reviewed punctuation mappings and 95 compatibility width folds. Width folding +takes precedence when a UTS #39 visual skeleton would produce a different +target, such as fullwidth `I` mapping to `I` rather than `l`. That is 1,884 +mappings in the current profile. Each entry has a stable ID, source, target, rationale, and enabled switch, and the same asset owns the diagnostic metadata. @@ -40,8 +43,8 @@ The default `[unicode] confusables = "replace"` policy emits `unicode-ascii-replacement` diagnostics and supplies the same edits to CLI output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to `review` for identical findings without edits, or `off` to suppress generated -lookalikes. The ten curated punctuation mappings remain active under all three -settings and can still be disabled individually. +lookalikes. The reviewed punctuation and width-folding mappings remain active +under all three settings and can still be disabled individually. This broad mapping is intentionally more aggressive than Gitea's locale-aware display warning. Greek, Cyrillic, mathematical, and other characters may be diff --git a/examples/README.md b/examples/README.md index fda1ddc..7b90638 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,8 +16,8 @@ cargo run --example ascii_replacements optional path defaults to `examples/fixtures`. - `plan_replacements` prints the exact UTF-8 byte edits shared by the formatter and LSP quick fixes. -- `ascii_replacements` builds an opt-in catalog through the public API and maps - fullwidth characters to all 95 printable ASCII targets. +- `ascii_replacements` demonstrates the embedded width-folding catalog across + all 95 printable ASCII targets. These examples use embedded defaults so their behavior does not depend on a developer's user configuration. Applications that want normal configuration @@ -52,9 +52,9 @@ as the formatter. ## Printable ASCII replacement showcase -[`ascii_replacements.rs`](ascii_replacements.rs) extends the embedded defaults -in memory with fullwidth mappings for ASCII `!` through `~` and maps ideographic -space to ASCII space. Its [Unicode input](ascii-replacement-showcase/fullwidth.txt) +[`ascii_replacements.rs`](ascii_replacements.rs) applies the embedded fullwidth +mappings for ASCII `!` through `~` and maps ideographic space to ASCII space. +Its [Unicode input](ascii-replacement-showcase/fullwidth.txt) and [expected output](ascii-replacement-showcase/ascii.txt) display all 95 printable ASCII targets in one place: @@ -63,17 +63,16 @@ cargo run --example ascii_replacements ``` The example reports 95 planned edits on stderr and writes only the transformed -fixture to stdout. These broad letter, digit, punctuation, and space mappings -are an API/configuration demonstration, not additions to the production -defaults or a recommendation to rewrite arbitrary multilingual text. The -embedded catalog remains limited to individually reviewed conservative rules. +fixture to stdout. These compatibility width folds are production defaults; +broader script-confusable replacements remain controlled separately by the +`[unicode] confusables` policy. ## Expected-failure diagnostic showcase [`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently shipped prose signal, while [`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the -source-placeholder and tutorial-narration rules plus the ten curated +source-placeholder and tutorial-narration rules plus the reviewed punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a right-to-left override. The adjacent example configuration promotes assistant self-identification from warning to error and lifts the typography occurrence diff --git a/examples/ascii_replacements.rs b/examples/ascii_replacements.rs index a71eacf..e754f6d 100644 --- a/examples/ascii_replacements.rs +++ b/examples/ascii_replacements.rs @@ -1,59 +1,22 @@ -//! Showcase opt-in replacements for every printable ASCII target. +//! Showcase embedded width folding for every printable ASCII target. use std::io::Write; -use john_slop::{ - Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements, -}; +use john_slop::{Config, apply_replacements, plan_replacements}; const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt"); -fn showcase_config() -> Result { - let mut config = Config::embedded(); - let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| { - // U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values. - let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0) - .expect("the fullwidth ASCII range contains valid scalar values"); - (fullwidth, char::from(ascii)) - })); - - for (source, target) in mappings { - let source = source.to_string(); - if let Some(rule) = config - .replacements - .iter_mut() - .find(|rule| rule.source == source) - { - rule.replacement = target.to_string(); - rule.rationale = format!( - "Override U+{:04X} for the complete printable-ASCII showcase.", - u32::from(source.chars().next().expect("source contains one scalar")) - ); - } else { - config.replacements.push(ReplacementRule { - id: format!( - "showcase-fullwidth-{:04x}", - u32::from(source.chars().next().expect("source contains one scalar")) - ), - source, - replacement: target.to_string(), - rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."), - enabled: true, - }); - } - } - - config.validate()?; - Ok(config) +fn showcase_config() -> Config { + Config::embedded() } fn main() -> Result<(), Box> { - let config = showcase_config()?; + let config = showcase_config(); let edits = plan_replacements(&config, SOURCE); let formatted = apply_replacements(SOURCE, &edits)?; eprintln!( - "planned {} opt-in replacements covering ASCII space through tilde", + "planned {} embedded replacements covering ASCII space through tilde", edits.len() ); std::io::stdout().write_all(formatted.as_bytes())?; @@ -70,7 +33,7 @@ mod tests { #[test] fn showcase_maps_each_printable_ascii_target_once() { - let config = showcase_config().expect("build valid showcase configuration"); + let config = showcase_config(); let edits = plan_replacements(&config, SOURCE); let targets = edits .iter() diff --git a/tasks.md b/tasks.md index 93aaa14..396ecee 100644 --- a/tasks.md +++ b/tasks.md @@ -636,6 +636,29 @@ checks, Unicode-catalog reproducibility, and workspace package assembly pass; coverage is 92.83% (1,579/1,701 lines), and all release metadata and binary versions agree on `0.1.24`. +## Milestone 25 — Deterministic fullwidth ASCII folding + +- [x] Give ideographic space and every fullwidth form in U+FF01..=U+FF5E an + explicit one-to-one printable ASCII replacement before generating broader + UTS #39 skeleton mappings. Preserve all previously shipped fullwidth IDs and + cover the complete sequence through core, formatter-example, and real LSP + formatting tests. +- [ ] Update Unicode/configuration guidance, changelog, release metadata, + coverage, badges, and recap; run all quality, audit, regeneration, and + package gates. + +Decision: compatibility width folding takes precedence over visual-confusable +skeletons for this contiguous Unicode block. These 95 reviewed mappings remain +active under `replace`, `review`, and `off`; the bulk policy continues to govern +only `uts39-*` mappings. This prevents visually motivated results such as +fullwidth `I` becoming lowercase `l`, while preserving policy compatibility +and the shared planner used by the CLI, formatter, and LSP. + +Acceptance: formatting ideographic space followed by U+FF01..=U+FF5E produces +ASCII space through tilde exactly, with 95 source-ordered edits and identical +standalone/LSP output; the generated catalog remains deterministic; all release +artifacts agree on `0.1.25` before the immutable `v0.1.25` tag is created. + ## Deferred by design - Statistical or model-based authorship classification. diff --git a/tests/lsp.rs b/tests/lsp.rs index 7a1de82..b764f96 100644 --- a/tests/lsp.rs +++ b/tests/lsp.rs @@ -447,6 +447,52 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() { assert!(closed["version"].is_null()); } +#[tokio::test] +async fn formatting_width_folds_the_complete_fullwidth_ascii_block() { + let mut harness = Harness::new(None).await; + let uri = "file:///tmp/fullwidth.txt"; + let source = std::iter::once('\u{3000}') + .chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0))) + .collect::(); + let expected = (b' '..=b'~').map(char::from).collect::(); + harness + .notify("textDocument/didOpen", open_params(uri, 1, &source)) + .await; + let _published = harness + .notification("textDocument/publishDiagnostics") + .await; + + let formatting = harness + .request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": uri }, + "options": { "tabSize": 4, "insertSpaces": true } + }), + 2, + ) + .await; + let mut formatted = source.clone(); + for edit in formatting + .result() + .unwrap() + .as_array() + .unwrap() + .iter() + .rev() + { + let start = utf16_position_to_byte(&source, &edit["range"]["start"]); + let end = utf16_position_to_byte(&source, &edit["range"]["end"]); + formatted.replace_range(start..end, edit["newText"].as_str().unwrap()); + } + + assert_eq!(formatted, expected); + assert_eq!( + apply_replacements(&source, &plan_replacements(&Config::embedded(), &source)).unwrap(), + expected + ); +} + #[tokio::test] async fn configuration_reload_and_concurrent_documents_remain_independent() { let disabled = r#"