fix: width-fold fullwidth ASCII deterministically

This commit is contained in:
Dandaedre 2026-09-30 01:50:49 +02:00
parent 215a164a04
commit 861fba5a81
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
10 changed files with 868 additions and 365 deletions

View File

@ -1,5 +1,5 @@
# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.
# Do not edit generated mappings by hand; edit this generator and regenerate.
schema_version = 1
@ -69,11 +69,669 @@ replacement = "-"
rationale = "Replace the small hyphen-minus with reviewed ASCII `-`."
enabled = true
[[replacements]]
id = "ideographic-space"
source = " "
replacement = " "
rationale = "Width-fold the ideographic space to ASCII space."
enabled = true
[[replacements]]
id = "uts39-uff01"
source = "!"
replacement = "!"
rationale = 'Width-fold Unicode fullwidth form U+FF01 to ASCII "!".'
enabled = true
[[replacements]]
id = "uts39-uff02"
source = """
replacement = '"'
rationale = 'Width-fold Unicode fullwidth form U+FF02 to ASCII "\"".'
enabled = true
[[replacements]]
id = "fullwidth-uff03"
source = "#"
replacement = "#"
rationale = 'Width-fold Unicode fullwidth form U+FF03 to ASCII "#".'
enabled = true
[[replacements]]
id = "fullwidth-uff04"
source = "$"
replacement = "$"
rationale = 'Width-fold Unicode fullwidth form U+FF04 to ASCII "$".'
enabled = true
[[replacements]]
id = "fullwidth-uff05"
source = "%"
replacement = "%"
rationale = 'Width-fold Unicode fullwidth form U+FF05 to ASCII "%".'
enabled = true
[[replacements]]
id = "fullwidth-uff06"
source = "&"
replacement = "&"
rationale = 'Width-fold Unicode fullwidth form U+FF06 to ASCII "&".'
enabled = true
[[replacements]]
id = "uts39-uff07"
source = "'"
replacement = "'"
rationale = """Width-fold Unicode fullwidth form U+FF07 to ASCII "'"."""
enabled = true
[[replacements]]
id = "fullwidth-uff08"
source = "("
replacement = "("
rationale = 'Width-fold Unicode fullwidth form U+FF08 to ASCII "(".'
enabled = true
[[replacements]]
id = "fullwidth-uff09"
source = ")"
replacement = ")"
rationale = 'Width-fold Unicode fullwidth form U+FF09 to ASCII ")".'
enabled = true
[[replacements]]
id = "fullwidth-uff0a"
source = "*"
replacement = "*"
rationale = 'Width-fold Unicode fullwidth form U+FF0A to ASCII "*".'
enabled = true
[[replacements]]
id = "fullwidth-uff0b"
source = "+"
replacement = "+"
rationale = 'Width-fold Unicode fullwidth form U+FF0B to ASCII "+".'
enabled = true
[[replacements]]
id = "fullwidth-uff0c"
source = ","
replacement = ","
rationale = 'Width-fold Unicode fullwidth form U+FF0C to ASCII ",".'
enabled = true
[[replacements]]
id = "fullwidth-hyphen-minus"
source = "-"
replacement = "-"
rationale = "Replace the fullwidth hyphen-minus with reviewed ASCII `-`."
rationale = 'Width-fold Unicode fullwidth form U+FF0D to ASCII "-".'
enabled = true
[[replacements]]
id = "fullwidth-uff0e"
source = "."
replacement = "."
rationale = 'Width-fold Unicode fullwidth form U+FF0E to ASCII ".".'
enabled = true
[[replacements]]
id = "fullwidth-uff0f"
source = "/"
replacement = "/"
rationale = 'Width-fold Unicode fullwidth form U+FF0F to ASCII "/".'
enabled = true
[[replacements]]
id = "fullwidth-uff10"
source = "0"
replacement = "0"
rationale = 'Width-fold Unicode fullwidth form U+FF10 to ASCII "0".'
enabled = true
[[replacements]]
id = "fullwidth-uff11"
source = "1"
replacement = "1"
rationale = 'Width-fold Unicode fullwidth form U+FF11 to ASCII "1".'
enabled = true
[[replacements]]
id = "fullwidth-uff12"
source = "2"
replacement = "2"
rationale = 'Width-fold Unicode fullwidth form U+FF12 to ASCII "2".'
enabled = true
[[replacements]]
id = "fullwidth-uff13"
source = "3"
replacement = "3"
rationale = 'Width-fold Unicode fullwidth form U+FF13 to ASCII "3".'
enabled = true
[[replacements]]
id = "fullwidth-uff14"
source = "4"
replacement = "4"
rationale = 'Width-fold Unicode fullwidth form U+FF14 to ASCII "4".'
enabled = true
[[replacements]]
id = "fullwidth-uff15"
source = "5"
replacement = "5"
rationale = 'Width-fold Unicode fullwidth form U+FF15 to ASCII "5".'
enabled = true
[[replacements]]
id = "fullwidth-uff16"
source = "6"
replacement = "6"
rationale = 'Width-fold Unicode fullwidth form U+FF16 to ASCII "6".'
enabled = true
[[replacements]]
id = "fullwidth-uff17"
source = "7"
replacement = "7"
rationale = 'Width-fold Unicode fullwidth form U+FF17 to ASCII "7".'
enabled = true
[[replacements]]
id = "fullwidth-uff18"
source = "8"
replacement = "8"
rationale = 'Width-fold Unicode fullwidth form U+FF18 to ASCII "8".'
enabled = true
[[replacements]]
id = "fullwidth-uff19"
source = "9"
replacement = "9"
rationale = 'Width-fold Unicode fullwidth form U+FF19 to ASCII "9".'
enabled = true
[[replacements]]
id = "uts39-uff1a"
source = ":"
replacement = ":"
rationale = 'Width-fold Unicode fullwidth form U+FF1A to ASCII ":".'
enabled = true
[[replacements]]
id = "fullwidth-uff1b"
source = ";"
replacement = ";"
rationale = 'Width-fold Unicode fullwidth form U+FF1B to ASCII ";".'
enabled = true
[[replacements]]
id = "fullwidth-uff1c"
source = "<"
replacement = "<"
rationale = 'Width-fold Unicode fullwidth form U+FF1C to ASCII "<".'
enabled = true
[[replacements]]
id = "fullwidth-uff1d"
source = "="
replacement = "="
rationale = 'Width-fold Unicode fullwidth form U+FF1D to ASCII "=".'
enabled = true
[[replacements]]
id = "fullwidth-uff1e"
source = ">"
replacement = ">"
rationale = 'Width-fold Unicode fullwidth form U+FF1E to ASCII ">".'
enabled = true
[[replacements]]
id = "fullwidth-uff1f"
source = "?"
replacement = "?"
rationale = 'Width-fold Unicode fullwidth form U+FF1F to ASCII "?".'
enabled = true
[[replacements]]
id = "fullwidth-uff20"
source = "@"
replacement = "@"
rationale = 'Width-fold Unicode fullwidth form U+FF20 to ASCII "@".'
enabled = true
[[replacements]]
id = "uts39-uff21"
source = "A"
replacement = "A"
rationale = 'Width-fold Unicode fullwidth form U+FF21 to ASCII "A".'
enabled = true
[[replacements]]
id = "uts39-uff22"
source = "B"
replacement = "B"
rationale = 'Width-fold Unicode fullwidth form U+FF22 to ASCII "B".'
enabled = true
[[replacements]]
id = "uts39-uff23"
source = "C"
replacement = "C"
rationale = 'Width-fold Unicode fullwidth form U+FF23 to ASCII "C".'
enabled = true
[[replacements]]
id = "fullwidth-uff24"
source = "D"
replacement = "D"
rationale = 'Width-fold Unicode fullwidth form U+FF24 to ASCII "D".'
enabled = true
[[replacements]]
id = "uts39-uff25"
source = "E"
replacement = "E"
rationale = 'Width-fold Unicode fullwidth form U+FF25 to ASCII "E".'
enabled = true
[[replacements]]
id = "fullwidth-uff26"
source = "F"
replacement = "F"
rationale = 'Width-fold Unicode fullwidth form U+FF26 to ASCII "F".'
enabled = true
[[replacements]]
id = "fullwidth-uff27"
source = "G"
replacement = "G"
rationale = 'Width-fold Unicode fullwidth form U+FF27 to ASCII "G".'
enabled = true
[[replacements]]
id = "uts39-uff28"
source = "H"
replacement = "H"
rationale = 'Width-fold Unicode fullwidth form U+FF28 to ASCII "H".'
enabled = true
[[replacements]]
id = "uts39-uff29"
source = "I"
replacement = "I"
rationale = 'Width-fold Unicode fullwidth form U+FF29 to ASCII "I".'
enabled = true
[[replacements]]
id = "uts39-uff2a"
source = "J"
replacement = "J"
rationale = 'Width-fold Unicode fullwidth form U+FF2A to ASCII "J".'
enabled = true
[[replacements]]
id = "uts39-uff2b"
source = "K"
replacement = "K"
rationale = 'Width-fold Unicode fullwidth form U+FF2B to ASCII "K".'
enabled = true
[[replacements]]
id = "fullwidth-uff2c"
source = "L"
replacement = "L"
rationale = 'Width-fold Unicode fullwidth form U+FF2C to ASCII "L".'
enabled = true
[[replacements]]
id = "uts39-uff2d"
source = "M"
replacement = "M"
rationale = 'Width-fold Unicode fullwidth form U+FF2D to ASCII "M".'
enabled = true
[[replacements]]
id = "uts39-uff2e"
source = "N"
replacement = "N"
rationale = 'Width-fold Unicode fullwidth form U+FF2E to ASCII "N".'
enabled = true
[[replacements]]
id = "uts39-uff2f"
source = "O"
replacement = "O"
rationale = 'Width-fold Unicode fullwidth form U+FF2F to ASCII "O".'
enabled = true
[[replacements]]
id = "uts39-uff30"
source = "P"
replacement = "P"
rationale = 'Width-fold Unicode fullwidth form U+FF30 to ASCII "P".'
enabled = true
[[replacements]]
id = "fullwidth-uff31"
source = "Q"
replacement = "Q"
rationale = 'Width-fold Unicode fullwidth form U+FF31 to ASCII "Q".'
enabled = true
[[replacements]]
id = "fullwidth-uff32"
source = "R"
replacement = "R"
rationale = 'Width-fold Unicode fullwidth form U+FF32 to ASCII "R".'
enabled = true
[[replacements]]
id = "uts39-uff33"
source = "S"
replacement = "S"
rationale = 'Width-fold Unicode fullwidth form U+FF33 to ASCII "S".'
enabled = true
[[replacements]]
id = "uts39-uff34"
source = "T"
replacement = "T"
rationale = 'Width-fold Unicode fullwidth form U+FF34 to ASCII "T".'
enabled = true
[[replacements]]
id = "fullwidth-uff35"
source = "U"
replacement = "U"
rationale = 'Width-fold Unicode fullwidth form U+FF35 to ASCII "U".'
enabled = true
[[replacements]]
id = "fullwidth-uff36"
source = "V"
replacement = "V"
rationale = 'Width-fold Unicode fullwidth form U+FF36 to ASCII "V".'
enabled = true
[[replacements]]
id = "fullwidth-uff37"
source = "W"
replacement = "W"
rationale = 'Width-fold Unicode fullwidth form U+FF37 to ASCII "W".'
enabled = true
[[replacements]]
id = "uts39-uff38"
source = "X"
replacement = "X"
rationale = 'Width-fold Unicode fullwidth form U+FF38 to ASCII "X".'
enabled = true
[[replacements]]
id = "uts39-uff39"
source = "Y"
replacement = "Y"
rationale = 'Width-fold Unicode fullwidth form U+FF39 to ASCII "Y".'
enabled = true
[[replacements]]
id = "uts39-uff3a"
source = "Z"
replacement = "Z"
rationale = 'Width-fold Unicode fullwidth form U+FF3A to ASCII "Z".'
enabled = true
[[replacements]]
id = "uts39-uff3b"
source = "["
replacement = "["
rationale = 'Width-fold Unicode fullwidth form U+FF3B to ASCII "[".'
enabled = true
[[replacements]]
id = "uts39-uff3c"
source = "\"
replacement = '\'
rationale = 'Width-fold Unicode fullwidth form U+FF3C to ASCII "\\".'
enabled = true
[[replacements]]
id = "uts39-uff3d"
source = "]"
replacement = "]"
rationale = 'Width-fold Unicode fullwidth form U+FF3D to ASCII "]".'
enabled = true
[[replacements]]
id = "fullwidth-uff3e"
source = "^"
replacement = "^"
rationale = 'Width-fold Unicode fullwidth form U+FF3E to ASCII "^".'
enabled = true
[[replacements]]
id = "fullwidth-uff3f"
source = "_"
replacement = "_"
rationale = 'Width-fold Unicode fullwidth form U+FF3F to ASCII "_".'
enabled = true
[[replacements]]
id = "uts39-uff40"
source = "`"
replacement = "`"
rationale = 'Width-fold Unicode fullwidth form U+FF40 to ASCII "`".'
enabled = true
[[replacements]]
id = "uts39-uff41"
source = "a"
replacement = "a"
rationale = 'Width-fold Unicode fullwidth form U+FF41 to ASCII "a".'
enabled = true
[[replacements]]
id = "fullwidth-uff42"
source = "b"
replacement = "b"
rationale = 'Width-fold Unicode fullwidth form U+FF42 to ASCII "b".'
enabled = true
[[replacements]]
id = "uts39-uff43"
source = "c"
replacement = "c"
rationale = 'Width-fold Unicode fullwidth form U+FF43 to ASCII "c".'
enabled = true
[[replacements]]
id = "fullwidth-uff44"
source = "d"
replacement = "d"
rationale = 'Width-fold Unicode fullwidth form U+FF44 to ASCII "d".'
enabled = true
[[replacements]]
id = "uts39-uff45"
source = "e"
replacement = "e"
rationale = 'Width-fold Unicode fullwidth form U+FF45 to ASCII "e".'
enabled = true
[[replacements]]
id = "fullwidth-uff46"
source = "f"
replacement = "f"
rationale = 'Width-fold Unicode fullwidth form U+FF46 to ASCII "f".'
enabled = true
[[replacements]]
id = "uts39-uff47"
source = "g"
replacement = "g"
rationale = 'Width-fold Unicode fullwidth form U+FF47 to ASCII "g".'
enabled = true
[[replacements]]
id = "uts39-uff48"
source = "h"
replacement = "h"
rationale = 'Width-fold Unicode fullwidth form U+FF48 to ASCII "h".'
enabled = true
[[replacements]]
id = "uts39-uff49"
source = "i"
replacement = "i"
rationale = 'Width-fold Unicode fullwidth form U+FF49 to ASCII "i".'
enabled = true
[[replacements]]
id = "uts39-uff4a"
source = "j"
replacement = "j"
rationale = 'Width-fold Unicode fullwidth form U+FF4A to ASCII "j".'
enabled = true
[[replacements]]
id = "fullwidth-uff4b"
source = "k"
replacement = "k"
rationale = 'Width-fold Unicode fullwidth form U+FF4B to ASCII "k".'
enabled = true
[[replacements]]
id = "uts39-uff4c"
source = "l"
replacement = "l"
rationale = 'Width-fold Unicode fullwidth form U+FF4C to ASCII "l".'
enabled = true
[[replacements]]
id = "fullwidth-uff4d"
source = "m"
replacement = "m"
rationale = 'Width-fold Unicode fullwidth form U+FF4D to ASCII "m".'
enabled = true
[[replacements]]
id = "fullwidth-uff4e"
source = "n"
replacement = "n"
rationale = 'Width-fold Unicode fullwidth form U+FF4E to ASCII "n".'
enabled = true
[[replacements]]
id = "uts39-uff4f"
source = "o"
replacement = "o"
rationale = 'Width-fold Unicode fullwidth form U+FF4F to ASCII "o".'
enabled = true
[[replacements]]
id = "uts39-uff50"
source = "p"
replacement = "p"
rationale = 'Width-fold Unicode fullwidth form U+FF50 to ASCII "p".'
enabled = true
[[replacements]]
id = "fullwidth-uff51"
source = "q"
replacement = "q"
rationale = 'Width-fold Unicode fullwidth form U+FF51 to ASCII "q".'
enabled = true
[[replacements]]
id = "fullwidth-uff52"
source = "r"
replacement = "r"
rationale = 'Width-fold Unicode fullwidth form U+FF52 to ASCII "r".'
enabled = true
[[replacements]]
id = "uts39-uff53"
source = "s"
replacement = "s"
rationale = 'Width-fold Unicode fullwidth form U+FF53 to ASCII "s".'
enabled = true
[[replacements]]
id = "fullwidth-uff54"
source = "t"
replacement = "t"
rationale = 'Width-fold Unicode fullwidth form U+FF54 to ASCII "t".'
enabled = true
[[replacements]]
id = "fullwidth-uff55"
source = "u"
replacement = "u"
rationale = 'Width-fold Unicode fullwidth form U+FF55 to ASCII "u".'
enabled = true
[[replacements]]
id = "uts39-uff56"
source = "v"
replacement = "v"
rationale = 'Width-fold Unicode fullwidth form U+FF56 to ASCII "v".'
enabled = true
[[replacements]]
id = "fullwidth-uff57"
source = "w"
replacement = "w"
rationale = 'Width-fold Unicode fullwidth form U+FF57 to ASCII "w".'
enabled = true
[[replacements]]
id = "uts39-uff58"
source = "x"
replacement = "x"
rationale = 'Width-fold Unicode fullwidth form U+FF58 to ASCII "x".'
enabled = true
[[replacements]]
id = "uts39-uff59"
source = "y"
replacement = "y"
rationale = 'Width-fold Unicode fullwidth form U+FF59 to ASCII "y".'
enabled = true
[[replacements]]
id = "fullwidth-uff5a"
source = "z"
replacement = "z"
rationale = 'Width-fold Unicode fullwidth form U+FF5A to ASCII "z".'
enabled = true
[[replacements]]
id = "fullwidth-uff5b"
source = "{"
replacement = "{"
rationale = 'Width-fold Unicode fullwidth form U+FF5B to ASCII "{".'
enabled = true
[[replacements]]
id = "fullwidth-uff5c"
source = "|"
replacement = "|"
rationale = 'Width-fold Unicode fullwidth form U+FF5C to ASCII "|".'
enabled = true
[[replacements]]
id = "fullwidth-uff5d"
source = "}"
replacement = "}"
rationale = 'Width-fold Unicode fullwidth form U+FF5D to ASCII "}".'
enabled = true
[[replacements]]
id = "fullwidth-uff5e"
source = "~"
replacement = "~"
rationale = 'Width-fold Unicode fullwidth form U+FF5E to ASCII "~".'
enabled = true
[[replacements]]
@ -5263,279 +5921,6 @@ replacement = "o"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FEEC with its printable ASCII skeleton "o".'
enabled = true
[[replacements]]
id = "uts39-uff01"
source = "!"
replacement = "!"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF01 with its printable ASCII skeleton "!".'
enabled = true
[[replacements]]
id = "uts39-uff02"
source = """
replacement = "''"
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF02 with its printable ASCII skeleton "''"."""
enabled = true
[[replacements]]
id = "uts39-uff07"
source = "'"
replacement = "'"
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF07 with its printable ASCII skeleton "'"."""
enabled = true
[[replacements]]
id = "uts39-uff1a"
source = ":"
replacement = ":"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF1A with its printable ASCII skeleton ":".'
enabled = true
[[replacements]]
id = "uts39-uff21"
source = "A"
replacement = "A"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF21 with its printable ASCII skeleton "A".'
enabled = true
[[replacements]]
id = "uts39-uff22"
source = "B"
replacement = "B"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF22 with its printable ASCII skeleton "B".'
enabled = true
[[replacements]]
id = "uts39-uff23"
source = "C"
replacement = "C"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF23 with its printable ASCII skeleton "C".'
enabled = true
[[replacements]]
id = "uts39-uff25"
source = "E"
replacement = "E"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF25 with its printable ASCII skeleton "E".'
enabled = true
[[replacements]]
id = "uts39-uff28"
source = "H"
replacement = "H"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF28 with its printable ASCII skeleton "H".'
enabled = true
[[replacements]]
id = "uts39-uff29"
source = "I"
replacement = "l"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF29 with its printable ASCII skeleton "l".'
enabled = true
[[replacements]]
id = "uts39-uff2a"
source = "J"
replacement = "J"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2A with its printable ASCII skeleton "J".'
enabled = true
[[replacements]]
id = "uts39-uff2b"
source = "K"
replacement = "K"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2B with its printable ASCII skeleton "K".'
enabled = true
[[replacements]]
id = "uts39-uff2d"
source = "M"
replacement = "M"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2D with its printable ASCII skeleton "M".'
enabled = true
[[replacements]]
id = "uts39-uff2e"
source = "N"
replacement = "N"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2E with its printable ASCII skeleton "N".'
enabled = true
[[replacements]]
id = "uts39-uff2f"
source = "O"
replacement = "O"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2F with its printable ASCII skeleton "O".'
enabled = true
[[replacements]]
id = "uts39-uff30"
source = "P"
replacement = "P"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF30 with its printable ASCII skeleton "P".'
enabled = true
[[replacements]]
id = "uts39-uff33"
source = "S"
replacement = "S"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF33 with its printable ASCII skeleton "S".'
enabled = true
[[replacements]]
id = "uts39-uff34"
source = "T"
replacement = "T"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF34 with its printable ASCII skeleton "T".'
enabled = true
[[replacements]]
id = "uts39-uff38"
source = "X"
replacement = "X"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF38 with its printable ASCII skeleton "X".'
enabled = true
[[replacements]]
id = "uts39-uff39"
source = "Y"
replacement = "Y"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF39 with its printable ASCII skeleton "Y".'
enabled = true
[[replacements]]
id = "uts39-uff3a"
source = "Z"
replacement = "Z"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3A with its printable ASCII skeleton "Z".'
enabled = true
[[replacements]]
id = "uts39-uff3b"
source = "["
replacement = "("
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3B with its printable ASCII skeleton "(".'
enabled = true
[[replacements]]
id = "uts39-uff3c"
source = "\"
replacement = '\'
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3C with its printable ASCII skeleton "\\".'
enabled = true
[[replacements]]
id = "uts39-uff3d"
source = "]"
replacement = ")"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3D with its printable ASCII skeleton ")".'
enabled = true
[[replacements]]
id = "uts39-uff40"
source = "`"
replacement = "'"
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF40 with its printable ASCII skeleton "'"."""
enabled = true
[[replacements]]
id = "uts39-uff41"
source = "a"
replacement = "a"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF41 with its printable ASCII skeleton "a".'
enabled = true
[[replacements]]
id = "uts39-uff43"
source = "c"
replacement = "c"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF43 with its printable ASCII skeleton "c".'
enabled = true
[[replacements]]
id = "uts39-uff45"
source = "e"
replacement = "e"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF45 with its printable ASCII skeleton "e".'
enabled = true
[[replacements]]
id = "uts39-uff47"
source = "g"
replacement = "g"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF47 with its printable ASCII skeleton "g".'
enabled = true
[[replacements]]
id = "uts39-uff48"
source = "h"
replacement = "h"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF48 with its printable ASCII skeleton "h".'
enabled = true
[[replacements]]
id = "uts39-uff49"
source = "i"
replacement = "i"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF49 with its printable ASCII skeleton "i".'
enabled = true
[[replacements]]
id = "uts39-uff4a"
source = "j"
replacement = "j"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4A with its printable ASCII skeleton "j".'
enabled = true
[[replacements]]
id = "uts39-uff4c"
source = "l"
replacement = "l"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4C with its printable ASCII skeleton "l".'
enabled = true
[[replacements]]
id = "uts39-uff4f"
source = "o"
replacement = "o"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4F with its printable ASCII skeleton "o".'
enabled = true
[[replacements]]
id = "uts39-uff50"
source = "p"
replacement = "p"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF50 with its printable ASCII skeleton "p".'
enabled = true
[[replacements]]
id = "uts39-uff53"
source = "s"
replacement = "s"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF53 with its printable ASCII skeleton "s".'
enabled = true
[[replacements]]
id = "uts39-uff56"
source = "v"
replacement = "v"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF56 with its printable ASCII skeleton "v".'
enabled = true
[[replacements]]
id = "uts39-uff58"
source = "x"
replacement = "x"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF58 with its printable ASCII skeleton "x".'
enabled = true
[[replacements]]
id = "uts39-uff59"
source = "y"
replacement = "y"
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF59 with its printable ASCII skeleton "y".'
enabled = true
[[replacements]]
id = "uts39-uffe8"
source = "│"

View File

@ -1,4 +1,6 @@
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
//! Regenerate `assets/unicode.toml` from reviewed mappings and Unicode Security.
use std::collections::BTreeSet;
use serde::Serialize;
@ -12,12 +14,6 @@ const CURATED: &[(&str, &str, &str, &str)] = &[
("minus-sign", "−", "-", "mathematical minus sign"),
("small-em-dash", "﹘", "-", "small em dash"),
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
(
"fullwidth-hyphen-minus",
"-",
"-",
"fullwidth hyphen-minus",
),
];
#[derive(Serialize)]
@ -75,25 +71,56 @@ fn main() {
})
.collect::<Vec<_>>();
replacements.push(Replacement {
id: "ideographic-space".to_owned(),
source: '\u{3000}'.to_string(),
replacement: " ".to_owned(),
rationale: "Width-fold the ideographic space to ASCII space.".to_owned(),
enabled: true,
});
for ascii in b'!'..=b'~' {
// U+FF01..=U+FF5E is a contiguous block corresponding to ASCII !..=~.
let value = u32::from(ascii) + 0xfee0;
let Some(fullwidth) = char::from_u32(value) else {
continue;
};
let id = if ascii == b'-' {
// Preserve the public ID shipped before the complete width-folding set.
"fullwidth-hyphen-minus".to_owned()
} else if printable_ascii_skeleton(fullwidth).is_some() {
// Preserve IDs already shipped as generated UTS #39 mappings.
format!("uts39-u{value:04x}")
} else {
format!("fullwidth-u{value:04x}")
};
let replacement = char::from(ascii).to_string();
replacements.push(Replacement {
id,
source: fullwidth.to_string(),
rationale: format!(
"Width-fold Unicode fullwidth form U+{value:04X} to ASCII {replacement:?}."
),
replacement,
enabled: true,
});
}
let reviewed_sources = replacements
.iter()
.filter_map(|rule| rule.source.chars().next())
.collect::<BTreeSet<_>>();
for value in 0..=char::MAX as u32 {
let Some(character) = char::from_u32(value) else {
continue;
};
if character.is_ascii()
|| CURATED
.iter()
.any(|(_, source, _, _)| source.starts_with(character))
{
if character.is_ascii() || reviewed_sources.contains(&character) {
continue;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if skeleton.is_empty()
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|| skeleton == source
{
let Some(skeleton) = printable_ascii_skeleton(character) else {
continue;
}
};
replacements.push(Replacement {
id: format!("uts39-u{value:04x}"),
source,
@ -149,8 +176,17 @@ fn main() {
println!(
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.\n\
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
);
}
fn printable_ascii_skeleton(character: char) -> Option<String> {
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
(!skeleton.is_empty()
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
&& skeleton != source)
.then_some(skeleton)
}

View File

@ -1383,7 +1383,7 @@ mod tests {
.map(|rule| rule.source.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(config.replacements.len(), 1_829);
assert_eq!(config.replacements.len(), 1_884);
assert_eq!(config.schema_version, 1);
assert_eq!(ids.len(), config.replacements.len());
assert_eq!(sources.len(), config.replacements.len());
@ -1411,6 +1411,11 @@ mod tests {
continue;
}
let source = character.to_string();
let width_folded = match value {
0x3000 => Some(" ".to_owned()),
0xff01..=0xff5e => char::from_u32(value - 0xfee0).map(|ascii| ascii.to_string()),
_ => None,
};
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if !skeleton.is_empty()
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
@ -1418,13 +1423,33 @@ mod tests {
{
assert_eq!(
replacements.get(source.as_str()),
Some(&skeleton.as_str()),
Some(&width_folded.as_deref().unwrap_or(skeleton.as_str())),
"catalog drift for U+{value:04X}"
);
}
}
}
#[test]
fn embedded_width_folds_preserve_existing_ids_and_cover_new_sources() {
let config = Config::embedded();
let fullwidth_i = config
.replacements
.iter()
.find(|rule| rule.source == "I")
.unwrap();
let fullwidth_number_sign = config
.replacements
.iter()
.find(|rule| rule.source == "#")
.unwrap();
assert_eq!(fullwidth_i.id, "uts39-uff29");
assert_eq!(fullwidth_i.replacement, "I");
assert_eq!(fullwidth_number_sign.id, "fullwidth-uff03");
assert_eq!(fullwidth_number_sign.replacement, "#");
}
#[test]
fn default_input_limits_match_the_accepted_policy() {
let limits = InputLimits::default();
@ -1745,7 +1770,7 @@ weight = 9.5
let config = ConfigLoader::from_paths(paths).load().unwrap();
assert_eq!(config.replacements.len(), 1_830);
assert_eq!(config.replacements.len(), 1_885);
let replacement = config
.replacements
.iter()

View File

@ -38,7 +38,7 @@ pub enum ReplacementError {
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
plan_matching(config, source, |rule| {
rule.enabled
&& (!rule.id.starts_with("uts39-")
&& (!is_bulk_confusable(rule)
|| config.unicode.confusables == ConfusablePolicy::Replace)
})
}
@ -48,10 +48,17 @@ pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec<
return Vec::new();
}
plan_matching(config, source, |rule| {
rule.enabled && rule.id.starts_with("uts39-")
rule.enabled && is_bulk_confusable(rule)
})
}
fn is_bulk_confusable(rule: &ReplacementRule) -> bool {
let mut characters = rule.source.chars();
let is_fullwidth_scalar =
matches!(characters.next(), Some('\u{ff01}'..='\u{ff5e}')) && characters.next().is_none();
rule.id.starts_with("uts39-") && !is_fullwidth_scalar
}
fn plan_matching(
config: &Config,
source: &str,
@ -196,16 +203,28 @@ mod tests {
assert_eq!(edits[4].replacement, "rn");
}
#[test]
fn generated_catalog_width_folds_every_printable_fullwidth_ascii_character() {
let source = std::iter::once('\u{3000}')
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
.collect::<String>();
let expected = (b' '..=b'~').map(char::from).collect::<String>();
let edits = plan(&Config::embedded(), &source);
assert_eq!(edits.len(), 95);
assert_eq!(apply(&source, &edits).unwrap(), expected);
}
#[test]
fn bulk_confusable_policy_preserves_curated_replacements() {
let mut config = Config::embedded();
config.unicode.confusables = ConfusablePolicy::Review;
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1);
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
assert_eq!(plan_confusables_for_review(&config, "а—I ").len(), 1);
config.unicode.confusables = ConfusablePolicy::Off;
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
assert!(plan_confusables_for_review(&config, "а—").is_empty());
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
assert!(plan_confusables_for_review(&config, "а—I ").is_empty());
}
#[test]

View File

@ -158,8 +158,9 @@ confusables = "review" # replace (default), review, or off
```
`replace` enables diagnostics and edits, `review` retains diagnostics without
formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated
punctuation replacements and invisible/control diagnostics are unaffected.
formatter/LSP edits, and `off` suppresses generated lookalikes. The reviewed
punctuation and width-folding replacements and invisible/control diagnostics
are unaffected.
Set `enabled = false` on a known rule or replacement ID to disable it. Because
records merge by ID, do not copy the full embedded table merely to change one
@ -171,8 +172,11 @@ must be unique. When enabled sources share a prefix, the longest matching source
wins at each byte offset, with stable rule ID ordering as the deterministic
tie-break. Edits always refer to the original UTF-8 snapshot.
Generated mapping IDs use `uts39-u<lowercase-codepoint>`, so an individual
entry can still be disabled or overridden through normal merge-by-ID behavior.
Generated confusable IDs use `uts39-u<lowercase-codepoint>`. New fullwidth
mappings use `fullwidth-u<lowercase-codepoint>`; mappings shipped previously
retain their `uts39-*` or `fullwidth-hyphen-minus` IDs for compatibility, while
ideographic space uses `ideographic-space`. Every entry can be disabled or
overridden through normal merge-by-ID behavior.
Maintain the embedded list by editing
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:

View File

@ -31,8 +31,11 @@ locale-aware lookalike table.
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
scalar whose single-character skeleton is non-empty printable ASCII, plus ten
reviewed punctuation mappings. That is 1,829 mappings in the current profile.
scalar whose single-character skeleton is non-empty printable ASCII, plus nine
reviewed punctuation mappings and 95 compatibility width folds. Width folding
takes precedence when a UTS #39 visual skeleton would produce a different
target, such as fullwidth `I` mapping to `I` rather than `l`. That is 1,884
mappings in the current profile.
Each entry has a stable ID, source, target, rationale, and enabled switch, and
the same asset owns the diagnostic metadata.
@ -40,8 +43,8 @@ The default `[unicode] confusables = "replace"` policy emits
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
`review` for identical findings without edits, or `off` to suppress generated
lookalikes. The ten curated punctuation mappings remain active under all three
settings and can still be disabled individually.
lookalikes. The reviewed punctuation and width-folding mappings remain active
under all three settings and can still be disabled individually.
This broad mapping is intentionally more aggressive than Gitea's locale-aware
display warning. Greek, Cyrillic, mathematical, and other characters may be

View File

@ -16,8 +16,8 @@ cargo run --example ascii_replacements
optional path defaults to `examples/fixtures`.
- `plan_replacements` prints the exact UTF-8 byte edits shared by the formatter
and LSP quick fixes.
- `ascii_replacements` builds an opt-in catalog through the public API and maps
fullwidth characters to all 95 printable ASCII targets.
- `ascii_replacements` demonstrates the embedded width-folding catalog across
all 95 printable ASCII targets.
These examples use embedded defaults so their behavior does not depend on a
developer's user configuration. Applications that want normal configuration
@ -52,9 +52,9 @@ as the formatter.
## Printable ASCII replacement showcase
[`ascii_replacements.rs`](ascii_replacements.rs) extends the embedded defaults
in memory with fullwidth mappings for ASCII `!` through `~` and maps ideographic
space to ASCII space. Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
[`ascii_replacements.rs`](ascii_replacements.rs) applies the embedded fullwidth
mappings for ASCII `!` through `~` and maps ideographic space to ASCII space.
Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
and [expected output](ascii-replacement-showcase/ascii.txt) display all 95
printable ASCII targets in one place:
@ -63,17 +63,16 @@ cargo run --example ascii_replacements
```
The example reports 95 planned edits on stderr and writes only the transformed
fixture to stdout. These broad letter, digit, punctuation, and space mappings
are an API/configuration demonstration, not additions to the production
defaults or a recommendation to rewrite arbitrary multilingual text. The
embedded catalog remains limited to individually reviewed conservative rules.
fixture to stdout. These compatibility width folds are production defaults;
broader script-confusable replacements remain controlled separately by the
`[unicode] confusables` policy.
## Expected-failure diagnostic showcase
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
shipped prose signal, while
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
source-placeholder and tutorial-narration rules plus the ten curated
source-placeholder and tutorial-narration rules plus the reviewed
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
right-to-left override. The adjacent example configuration promotes assistant
self-identification from warning to error and lifts the typography occurrence

View File

@ -1,59 +1,22 @@
//! Showcase opt-in replacements for every printable ASCII target.
//! Showcase embedded width folding for every printable ASCII target.
use std::io::Write;
use john_slop::{
Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements,
};
use john_slop::{Config, apply_replacements, plan_replacements};
const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
fn showcase_config() -> Result<Config, ConfigValidationError> {
let mut config = Config::embedded();
let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| {
// U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values.
let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0)
.expect("the fullwidth ASCII range contains valid scalar values");
(fullwidth, char::from(ascii))
}));
for (source, target) in mappings {
let source = source.to_string();
if let Some(rule) = config
.replacements
.iter_mut()
.find(|rule| rule.source == source)
{
rule.replacement = target.to_string();
rule.rationale = format!(
"Override U+{:04X} for the complete printable-ASCII showcase.",
u32::from(source.chars().next().expect("source contains one scalar"))
);
} else {
config.replacements.push(ReplacementRule {
id: format!(
"showcase-fullwidth-{:04x}",
u32::from(source.chars().next().expect("source contains one scalar"))
),
source,
replacement: target.to_string(),
rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."),
enabled: true,
});
}
}
config.validate()?;
Ok(config)
fn showcase_config() -> Config {
Config::embedded()
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = showcase_config()?;
let config = showcase_config();
let edits = plan_replacements(&config, SOURCE);
let formatted = apply_replacements(SOURCE, &edits)?;
eprintln!(
"planned {} opt-in replacements covering ASCII space through tilde",
"planned {} embedded replacements covering ASCII space through tilde",
edits.len()
);
std::io::stdout().write_all(formatted.as_bytes())?;
@ -70,7 +33,7 @@ mod tests {
#[test]
fn showcase_maps_each_printable_ascii_target_once() {
let config = showcase_config().expect("build valid showcase configuration");
let config = showcase_config();
let edits = plan_replacements(&config, SOURCE);
let targets = edits
.iter()

View File

@ -636,6 +636,29 @@ checks, Unicode-catalog reproducibility, and workspace package assembly pass;
coverage is 92.83% (1,579/1,701 lines), and all release metadata and binary
versions agree on `0.1.24`.
## Milestone 25 — Deterministic fullwidth ASCII folding
- [x] Give ideographic space and every fullwidth form in U+FF01..=U+FF5E an
explicit one-to-one printable ASCII replacement before generating broader
UTS #39 skeleton mappings. Preserve all previously shipped fullwidth IDs and
cover the complete sequence through core, formatter-example, and real LSP
formatting tests.
- [ ] Update Unicode/configuration guidance, changelog, release metadata,
coverage, badges, and recap; run all quality, audit, regeneration, and
package gates.
Decision: compatibility width folding takes precedence over visual-confusable
skeletons for this contiguous Unicode block. These 95 reviewed mappings remain
active under `replace`, `review`, and `off`; the bulk policy continues to govern
only `uts39-*` mappings. This prevents visually motivated results such as
fullwidth `I` becoming lowercase `l`, while preserving policy compatibility
and the shared planner used by the CLI, formatter, and LSP.
Acceptance: formatting ideographic space followed by U+FF01..=U+FF5E produces
ASCII space through tilde exactly, with 95 source-ordered edits and identical
standalone/LSP output; the generated catalog remains deterministic; all release
artifacts agree on `0.1.25` before the immutable `v0.1.25` tag is created.
## Deferred by design
- Statistical or model-based authorship classification.

View File

@ -447,6 +447,52 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
assert!(closed["version"].is_null());
}
#[tokio::test]
async fn formatting_width_folds_the_complete_fullwidth_ascii_block() {
let mut harness = Harness::new(None).await;
let uri = "file:///tmp/fullwidth.txt";
let source = std::iter::once('\u{3000}')
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
.collect::<String>();
let expected = (b' '..=b'~').map(char::from).collect::<String>();
harness
.notify("textDocument/didOpen", open_params(uri, 1, &source))
.await;
let _published = harness
.notification("textDocument/publishDiagnostics")
.await;
let formatting = harness
.request(
"textDocument/formatting",
json!({
"textDocument": { "uri": uri },
"options": { "tabSize": 4, "insertSpaces": true }
}),
2,
)
.await;
let mut formatted = source.clone();
for edit in formatting
.result()
.unwrap()
.as_array()
.unwrap()
.iter()
.rev()
{
let start = utf16_position_to_byte(&source, &edit["range"]["start"]);
let end = utf16_position_to_byte(&source, &edit["range"]["end"]);
formatted.replace_range(start..end, edit["newText"].as_str().unwrap());
}
assert_eq!(formatted, expected);
assert_eq!(
apply_replacements(&source, &plan_replacements(&Config::embedded(), &source)).unwrap(),
expected
);
}
#[tokio::test]
async fn configuration_reload_and_concurrent_documents_remain_independent() {
let disabled = r#"