fix: width-fold fullwidth ASCII deterministically
This commit is contained in:
parent
215a164a04
commit
861fba5a81
|
|
@ -1,5 +1,5 @@
|
|||
# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.
|
||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.
|
||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.
|
||||
# Do not edit generated mappings by hand; edit this generator and regenerate.
|
||||
schema_version = 1
|
||||
|
||||
|
|
@ -69,11 +69,669 @@ replacement = "-"
|
|||
rationale = "Replace the small hyphen-minus with reviewed ASCII `-`."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "ideographic-space"
|
||||
source = " "
|
||||
replacement = " "
|
||||
rationale = "Width-fold the ideographic space to ASCII space."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff01"
|
||||
source = "!"
|
||||
replacement = "!"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF01 to ASCII "!".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff02"
|
||||
source = """
|
||||
replacement = '"'
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF02 to ASCII "\"".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff03"
|
||||
source = "#"
|
||||
replacement = "#"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF03 to ASCII "#".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff04"
|
||||
source = "$"
|
||||
replacement = "$"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF04 to ASCII "$".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff05"
|
||||
source = "%"
|
||||
replacement = "%"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF05 to ASCII "%".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff06"
|
||||
source = "&"
|
||||
replacement = "&"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF06 to ASCII "&".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff07"
|
||||
source = "'"
|
||||
replacement = "'"
|
||||
rationale = """Width-fold Unicode fullwidth form U+FF07 to ASCII "'"."""
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff08"
|
||||
source = "("
|
||||
replacement = "("
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF08 to ASCII "(".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff09"
|
||||
source = ")"
|
||||
replacement = ")"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF09 to ASCII ")".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff0a"
|
||||
source = "*"
|
||||
replacement = "*"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0A to ASCII "*".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff0b"
|
||||
source = "+"
|
||||
replacement = "+"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0B to ASCII "+".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff0c"
|
||||
source = ","
|
||||
replacement = ","
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0C to ASCII ",".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-hyphen-minus"
|
||||
source = "-"
|
||||
replacement = "-"
|
||||
rationale = "Replace the fullwidth hyphen-minus with reviewed ASCII `-`."
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0D to ASCII "-".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff0e"
|
||||
source = "."
|
||||
replacement = "."
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0E to ASCII ".".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff0f"
|
||||
source = "/"
|
||||
replacement = "/"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF0F to ASCII "/".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff10"
|
||||
source = "0"
|
||||
replacement = "0"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF10 to ASCII "0".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff11"
|
||||
source = "1"
|
||||
replacement = "1"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF11 to ASCII "1".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff12"
|
||||
source = "2"
|
||||
replacement = "2"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF12 to ASCII "2".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff13"
|
||||
source = "3"
|
||||
replacement = "3"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF13 to ASCII "3".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff14"
|
||||
source = "4"
|
||||
replacement = "4"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF14 to ASCII "4".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff15"
|
||||
source = "5"
|
||||
replacement = "5"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF15 to ASCII "5".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff16"
|
||||
source = "6"
|
||||
replacement = "6"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF16 to ASCII "6".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff17"
|
||||
source = "7"
|
||||
replacement = "7"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF17 to ASCII "7".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff18"
|
||||
source = "8"
|
||||
replacement = "8"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF18 to ASCII "8".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff19"
|
||||
source = "9"
|
||||
replacement = "9"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF19 to ASCII "9".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff1a"
|
||||
source = ":"
|
||||
replacement = ":"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1A to ASCII ":".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff1b"
|
||||
source = ";"
|
||||
replacement = ";"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1B to ASCII ";".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff1c"
|
||||
source = "<"
|
||||
replacement = "<"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1C to ASCII "<".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff1d"
|
||||
source = "="
|
||||
replacement = "="
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1D to ASCII "=".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff1e"
|
||||
source = ">"
|
||||
replacement = ">"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1E to ASCII ">".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff1f"
|
||||
source = "?"
|
||||
replacement = "?"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF1F to ASCII "?".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff20"
|
||||
source = "@"
|
||||
replacement = "@"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF20 to ASCII "@".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff21"
|
||||
source = "A"
|
||||
replacement = "A"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF21 to ASCII "A".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff22"
|
||||
source = "B"
|
||||
replacement = "B"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF22 to ASCII "B".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff23"
|
||||
source = "C"
|
||||
replacement = "C"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF23 to ASCII "C".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff24"
|
||||
source = "D"
|
||||
replacement = "D"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF24 to ASCII "D".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff25"
|
||||
source = "E"
|
||||
replacement = "E"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF25 to ASCII "E".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff26"
|
||||
source = "F"
|
||||
replacement = "F"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF26 to ASCII "F".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff27"
|
||||
source = "G"
|
||||
replacement = "G"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF27 to ASCII "G".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff28"
|
||||
source = "H"
|
||||
replacement = "H"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF28 to ASCII "H".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff29"
|
||||
source = "I"
|
||||
replacement = "I"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF29 to ASCII "I".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2a"
|
||||
source = "J"
|
||||
replacement = "J"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2A to ASCII "J".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2b"
|
||||
source = "K"
|
||||
replacement = "K"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2B to ASCII "K".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff2c"
|
||||
source = "L"
|
||||
replacement = "L"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2C to ASCII "L".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2d"
|
||||
source = "M"
|
||||
replacement = "M"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2D to ASCII "M".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2e"
|
||||
source = "N"
|
||||
replacement = "N"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2E to ASCII "N".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2f"
|
||||
source = "O"
|
||||
replacement = "O"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF2F to ASCII "O".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff30"
|
||||
source = "P"
|
||||
replacement = "P"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF30 to ASCII "P".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff31"
|
||||
source = "Q"
|
||||
replacement = "Q"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF31 to ASCII "Q".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff32"
|
||||
source = "R"
|
||||
replacement = "R"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF32 to ASCII "R".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff33"
|
||||
source = "S"
|
||||
replacement = "S"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF33 to ASCII "S".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff34"
|
||||
source = "T"
|
||||
replacement = "T"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF34 to ASCII "T".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff35"
|
||||
source = "U"
|
||||
replacement = "U"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF35 to ASCII "U".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff36"
|
||||
source = "V"
|
||||
replacement = "V"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF36 to ASCII "V".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff37"
|
||||
source = "W"
|
||||
replacement = "W"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF37 to ASCII "W".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff38"
|
||||
source = "X"
|
||||
replacement = "X"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF38 to ASCII "X".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff39"
|
||||
source = "Y"
|
||||
replacement = "Y"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF39 to ASCII "Y".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3a"
|
||||
source = "Z"
|
||||
replacement = "Z"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3A to ASCII "Z".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3b"
|
||||
source = "["
|
||||
replacement = "["
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3B to ASCII "[".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3c"
|
||||
source = "\"
|
||||
replacement = '\'
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3C to ASCII "\\".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3d"
|
||||
source = "]"
|
||||
replacement = "]"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3D to ASCII "]".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff3e"
|
||||
source = "^"
|
||||
replacement = "^"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3E to ASCII "^".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff3f"
|
||||
source = "_"
|
||||
replacement = "_"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF3F to ASCII "_".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff40"
|
||||
source = "`"
|
||||
replacement = "`"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF40 to ASCII "`".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff41"
|
||||
source = "a"
|
||||
replacement = "a"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF41 to ASCII "a".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff42"
|
||||
source = "b"
|
||||
replacement = "b"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF42 to ASCII "b".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff43"
|
||||
source = "c"
|
||||
replacement = "c"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF43 to ASCII "c".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff44"
|
||||
source = "d"
|
||||
replacement = "d"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF44 to ASCII "d".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff45"
|
||||
source = "e"
|
||||
replacement = "e"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF45 to ASCII "e".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff46"
|
||||
source = "f"
|
||||
replacement = "f"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF46 to ASCII "f".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff47"
|
||||
source = "g"
|
||||
replacement = "g"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF47 to ASCII "g".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff48"
|
||||
source = "h"
|
||||
replacement = "h"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF48 to ASCII "h".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff49"
|
||||
source = "i"
|
||||
replacement = "i"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF49 to ASCII "i".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4a"
|
||||
source = "j"
|
||||
replacement = "j"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4A to ASCII "j".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff4b"
|
||||
source = "k"
|
||||
replacement = "k"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4B to ASCII "k".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4c"
|
||||
source = "l"
|
||||
replacement = "l"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4C to ASCII "l".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff4d"
|
||||
source = "m"
|
||||
replacement = "m"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4D to ASCII "m".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff4e"
|
||||
source = "n"
|
||||
replacement = "n"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4E to ASCII "n".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4f"
|
||||
source = "o"
|
||||
replacement = "o"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF4F to ASCII "o".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff50"
|
||||
source = "p"
|
||||
replacement = "p"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF50 to ASCII "p".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff51"
|
||||
source = "q"
|
||||
replacement = "q"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF51 to ASCII "q".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff52"
|
||||
source = "r"
|
||||
replacement = "r"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF52 to ASCII "r".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff53"
|
||||
source = "s"
|
||||
replacement = "s"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF53 to ASCII "s".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff54"
|
||||
source = "t"
|
||||
replacement = "t"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF54 to ASCII "t".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff55"
|
||||
source = "u"
|
||||
replacement = "u"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF55 to ASCII "u".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff56"
|
||||
source = "v"
|
||||
replacement = "v"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF56 to ASCII "v".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff57"
|
||||
source = "w"
|
||||
replacement = "w"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF57 to ASCII "w".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff58"
|
||||
source = "x"
|
||||
replacement = "x"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF58 to ASCII "x".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff59"
|
||||
source = "y"
|
||||
replacement = "y"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF59 to ASCII "y".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff5a"
|
||||
source = "z"
|
||||
replacement = "z"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF5A to ASCII "z".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff5b"
|
||||
source = "{"
|
||||
replacement = "{"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF5B to ASCII "{".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff5c"
|
||||
source = "|"
|
||||
replacement = "|"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF5C to ASCII "|".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff5d"
|
||||
source = "}"
|
||||
replacement = "}"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF5D to ASCII "}".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-uff5e"
|
||||
source = "~"
|
||||
replacement = "~"
|
||||
rationale = 'Width-fold Unicode fullwidth form U+FF5E to ASCII "~".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
|
|
@ -5263,279 +5921,6 @@ replacement = "o"
|
|||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FEEC with its printable ASCII skeleton "o".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff01"
|
||||
source = "!"
|
||||
replacement = "!"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF01 with its printable ASCII skeleton "!".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff02"
|
||||
source = """
|
||||
replacement = "''"
|
||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF02 with its printable ASCII skeleton "''"."""
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff07"
|
||||
source = "'"
|
||||
replacement = "'"
|
||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF07 with its printable ASCII skeleton "'"."""
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff1a"
|
||||
source = ":"
|
||||
replacement = ":"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF1A with its printable ASCII skeleton ":".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff21"
|
||||
source = "A"
|
||||
replacement = "A"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF21 with its printable ASCII skeleton "A".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff22"
|
||||
source = "B"
|
||||
replacement = "B"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF22 with its printable ASCII skeleton "B".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff23"
|
||||
source = "C"
|
||||
replacement = "C"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF23 with its printable ASCII skeleton "C".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff25"
|
||||
source = "E"
|
||||
replacement = "E"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF25 with its printable ASCII skeleton "E".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff28"
|
||||
source = "H"
|
||||
replacement = "H"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF28 with its printable ASCII skeleton "H".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff29"
|
||||
source = "I"
|
||||
replacement = "l"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF29 with its printable ASCII skeleton "l".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2a"
|
||||
source = "J"
|
||||
replacement = "J"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2A with its printable ASCII skeleton "J".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2b"
|
||||
source = "K"
|
||||
replacement = "K"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2B with its printable ASCII skeleton "K".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2d"
|
||||
source = "M"
|
||||
replacement = "M"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2D with its printable ASCII skeleton "M".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2e"
|
||||
source = "N"
|
||||
replacement = "N"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2E with its printable ASCII skeleton "N".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff2f"
|
||||
source = "O"
|
||||
replacement = "O"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2F with its printable ASCII skeleton "O".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff30"
|
||||
source = "P"
|
||||
replacement = "P"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF30 with its printable ASCII skeleton "P".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff33"
|
||||
source = "S"
|
||||
replacement = "S"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF33 with its printable ASCII skeleton "S".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff34"
|
||||
source = "T"
|
||||
replacement = "T"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF34 with its printable ASCII skeleton "T".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff38"
|
||||
source = "X"
|
||||
replacement = "X"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF38 with its printable ASCII skeleton "X".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff39"
|
||||
source = "Y"
|
||||
replacement = "Y"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF39 with its printable ASCII skeleton "Y".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3a"
|
||||
source = "Z"
|
||||
replacement = "Z"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3A with its printable ASCII skeleton "Z".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3b"
|
||||
source = "["
|
||||
replacement = "("
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3B with its printable ASCII skeleton "(".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3c"
|
||||
source = "\"
|
||||
replacement = '\'
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3C with its printable ASCII skeleton "\\".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff3d"
|
||||
source = "]"
|
||||
replacement = ")"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3D with its printable ASCII skeleton ")".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff40"
|
||||
source = "`"
|
||||
replacement = "'"
|
||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF40 with its printable ASCII skeleton "'"."""
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff41"
|
||||
source = "a"
|
||||
replacement = "a"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF41 with its printable ASCII skeleton "a".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff43"
|
||||
source = "c"
|
||||
replacement = "c"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF43 with its printable ASCII skeleton "c".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff45"
|
||||
source = "e"
|
||||
replacement = "e"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF45 with its printable ASCII skeleton "e".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff47"
|
||||
source = "g"
|
||||
replacement = "g"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF47 with its printable ASCII skeleton "g".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff48"
|
||||
source = "h"
|
||||
replacement = "h"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF48 with its printable ASCII skeleton "h".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff49"
|
||||
source = "i"
|
||||
replacement = "i"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF49 with its printable ASCII skeleton "i".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4a"
|
||||
source = "j"
|
||||
replacement = "j"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4A with its printable ASCII skeleton "j".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4c"
|
||||
source = "l"
|
||||
replacement = "l"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4C with its printable ASCII skeleton "l".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff4f"
|
||||
source = "o"
|
||||
replacement = "o"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4F with its printable ASCII skeleton "o".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff50"
|
||||
source = "p"
|
||||
replacement = "p"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF50 with its printable ASCII skeleton "p".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff53"
|
||||
source = "s"
|
||||
replacement = "s"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF53 with its printable ASCII skeleton "s".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff56"
|
||||
source = "v"
|
||||
replacement = "v"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF56 with its printable ASCII skeleton "v".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff58"
|
||||
source = "x"
|
||||
replacement = "x"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF58 with its printable ASCII skeleton "x".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uff59"
|
||||
source = "y"
|
||||
replacement = "y"
|
||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF59 with its printable ASCII skeleton "y".'
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-uffe8"
|
||||
source = "│"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
|
||||
//! Regenerate `assets/unicode.toml` from reviewed mappings and Unicode Security.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
|
|
@ -12,12 +14,6 @@ const CURATED: &[(&str, &str, &str, &str)] = &[
|
|||
("minus-sign", "−", "-", "mathematical minus sign"),
|
||||
("small-em-dash", "﹘", "-", "small em dash"),
|
||||
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
|
||||
(
|
||||
"fullwidth-hyphen-minus",
|
||||
"-",
|
||||
"-",
|
||||
"fullwidth hyphen-minus",
|
||||
),
|
||||
];
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -75,25 +71,56 @@ fn main() {
|
|||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
replacements.push(Replacement {
|
||||
id: "ideographic-space".to_owned(),
|
||||
source: '\u{3000}'.to_string(),
|
||||
replacement: " ".to_owned(),
|
||||
rationale: "Width-fold the ideographic space to ASCII space.".to_owned(),
|
||||
enabled: true,
|
||||
});
|
||||
for ascii in b'!'..=b'~' {
|
||||
// U+FF01..=U+FF5E is a contiguous block corresponding to ASCII !..=~.
|
||||
let value = u32::from(ascii) + 0xfee0;
|
||||
let Some(fullwidth) = char::from_u32(value) else {
|
||||
continue;
|
||||
};
|
||||
let id = if ascii == b'-' {
|
||||
// Preserve the public ID shipped before the complete width-folding set.
|
||||
"fullwidth-hyphen-minus".to_owned()
|
||||
} else if printable_ascii_skeleton(fullwidth).is_some() {
|
||||
// Preserve IDs already shipped as generated UTS #39 mappings.
|
||||
format!("uts39-u{value:04x}")
|
||||
} else {
|
||||
format!("fullwidth-u{value:04x}")
|
||||
};
|
||||
let replacement = char::from(ascii).to_string();
|
||||
replacements.push(Replacement {
|
||||
id,
|
||||
source: fullwidth.to_string(),
|
||||
rationale: format!(
|
||||
"Width-fold Unicode fullwidth form U+{value:04X} to ASCII {replacement:?}."
|
||||
),
|
||||
replacement,
|
||||
enabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
let reviewed_sources = replacements
|
||||
.iter()
|
||||
.filter_map(|rule| rule.source.chars().next())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
for value in 0..=char::MAX as u32 {
|
||||
let Some(character) = char::from_u32(value) else {
|
||||
continue;
|
||||
};
|
||||
if character.is_ascii()
|
||||
|| CURATED
|
||||
.iter()
|
||||
.any(|(_, source, _, _)| source.starts_with(character))
|
||||
{
|
||||
if character.is_ascii() || reviewed_sources.contains(&character) {
|
||||
continue;
|
||||
}
|
||||
let source = character.to_string();
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
if skeleton.is_empty()
|
||||
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||
|| skeleton == source
|
||||
{
|
||||
let Some(skeleton) = printable_ascii_skeleton(character) else {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
replacements.push(Replacement {
|
||||
id: format!("uts39-u{value:04x}"),
|
||||
source,
|
||||
|
|
@ -149,8 +176,17 @@ fn main() {
|
|||
|
||||
println!(
|
||||
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
|
||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
|
||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.\n\
|
||||
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
|
||||
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
|
||||
);
|
||||
}
|
||||
|
||||
fn printable_ascii_skeleton(character: char) -> Option<String> {
|
||||
let source = character.to_string();
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
(!skeleton.is_empty()
|
||||
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||
&& skeleton != source)
|
||||
.then_some(skeleton)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1383,7 +1383,7 @@ mod tests {
|
|||
.map(|rule| rule.source.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(config.replacements.len(), 1_829);
|
||||
assert_eq!(config.replacements.len(), 1_884);
|
||||
assert_eq!(config.schema_version, 1);
|
||||
assert_eq!(ids.len(), config.replacements.len());
|
||||
assert_eq!(sources.len(), config.replacements.len());
|
||||
|
|
@ -1411,6 +1411,11 @@ mod tests {
|
|||
continue;
|
||||
}
|
||||
let source = character.to_string();
|
||||
let width_folded = match value {
|
||||
0x3000 => Some(" ".to_owned()),
|
||||
0xff01..=0xff5e => char::from_u32(value - 0xfee0).map(|ascii| ascii.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
if !skeleton.is_empty()
|
||||
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||
|
|
@ -1418,13 +1423,33 @@ mod tests {
|
|||
{
|
||||
assert_eq!(
|
||||
replacements.get(source.as_str()),
|
||||
Some(&skeleton.as_str()),
|
||||
Some(&width_folded.as_deref().unwrap_or(skeleton.as_str())),
|
||||
"catalog drift for U+{value:04X}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_width_folds_preserve_existing_ids_and_cover_new_sources() {
|
||||
let config = Config::embedded();
|
||||
let fullwidth_i = config
|
||||
.replacements
|
||||
.iter()
|
||||
.find(|rule| rule.source == "I")
|
||||
.unwrap();
|
||||
let fullwidth_number_sign = config
|
||||
.replacements
|
||||
.iter()
|
||||
.find(|rule| rule.source == "#")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(fullwidth_i.id, "uts39-uff29");
|
||||
assert_eq!(fullwidth_i.replacement, "I");
|
||||
assert_eq!(fullwidth_number_sign.id, "fullwidth-uff03");
|
||||
assert_eq!(fullwidth_number_sign.replacement, "#");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_input_limits_match_the_accepted_policy() {
|
||||
let limits = InputLimits::default();
|
||||
|
|
@ -1745,7 +1770,7 @@ weight = 9.5
|
|||
|
||||
let config = ConfigLoader::from_paths(paths).load().unwrap();
|
||||
|
||||
assert_eq!(config.replacements.len(), 1_830);
|
||||
assert_eq!(config.replacements.len(), 1_885);
|
||||
let replacement = config
|
||||
.replacements
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ pub enum ReplacementError {
|
|||
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||
plan_matching(config, source, |rule| {
|
||||
rule.enabled
|
||||
&& (!rule.id.starts_with("uts39-")
|
||||
&& (!is_bulk_confusable(rule)
|
||||
|| config.unicode.confusables == ConfusablePolicy::Replace)
|
||||
})
|
||||
}
|
||||
|
|
@ -48,10 +48,17 @@ pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec<
|
|||
return Vec::new();
|
||||
}
|
||||
plan_matching(config, source, |rule| {
|
||||
rule.enabled && rule.id.starts_with("uts39-")
|
||||
rule.enabled && is_bulk_confusable(rule)
|
||||
})
|
||||
}
|
||||
|
||||
fn is_bulk_confusable(rule: &ReplacementRule) -> bool {
|
||||
let mut characters = rule.source.chars();
|
||||
let is_fullwidth_scalar =
|
||||
matches!(characters.next(), Some('\u{ff01}'..='\u{ff5e}')) && characters.next().is_none();
|
||||
rule.id.starts_with("uts39-") && !is_fullwidth_scalar
|
||||
}
|
||||
|
||||
fn plan_matching(
|
||||
config: &Config,
|
||||
source: &str,
|
||||
|
|
@ -196,16 +203,28 @@ mod tests {
|
|||
assert_eq!(edits[4].replacement, "rn");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_catalog_width_folds_every_printable_fullwidth_ascii_character() {
|
||||
let source = std::iter::once('\u{3000}')
|
||||
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
|
||||
.collect::<String>();
|
||||
let expected = (b' '..=b'~').map(char::from).collect::<String>();
|
||||
let edits = plan(&Config::embedded(), &source);
|
||||
|
||||
assert_eq!(edits.len(), 95);
|
||||
assert_eq!(apply(&source, &edits).unwrap(), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_confusable_policy_preserves_curated_replacements() {
|
||||
let mut config = Config::embedded();
|
||||
config.unicode.confusables = ConfusablePolicy::Review;
|
||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
||||
assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1);
|
||||
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
|
||||
assert_eq!(plan_confusables_for_review(&config, "а—I ").len(), 1);
|
||||
|
||||
config.unicode.confusables = ConfusablePolicy::Off;
|
||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
||||
assert!(plan_confusables_for_review(&config, "а—").is_empty());
|
||||
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
|
||||
assert!(plan_confusables_for_review(&config, "а—I ").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -158,8 +158,9 @@ confusables = "review" # replace (default), review, or off
|
|||
```
|
||||
|
||||
`replace` enables diagnostics and edits, `review` retains diagnostics without
|
||||
formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated
|
||||
punctuation replacements and invisible/control diagnostics are unaffected.
|
||||
formatter/LSP edits, and `off` suppresses generated lookalikes. The reviewed
|
||||
punctuation and width-folding replacements and invisible/control diagnostics
|
||||
are unaffected.
|
||||
|
||||
Set `enabled = false` on a known rule or replacement ID to disable it. Because
|
||||
records merge by ID, do not copy the full embedded table merely to change one
|
||||
|
|
@ -171,8 +172,11 @@ must be unique. When enabled sources share a prefix, the longest matching source
|
|||
wins at each byte offset, with stable rule ID ordering as the deterministic
|
||||
tie-break. Edits always refer to the original UTF-8 snapshot.
|
||||
|
||||
Generated mapping IDs use `uts39-u<lowercase-codepoint>`, so an individual
|
||||
entry can still be disabled or overridden through normal merge-by-ID behavior.
|
||||
Generated confusable IDs use `uts39-u<lowercase-codepoint>`. New fullwidth
|
||||
mappings use `fullwidth-u<lowercase-codepoint>`; mappings shipped previously
|
||||
retain their `uts39-*` or `fullwidth-hyphen-minus` IDs for compatibility, while
|
||||
ideographic space uses `ideographic-space`. Every entry can be disabled or
|
||||
overridden through normal merge-by-ID behavior.
|
||||
Maintain the embedded list by editing
|
||||
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:
|
||||
|
||||
|
|
|
|||
|
|
@ -31,8 +31,11 @@ locale-aware lookalike table.
|
|||
|
||||
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
|
||||
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
|
||||
scalar whose single-character skeleton is non-empty printable ASCII, plus ten
|
||||
reviewed punctuation mappings. That is 1,829 mappings in the current profile.
|
||||
scalar whose single-character skeleton is non-empty printable ASCII, plus nine
|
||||
reviewed punctuation mappings and 95 compatibility width folds. Width folding
|
||||
takes precedence when a UTS #39 visual skeleton would produce a different
|
||||
target, such as fullwidth `I` mapping to `I` rather than `l`. That is 1,884
|
||||
mappings in the current profile.
|
||||
Each entry has a stable ID, source, target, rationale, and enabled switch, and
|
||||
the same asset owns the diagnostic metadata.
|
||||
|
||||
|
|
@ -40,8 +43,8 @@ The default `[unicode] confusables = "replace"` policy emits
|
|||
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
|
||||
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
|
||||
`review` for identical findings without edits, or `off` to suppress generated
|
||||
lookalikes. The ten curated punctuation mappings remain active under all three
|
||||
settings and can still be disabled individually.
|
||||
lookalikes. The reviewed punctuation and width-folding mappings remain active
|
||||
under all three settings and can still be disabled individually.
|
||||
|
||||
This broad mapping is intentionally more aggressive than Gitea's locale-aware
|
||||
display warning. Greek, Cyrillic, mathematical, and other characters may be
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ cargo run --example ascii_replacements
|
|||
optional path defaults to `examples/fixtures`.
|
||||
- `plan_replacements` prints the exact UTF-8 byte edits shared by the formatter
|
||||
and LSP quick fixes.
|
||||
- `ascii_replacements` builds an opt-in catalog through the public API and maps
|
||||
fullwidth characters to all 95 printable ASCII targets.
|
||||
- `ascii_replacements` demonstrates the embedded width-folding catalog across
|
||||
all 95 printable ASCII targets.
|
||||
|
||||
These examples use embedded defaults so their behavior does not depend on a
|
||||
developer's user configuration. Applications that want normal configuration
|
||||
|
|
@ -52,9 +52,9 @@ as the formatter.
|
|||
|
||||
## Printable ASCII replacement showcase
|
||||
|
||||
[`ascii_replacements.rs`](ascii_replacements.rs) extends the embedded defaults
|
||||
in memory with fullwidth mappings for ASCII `!` through `~` and maps ideographic
|
||||
space to ASCII space. Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
|
||||
[`ascii_replacements.rs`](ascii_replacements.rs) applies the embedded fullwidth
|
||||
mappings for ASCII `!` through `~` and maps ideographic space to ASCII space.
|
||||
Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
|
||||
and [expected output](ascii-replacement-showcase/ascii.txt) display all 95
|
||||
printable ASCII targets in one place:
|
||||
|
||||
|
|
@ -63,17 +63,16 @@ cargo run --example ascii_replacements
|
|||
```
|
||||
|
||||
The example reports 95 planned edits on stderr and writes only the transformed
|
||||
fixture to stdout. These broad letter, digit, punctuation, and space mappings
|
||||
are an API/configuration demonstration, not additions to the production
|
||||
defaults or a recommendation to rewrite arbitrary multilingual text. The
|
||||
embedded catalog remains limited to individually reviewed conservative rules.
|
||||
fixture to stdout. These compatibility width folds are production defaults;
|
||||
broader script-confusable replacements remain controlled separately by the
|
||||
`[unicode] confusables` policy.
|
||||
|
||||
## Expected-failure diagnostic showcase
|
||||
|
||||
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
|
||||
shipped prose signal, while
|
||||
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
|
||||
source-placeholder and tutorial-narration rules plus the ten curated
|
||||
source-placeholder and tutorial-narration rules plus the reviewed
|
||||
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
|
||||
right-to-left override. The adjacent example configuration promotes assistant
|
||||
self-identification from warning to error and lifts the typography occurrence
|
||||
|
|
|
|||
|
|
@ -1,59 +1,22 @@
|
|||
//! Showcase opt-in replacements for every printable ASCII target.
|
||||
//! Showcase embedded width folding for every printable ASCII target.
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use john_slop::{
|
||||
Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements,
|
||||
};
|
||||
use john_slop::{Config, apply_replacements, plan_replacements};
|
||||
|
||||
const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
|
||||
|
||||
fn showcase_config() -> Result<Config, ConfigValidationError> {
|
||||
let mut config = Config::embedded();
|
||||
let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| {
|
||||
// U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values.
|
||||
let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0)
|
||||
.expect("the fullwidth ASCII range contains valid scalar values");
|
||||
(fullwidth, char::from(ascii))
|
||||
}));
|
||||
|
||||
for (source, target) in mappings {
|
||||
let source = source.to_string();
|
||||
if let Some(rule) = config
|
||||
.replacements
|
||||
.iter_mut()
|
||||
.find(|rule| rule.source == source)
|
||||
{
|
||||
rule.replacement = target.to_string();
|
||||
rule.rationale = format!(
|
||||
"Override U+{:04X} for the complete printable-ASCII showcase.",
|
||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
||||
);
|
||||
} else {
|
||||
config.replacements.push(ReplacementRule {
|
||||
id: format!(
|
||||
"showcase-fullwidth-{:04x}",
|
||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
||||
),
|
||||
source,
|
||||
replacement: target.to_string(),
|
||||
rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."),
|
||||
enabled: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
fn showcase_config() -> Config {
|
||||
Config::embedded()
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = showcase_config()?;
|
||||
let config = showcase_config();
|
||||
let edits = plan_replacements(&config, SOURCE);
|
||||
let formatted = apply_replacements(SOURCE, &edits)?;
|
||||
|
||||
eprintln!(
|
||||
"planned {} opt-in replacements covering ASCII space through tilde",
|
||||
"planned {} embedded replacements covering ASCII space through tilde",
|
||||
edits.len()
|
||||
);
|
||||
std::io::stdout().write_all(formatted.as_bytes())?;
|
||||
|
|
@ -70,7 +33,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn showcase_maps_each_printable_ascii_target_once() {
|
||||
let config = showcase_config().expect("build valid showcase configuration");
|
||||
let config = showcase_config();
|
||||
let edits = plan_replacements(&config, SOURCE);
|
||||
let targets = edits
|
||||
.iter()
|
||||
|
|
|
|||
23
tasks.md
23
tasks.md
|
|
@ -636,6 +636,29 @@ checks, Unicode-catalog reproducibility, and workspace package assembly pass;
|
|||
coverage is 92.83% (1,579/1,701 lines), and all release metadata and binary
|
||||
versions agree on `0.1.24`.
|
||||
|
||||
## Milestone 25 — Deterministic fullwidth ASCII folding
|
||||
|
||||
- [x] Give ideographic space and every fullwidth form in U+FF01..=U+FF5E an
|
||||
explicit one-to-one printable ASCII replacement before generating broader
|
||||
UTS #39 skeleton mappings. Preserve all previously shipped fullwidth IDs and
|
||||
cover the complete sequence through core, formatter-example, and real LSP
|
||||
formatting tests.
|
||||
- [ ] Update Unicode/configuration guidance, changelog, release metadata,
|
||||
coverage, badges, and recap; run all quality, audit, regeneration, and
|
||||
package gates.
|
||||
|
||||
Decision: compatibility width folding takes precedence over visual-confusable
|
||||
skeletons for this contiguous Unicode block. These 95 reviewed mappings remain
|
||||
active under `replace`, `review`, and `off`; the bulk policy continues to govern
|
||||
only `uts39-*` mappings. This prevents visually motivated results such as
|
||||
fullwidth `I` becoming lowercase `l`, while preserving policy compatibility
|
||||
and the shared planner used by the CLI, formatter, and LSP.
|
||||
|
||||
Acceptance: formatting ideographic space followed by U+FF01..=U+FF5E produces
|
||||
ASCII space through tilde exactly, with 95 source-ordered edits and identical
|
||||
standalone/LSP output; the generated catalog remains deterministic; all release
|
||||
artifacts agree on `0.1.25` before the immutable `v0.1.25` tag is created.
|
||||
|
||||
## Deferred by design
|
||||
|
||||
- Statistical or model-based authorship classification.
|
||||
|
|
|
|||
46
tests/lsp.rs
46
tests/lsp.rs
|
|
@ -447,6 +447,52 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
|
|||
assert!(closed["version"].is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn formatting_width_folds_the_complete_fullwidth_ascii_block() {
|
||||
let mut harness = Harness::new(None).await;
|
||||
let uri = "file:///tmp/fullwidth.txt";
|
||||
let source = std::iter::once('\u{3000}')
|
||||
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
|
||||
.collect::<String>();
|
||||
let expected = (b' '..=b'~').map(char::from).collect::<String>();
|
||||
harness
|
||||
.notify("textDocument/didOpen", open_params(uri, 1, &source))
|
||||
.await;
|
||||
let _published = harness
|
||||
.notification("textDocument/publishDiagnostics")
|
||||
.await;
|
||||
|
||||
let formatting = harness
|
||||
.request(
|
||||
"textDocument/formatting",
|
||||
json!({
|
||||
"textDocument": { "uri": uri },
|
||||
"options": { "tabSize": 4, "insertSpaces": true }
|
||||
}),
|
||||
2,
|
||||
)
|
||||
.await;
|
||||
let mut formatted = source.clone();
|
||||
for edit in formatting
|
||||
.result()
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.rev()
|
||||
{
|
||||
let start = utf16_position_to_byte(&source, &edit["range"]["start"]);
|
||||
let end = utf16_position_to_byte(&source, &edit["range"]["end"]);
|
||||
formatted.replace_range(start..end, edit["newText"].as_str().unwrap());
|
||||
}
|
||||
|
||||
assert_eq!(formatted, expected);
|
||||
assert_eq!(
|
||||
apply_replacements(&source, &plan_replacements(&Config::embedded(), &source)).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configuration_reload_and_concurrent_documents_remain_independent() {
|
||||
let disabled = r#"
|
||||
|
|
|
|||
Loading…
Reference in New Issue