fix: width-fold fullwidth ASCII deterministically
This commit is contained in:
parent
215a164a04
commit
861fba5a81
|
|
@ -1,5 +1,5 @@
|
||||||
# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.
|
# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.
|
||||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.
|
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.
|
||||||
# Do not edit generated mappings by hand; edit this generator and regenerate.
|
# Do not edit generated mappings by hand; edit this generator and regenerate.
|
||||||
schema_version = 1
|
schema_version = 1
|
||||||
|
|
||||||
|
|
@ -69,11 +69,669 @@ replacement = "-"
|
||||||
rationale = "Replace the small hyphen-minus with reviewed ASCII `-`."
|
rationale = "Replace the small hyphen-minus with reviewed ASCII `-`."
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "ideographic-space"
|
||||||
|
source = " "
|
||||||
|
replacement = " "
|
||||||
|
rationale = "Width-fold the ideographic space to ASCII space."
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff01"
|
||||||
|
source = "!"
|
||||||
|
replacement = "!"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF01 to ASCII "!".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff02"
|
||||||
|
source = """
|
||||||
|
replacement = '"'
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF02 to ASCII "\"".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff03"
|
||||||
|
source = "#"
|
||||||
|
replacement = "#"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF03 to ASCII "#".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff04"
|
||||||
|
source = "$"
|
||||||
|
replacement = "$"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF04 to ASCII "$".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff05"
|
||||||
|
source = "%"
|
||||||
|
replacement = "%"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF05 to ASCII "%".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff06"
|
||||||
|
source = "&"
|
||||||
|
replacement = "&"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF06 to ASCII "&".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff07"
|
||||||
|
source = "'"
|
||||||
|
replacement = "'"
|
||||||
|
rationale = """Width-fold Unicode fullwidth form U+FF07 to ASCII "'"."""
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff08"
|
||||||
|
source = "("
|
||||||
|
replacement = "("
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF08 to ASCII "(".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff09"
|
||||||
|
source = ")"
|
||||||
|
replacement = ")"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF09 to ASCII ")".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff0a"
|
||||||
|
source = "*"
|
||||||
|
replacement = "*"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF0A to ASCII "*".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff0b"
|
||||||
|
source = "+"
|
||||||
|
replacement = "+"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF0B to ASCII "+".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff0c"
|
||||||
|
source = ","
|
||||||
|
replacement = ","
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF0C to ASCII ",".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
[[replacements]]
|
[[replacements]]
|
||||||
id = "fullwidth-hyphen-minus"
|
id = "fullwidth-hyphen-minus"
|
||||||
source = "-"
|
source = "-"
|
||||||
replacement = "-"
|
replacement = "-"
|
||||||
rationale = "Replace the fullwidth hyphen-minus with reviewed ASCII `-`."
|
rationale = 'Width-fold Unicode fullwidth form U+FF0D to ASCII "-".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff0e"
|
||||||
|
source = "."
|
||||||
|
replacement = "."
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF0E to ASCII ".".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff0f"
|
||||||
|
source = "/"
|
||||||
|
replacement = "/"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF0F to ASCII "/".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff10"
|
||||||
|
source = "0"
|
||||||
|
replacement = "0"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF10 to ASCII "0".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff11"
|
||||||
|
source = "1"
|
||||||
|
replacement = "1"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF11 to ASCII "1".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff12"
|
||||||
|
source = "2"
|
||||||
|
replacement = "2"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF12 to ASCII "2".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff13"
|
||||||
|
source = "3"
|
||||||
|
replacement = "3"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF13 to ASCII "3".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff14"
|
||||||
|
source = "4"
|
||||||
|
replacement = "4"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF14 to ASCII "4".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff15"
|
||||||
|
source = "5"
|
||||||
|
replacement = "5"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF15 to ASCII "5".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff16"
|
||||||
|
source = "6"
|
||||||
|
replacement = "6"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF16 to ASCII "6".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff17"
|
||||||
|
source = "7"
|
||||||
|
replacement = "7"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF17 to ASCII "7".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff18"
|
||||||
|
source = "8"
|
||||||
|
replacement = "8"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF18 to ASCII "8".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff19"
|
||||||
|
source = "9"
|
||||||
|
replacement = "9"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF19 to ASCII "9".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff1a"
|
||||||
|
source = ":"
|
||||||
|
replacement = ":"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1A to ASCII ":".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff1b"
|
||||||
|
source = ";"
|
||||||
|
replacement = ";"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1B to ASCII ";".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff1c"
|
||||||
|
source = "<"
|
||||||
|
replacement = "<"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1C to ASCII "<".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff1d"
|
||||||
|
source = "="
|
||||||
|
replacement = "="
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1D to ASCII "=".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff1e"
|
||||||
|
source = ">"
|
||||||
|
replacement = ">"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1E to ASCII ">".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff1f"
|
||||||
|
source = "?"
|
||||||
|
replacement = "?"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF1F to ASCII "?".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff20"
|
||||||
|
source = "@"
|
||||||
|
replacement = "@"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF20 to ASCII "@".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff21"
|
||||||
|
source = "A"
|
||||||
|
replacement = "A"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF21 to ASCII "A".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff22"
|
||||||
|
source = "B"
|
||||||
|
replacement = "B"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF22 to ASCII "B".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff23"
|
||||||
|
source = "C"
|
||||||
|
replacement = "C"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF23 to ASCII "C".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff24"
|
||||||
|
source = "D"
|
||||||
|
replacement = "D"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF24 to ASCII "D".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff25"
|
||||||
|
source = "E"
|
||||||
|
replacement = "E"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF25 to ASCII "E".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff26"
|
||||||
|
source = "F"
|
||||||
|
replacement = "F"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF26 to ASCII "F".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff27"
|
||||||
|
source = "G"
|
||||||
|
replacement = "G"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF27 to ASCII "G".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff28"
|
||||||
|
source = "H"
|
||||||
|
replacement = "H"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF28 to ASCII "H".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff29"
|
||||||
|
source = "I"
|
||||||
|
replacement = "I"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF29 to ASCII "I".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff2a"
|
||||||
|
source = "J"
|
||||||
|
replacement = "J"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2A to ASCII "J".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff2b"
|
||||||
|
source = "K"
|
||||||
|
replacement = "K"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2B to ASCII "K".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff2c"
|
||||||
|
source = "L"
|
||||||
|
replacement = "L"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2C to ASCII "L".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff2d"
|
||||||
|
source = "M"
|
||||||
|
replacement = "M"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2D to ASCII "M".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff2e"
|
||||||
|
source = "N"
|
||||||
|
replacement = "N"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2E to ASCII "N".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff2f"
|
||||||
|
source = "O"
|
||||||
|
replacement = "O"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF2F to ASCII "O".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff30"
|
||||||
|
source = "P"
|
||||||
|
replacement = "P"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF30 to ASCII "P".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff31"
|
||||||
|
source = "Q"
|
||||||
|
replacement = "Q"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF31 to ASCII "Q".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff32"
|
||||||
|
source = "R"
|
||||||
|
replacement = "R"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF32 to ASCII "R".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff33"
|
||||||
|
source = "S"
|
||||||
|
replacement = "S"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF33 to ASCII "S".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff34"
|
||||||
|
source = "T"
|
||||||
|
replacement = "T"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF34 to ASCII "T".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff35"
|
||||||
|
source = "U"
|
||||||
|
replacement = "U"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF35 to ASCII "U".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff36"
|
||||||
|
source = "V"
|
||||||
|
replacement = "V"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF36 to ASCII "V".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff37"
|
||||||
|
source = "W"
|
||||||
|
replacement = "W"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF37 to ASCII "W".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff38"
|
||||||
|
source = "X"
|
||||||
|
replacement = "X"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF38 to ASCII "X".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff39"
|
||||||
|
source = "Y"
|
||||||
|
replacement = "Y"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF39 to ASCII "Y".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff3a"
|
||||||
|
source = "Z"
|
||||||
|
replacement = "Z"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3A to ASCII "Z".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff3b"
|
||||||
|
source = "["
|
||||||
|
replacement = "["
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3B to ASCII "[".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff3c"
|
||||||
|
source = "\"
|
||||||
|
replacement = '\'
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3C to ASCII "\\".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff3d"
|
||||||
|
source = "]"
|
||||||
|
replacement = "]"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3D to ASCII "]".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff3e"
|
||||||
|
source = "^"
|
||||||
|
replacement = "^"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3E to ASCII "^".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff3f"
|
||||||
|
source = "_"
|
||||||
|
replacement = "_"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF3F to ASCII "_".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff40"
|
||||||
|
source = "`"
|
||||||
|
replacement = "`"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF40 to ASCII "`".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff41"
|
||||||
|
source = "a"
|
||||||
|
replacement = "a"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF41 to ASCII "a".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff42"
|
||||||
|
source = "b"
|
||||||
|
replacement = "b"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF42 to ASCII "b".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff43"
|
||||||
|
source = "c"
|
||||||
|
replacement = "c"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF43 to ASCII "c".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff44"
|
||||||
|
source = "d"
|
||||||
|
replacement = "d"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF44 to ASCII "d".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff45"
|
||||||
|
source = "e"
|
||||||
|
replacement = "e"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF45 to ASCII "e".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff46"
|
||||||
|
source = "f"
|
||||||
|
replacement = "f"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF46 to ASCII "f".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff47"
|
||||||
|
source = "g"
|
||||||
|
replacement = "g"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF47 to ASCII "g".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff48"
|
||||||
|
source = "h"
|
||||||
|
replacement = "h"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF48 to ASCII "h".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff49"
|
||||||
|
source = "i"
|
||||||
|
replacement = "i"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF49 to ASCII "i".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff4a"
|
||||||
|
source = "j"
|
||||||
|
replacement = "j"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4A to ASCII "j".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff4b"
|
||||||
|
source = "k"
|
||||||
|
replacement = "k"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4B to ASCII "k".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff4c"
|
||||||
|
source = "l"
|
||||||
|
replacement = "l"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4C to ASCII "l".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff4d"
|
||||||
|
source = "m"
|
||||||
|
replacement = "m"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4D to ASCII "m".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff4e"
|
||||||
|
source = "n"
|
||||||
|
replacement = "n"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4E to ASCII "n".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff4f"
|
||||||
|
source = "o"
|
||||||
|
replacement = "o"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF4F to ASCII "o".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff50"
|
||||||
|
source = "p"
|
||||||
|
replacement = "p"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF50 to ASCII "p".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff51"
|
||||||
|
source = "q"
|
||||||
|
replacement = "q"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF51 to ASCII "q".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff52"
|
||||||
|
source = "r"
|
||||||
|
replacement = "r"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF52 to ASCII "r".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff53"
|
||||||
|
source = "s"
|
||||||
|
replacement = "s"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF53 to ASCII "s".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff54"
|
||||||
|
source = "t"
|
||||||
|
replacement = "t"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF54 to ASCII "t".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff55"
|
||||||
|
source = "u"
|
||||||
|
replacement = "u"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF55 to ASCII "u".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff56"
|
||||||
|
source = "v"
|
||||||
|
replacement = "v"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF56 to ASCII "v".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff57"
|
||||||
|
source = "w"
|
||||||
|
replacement = "w"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF57 to ASCII "w".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff58"
|
||||||
|
source = "x"
|
||||||
|
replacement = "x"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF58 to ASCII "x".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "uts39-uff59"
|
||||||
|
source = "y"
|
||||||
|
replacement = "y"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF59 to ASCII "y".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff5a"
|
||||||
|
source = "z"
|
||||||
|
replacement = "z"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF5A to ASCII "z".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff5b"
|
||||||
|
source = "{"
|
||||||
|
replacement = "{"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF5B to ASCII "{".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff5c"
|
||||||
|
source = "|"
|
||||||
|
replacement = "|"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF5C to ASCII "|".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff5d"
|
||||||
|
source = "}"
|
||||||
|
replacement = "}"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF5D to ASCII "}".'
|
||||||
|
enabled = true
|
||||||
|
|
||||||
|
[[replacements]]
|
||||||
|
id = "fullwidth-uff5e"
|
||||||
|
source = "~"
|
||||||
|
replacement = "~"
|
||||||
|
rationale = 'Width-fold Unicode fullwidth form U+FF5E to ASCII "~".'
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[[replacements]]
|
[[replacements]]
|
||||||
|
|
@ -5263,279 +5921,6 @@ replacement = "o"
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FEEC with its printable ASCII skeleton "o".'
|
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FEEC with its printable ASCII skeleton "o".'
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff01"
|
|
||||||
source = "!"
|
|
||||||
replacement = "!"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF01 with its printable ASCII skeleton "!".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff02"
|
|
||||||
source = """
|
|
||||||
replacement = "''"
|
|
||||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF02 with its printable ASCII skeleton "''"."""
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff07"
|
|
||||||
source = "'"
|
|
||||||
replacement = "'"
|
|
||||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF07 with its printable ASCII skeleton "'"."""
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff1a"
|
|
||||||
source = ":"
|
|
||||||
replacement = ":"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF1A with its printable ASCII skeleton ":".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff21"
|
|
||||||
source = "A"
|
|
||||||
replacement = "A"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF21 with its printable ASCII skeleton "A".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff22"
|
|
||||||
source = "B"
|
|
||||||
replacement = "B"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF22 with its printable ASCII skeleton "B".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff23"
|
|
||||||
source = "C"
|
|
||||||
replacement = "C"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF23 with its printable ASCII skeleton "C".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff25"
|
|
||||||
source = "E"
|
|
||||||
replacement = "E"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF25 with its printable ASCII skeleton "E".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff28"
|
|
||||||
source = "H"
|
|
||||||
replacement = "H"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF28 with its printable ASCII skeleton "H".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff29"
|
|
||||||
source = "I"
|
|
||||||
replacement = "l"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF29 with its printable ASCII skeleton "l".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff2a"
|
|
||||||
source = "J"
|
|
||||||
replacement = "J"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2A with its printable ASCII skeleton "J".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff2b"
|
|
||||||
source = "K"
|
|
||||||
replacement = "K"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2B with its printable ASCII skeleton "K".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff2d"
|
|
||||||
source = "M"
|
|
||||||
replacement = "M"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2D with its printable ASCII skeleton "M".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff2e"
|
|
||||||
source = "N"
|
|
||||||
replacement = "N"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2E with its printable ASCII skeleton "N".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff2f"
|
|
||||||
source = "O"
|
|
||||||
replacement = "O"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF2F with its printable ASCII skeleton "O".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff30"
|
|
||||||
source = "P"
|
|
||||||
replacement = "P"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF30 with its printable ASCII skeleton "P".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff33"
|
|
||||||
source = "S"
|
|
||||||
replacement = "S"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF33 with its printable ASCII skeleton "S".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff34"
|
|
||||||
source = "T"
|
|
||||||
replacement = "T"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF34 with its printable ASCII skeleton "T".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff38"
|
|
||||||
source = "X"
|
|
||||||
replacement = "X"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF38 with its printable ASCII skeleton "X".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff39"
|
|
||||||
source = "Y"
|
|
||||||
replacement = "Y"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF39 with its printable ASCII skeleton "Y".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff3a"
|
|
||||||
source = "Z"
|
|
||||||
replacement = "Z"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3A with its printable ASCII skeleton "Z".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff3b"
|
|
||||||
source = "["
|
|
||||||
replacement = "("
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3B with its printable ASCII skeleton "(".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff3c"
|
|
||||||
source = "\"
|
|
||||||
replacement = '\'
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3C with its printable ASCII skeleton "\\".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff3d"
|
|
||||||
source = "]"
|
|
||||||
replacement = ")"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF3D with its printable ASCII skeleton ")".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff40"
|
|
||||||
source = "`"
|
|
||||||
replacement = "'"
|
|
||||||
rationale = """Replace Unicode 16.0 UTS #39 lookalike U+FF40 with its printable ASCII skeleton "'"."""
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff41"
|
|
||||||
source = "a"
|
|
||||||
replacement = "a"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF41 with its printable ASCII skeleton "a".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff43"
|
|
||||||
source = "c"
|
|
||||||
replacement = "c"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF43 with its printable ASCII skeleton "c".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff45"
|
|
||||||
source = "e"
|
|
||||||
replacement = "e"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF45 with its printable ASCII skeleton "e".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff47"
|
|
||||||
source = "g"
|
|
||||||
replacement = "g"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF47 with its printable ASCII skeleton "g".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff48"
|
|
||||||
source = "h"
|
|
||||||
replacement = "h"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF48 with its printable ASCII skeleton "h".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff49"
|
|
||||||
source = "i"
|
|
||||||
replacement = "i"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF49 with its printable ASCII skeleton "i".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff4a"
|
|
||||||
source = "j"
|
|
||||||
replacement = "j"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4A with its printable ASCII skeleton "j".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff4c"
|
|
||||||
source = "l"
|
|
||||||
replacement = "l"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4C with its printable ASCII skeleton "l".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff4f"
|
|
||||||
source = "o"
|
|
||||||
replacement = "o"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF4F with its printable ASCII skeleton "o".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff50"
|
|
||||||
source = "p"
|
|
||||||
replacement = "p"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF50 with its printable ASCII skeleton "p".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff53"
|
|
||||||
source = "s"
|
|
||||||
replacement = "s"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF53 with its printable ASCII skeleton "s".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff56"
|
|
||||||
source = "v"
|
|
||||||
replacement = "v"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF56 with its printable ASCII skeleton "v".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff58"
|
|
||||||
source = "x"
|
|
||||||
replacement = "x"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF58 with its printable ASCII skeleton "x".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
|
||||||
id = "uts39-uff59"
|
|
||||||
source = "y"
|
|
||||||
replacement = "y"
|
|
||||||
rationale = 'Replace Unicode 16.0 UTS #39 lookalike U+FF59 with its printable ASCII skeleton "y".'
|
|
||||||
enabled = true
|
|
||||||
|
|
||||||
[[replacements]]
|
[[replacements]]
|
||||||
id = "uts39-uffe8"
|
id = "uts39-uffe8"
|
||||||
source = "│"
|
source = "│"
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
|
//! Regenerate `assets/unicode.toml` from reviewed mappings and Unicode Security.
|
||||||
|
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
|
|
@ -12,12 +14,6 @@ const CURATED: &[(&str, &str, &str, &str)] = &[
|
||||||
("minus-sign", "−", "-", "mathematical minus sign"),
|
("minus-sign", "−", "-", "mathematical minus sign"),
|
||||||
("small-em-dash", "﹘", "-", "small em dash"),
|
("small-em-dash", "﹘", "-", "small em dash"),
|
||||||
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
|
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
|
||||||
(
|
|
||||||
"fullwidth-hyphen-minus",
|
|
||||||
"-",
|
|
||||||
"-",
|
|
||||||
"fullwidth hyphen-minus",
|
|
||||||
),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -75,25 +71,56 @@ fn main() {
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
replacements.push(Replacement {
|
||||||
|
id: "ideographic-space".to_owned(),
|
||||||
|
source: '\u{3000}'.to_string(),
|
||||||
|
replacement: " ".to_owned(),
|
||||||
|
rationale: "Width-fold the ideographic space to ASCII space.".to_owned(),
|
||||||
|
enabled: true,
|
||||||
|
});
|
||||||
|
for ascii in b'!'..=b'~' {
|
||||||
|
// U+FF01..=U+FF5E is a contiguous block corresponding to ASCII !..=~.
|
||||||
|
let value = u32::from(ascii) + 0xfee0;
|
||||||
|
let Some(fullwidth) = char::from_u32(value) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let id = if ascii == b'-' {
|
||||||
|
// Preserve the public ID shipped before the complete width-folding set.
|
||||||
|
"fullwidth-hyphen-minus".to_owned()
|
||||||
|
} else if printable_ascii_skeleton(fullwidth).is_some() {
|
||||||
|
// Preserve IDs already shipped as generated UTS #39 mappings.
|
||||||
|
format!("uts39-u{value:04x}")
|
||||||
|
} else {
|
||||||
|
format!("fullwidth-u{value:04x}")
|
||||||
|
};
|
||||||
|
let replacement = char::from(ascii).to_string();
|
||||||
|
replacements.push(Replacement {
|
||||||
|
id,
|
||||||
|
source: fullwidth.to_string(),
|
||||||
|
rationale: format!(
|
||||||
|
"Width-fold Unicode fullwidth form U+{value:04X} to ASCII {replacement:?}."
|
||||||
|
),
|
||||||
|
replacement,
|
||||||
|
enabled: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let reviewed_sources = replacements
|
||||||
|
.iter()
|
||||||
|
.filter_map(|rule| rule.source.chars().next())
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
for value in 0..=char::MAX as u32 {
|
for value in 0..=char::MAX as u32 {
|
||||||
let Some(character) = char::from_u32(value) else {
|
let Some(character) = char::from_u32(value) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
if character.is_ascii()
|
if character.is_ascii() || reviewed_sources.contains(&character) {
|
||||||
|| CURATED
|
|
||||||
.iter()
|
|
||||||
.any(|(_, source, _, _)| source.starts_with(character))
|
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let source = character.to_string();
|
let source = character.to_string();
|
||||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
let Some(skeleton) = printable_ascii_skeleton(character) else {
|
||||||
if skeleton.is_empty()
|
|
||||||
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
|
||||||
|| skeleton == source
|
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
};
|
||||||
replacements.push(Replacement {
|
replacements.push(Replacement {
|
||||||
id: format!("uts39-u{value:04x}"),
|
id: format!("uts39-u{value:04x}"),
|
||||||
source,
|
source,
|
||||||
|
|
@ -149,8 +176,17 @@ fn main() {
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
|
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
|
||||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
|
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation and width folding.\n\
|
||||||
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
|
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
|
||||||
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
|
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn printable_ascii_skeleton(character: char) -> Option<String> {
|
||||||
|
let source = character.to_string();
|
||||||
|
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||||
|
(!skeleton.is_empty()
|
||||||
|
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||||
|
&& skeleton != source)
|
||||||
|
.then_some(skeleton)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1383,7 +1383,7 @@ mod tests {
|
||||||
.map(|rule| rule.source.as_str())
|
.map(|rule| rule.source.as_str())
|
||||||
.collect::<BTreeSet<_>>();
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
assert_eq!(config.replacements.len(), 1_829);
|
assert_eq!(config.replacements.len(), 1_884);
|
||||||
assert_eq!(config.schema_version, 1);
|
assert_eq!(config.schema_version, 1);
|
||||||
assert_eq!(ids.len(), config.replacements.len());
|
assert_eq!(ids.len(), config.replacements.len());
|
||||||
assert_eq!(sources.len(), config.replacements.len());
|
assert_eq!(sources.len(), config.replacements.len());
|
||||||
|
|
@ -1411,6 +1411,11 @@ mod tests {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let source = character.to_string();
|
let source = character.to_string();
|
||||||
|
let width_folded = match value {
|
||||||
|
0x3000 => Some(" ".to_owned()),
|
||||||
|
0xff01..=0xff5e => char::from_u32(value - 0xfee0).map(|ascii| ascii.to_string()),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||||
if !skeleton.is_empty()
|
if !skeleton.is_empty()
|
||||||
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||||
|
|
@ -1418,13 +1423,33 @@ mod tests {
|
||||||
{
|
{
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
replacements.get(source.as_str()),
|
replacements.get(source.as_str()),
|
||||||
Some(&skeleton.as_str()),
|
Some(&width_folded.as_deref().unwrap_or(skeleton.as_str())),
|
||||||
"catalog drift for U+{value:04X}"
|
"catalog drift for U+{value:04X}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn embedded_width_folds_preserve_existing_ids_and_cover_new_sources() {
|
||||||
|
let config = Config::embedded();
|
||||||
|
let fullwidth_i = config
|
||||||
|
.replacements
|
||||||
|
.iter()
|
||||||
|
.find(|rule| rule.source == "I")
|
||||||
|
.unwrap();
|
||||||
|
let fullwidth_number_sign = config
|
||||||
|
.replacements
|
||||||
|
.iter()
|
||||||
|
.find(|rule| rule.source == "#")
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(fullwidth_i.id, "uts39-uff29");
|
||||||
|
assert_eq!(fullwidth_i.replacement, "I");
|
||||||
|
assert_eq!(fullwidth_number_sign.id, "fullwidth-uff03");
|
||||||
|
assert_eq!(fullwidth_number_sign.replacement, "#");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn default_input_limits_match_the_accepted_policy() {
|
fn default_input_limits_match_the_accepted_policy() {
|
||||||
let limits = InputLimits::default();
|
let limits = InputLimits::default();
|
||||||
|
|
@ -1745,7 +1770,7 @@ weight = 9.5
|
||||||
|
|
||||||
let config = ConfigLoader::from_paths(paths).load().unwrap();
|
let config = ConfigLoader::from_paths(paths).load().unwrap();
|
||||||
|
|
||||||
assert_eq!(config.replacements.len(), 1_830);
|
assert_eq!(config.replacements.len(), 1_885);
|
||||||
let replacement = config
|
let replacement = config
|
||||||
.replacements
|
.replacements
|
||||||
.iter()
|
.iter()
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ pub enum ReplacementError {
|
||||||
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||||
plan_matching(config, source, |rule| {
|
plan_matching(config, source, |rule| {
|
||||||
rule.enabled
|
rule.enabled
|
||||||
&& (!rule.id.starts_with("uts39-")
|
&& (!is_bulk_confusable(rule)
|
||||||
|| config.unicode.confusables == ConfusablePolicy::Replace)
|
|| config.unicode.confusables == ConfusablePolicy::Replace)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -48,10 +48,17 @@ pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec<
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
plan_matching(config, source, |rule| {
|
plan_matching(config, source, |rule| {
|
||||||
rule.enabled && rule.id.starts_with("uts39-")
|
rule.enabled && is_bulk_confusable(rule)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_bulk_confusable(rule: &ReplacementRule) -> bool {
|
||||||
|
let mut characters = rule.source.chars();
|
||||||
|
let is_fullwidth_scalar =
|
||||||
|
matches!(characters.next(), Some('\u{ff01}'..='\u{ff5e}')) && characters.next().is_none();
|
||||||
|
rule.id.starts_with("uts39-") && !is_fullwidth_scalar
|
||||||
|
}
|
||||||
|
|
||||||
fn plan_matching(
|
fn plan_matching(
|
||||||
config: &Config,
|
config: &Config,
|
||||||
source: &str,
|
source: &str,
|
||||||
|
|
@ -196,16 +203,28 @@ mod tests {
|
||||||
assert_eq!(edits[4].replacement, "rn");
|
assert_eq!(edits[4].replacement, "rn");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generated_catalog_width_folds_every_printable_fullwidth_ascii_character() {
|
||||||
|
let source = std::iter::once('\u{3000}')
|
||||||
|
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
|
||||||
|
.collect::<String>();
|
||||||
|
let expected = (b' '..=b'~').map(char::from).collect::<String>();
|
||||||
|
let edits = plan(&Config::embedded(), &source);
|
||||||
|
|
||||||
|
assert_eq!(edits.len(), 95);
|
||||||
|
assert_eq!(apply(&source, &edits).unwrap(), expected);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bulk_confusable_policy_preserves_curated_replacements() {
|
fn bulk_confusable_policy_preserves_curated_replacements() {
|
||||||
let mut config = Config::embedded();
|
let mut config = Config::embedded();
|
||||||
config.unicode.confusables = ConfusablePolicy::Review;
|
config.unicode.confusables = ConfusablePolicy::Review;
|
||||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
|
||||||
assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1);
|
assert_eq!(plan_confusables_for_review(&config, "а—I ").len(), 1);
|
||||||
|
|
||||||
config.unicode.confusables = ConfusablePolicy::Off;
|
config.unicode.confusables = ConfusablePolicy::Off;
|
||||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
assert_eq!(apply("а—I ", &plan(&config, "а—I ")).unwrap(), "а-I ");
|
||||||
assert!(plan_confusables_for_review(&config, "а—").is_empty());
|
assert!(plan_confusables_for_review(&config, "а—I ").is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -158,8 +158,9 @@ confusables = "review" # replace (default), review, or off
|
||||||
```
|
```
|
||||||
|
|
||||||
`replace` enables diagnostics and edits, `review` retains diagnostics without
|
`replace` enables diagnostics and edits, `review` retains diagnostics without
|
||||||
formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated
|
formatter/LSP edits, and `off` suppresses generated lookalikes. The reviewed
|
||||||
punctuation replacements and invisible/control diagnostics are unaffected.
|
punctuation and width-folding replacements and invisible/control diagnostics
|
||||||
|
are unaffected.
|
||||||
|
|
||||||
Set `enabled = false` on a known rule or replacement ID to disable it. Because
|
Set `enabled = false` on a known rule or replacement ID to disable it. Because
|
||||||
records merge by ID, do not copy the full embedded table merely to change one
|
records merge by ID, do not copy the full embedded table merely to change one
|
||||||
|
|
@ -171,8 +172,11 @@ must be unique. When enabled sources share a prefix, the longest matching source
|
||||||
wins at each byte offset, with stable rule ID ordering as the deterministic
|
wins at each byte offset, with stable rule ID ordering as the deterministic
|
||||||
tie-break. Edits always refer to the original UTF-8 snapshot.
|
tie-break. Edits always refer to the original UTF-8 snapshot.
|
||||||
|
|
||||||
Generated mapping IDs use `uts39-u<lowercase-codepoint>`, so an individual
|
Generated confusable IDs use `uts39-u<lowercase-codepoint>`. New fullwidth
|
||||||
entry can still be disabled or overridden through normal merge-by-ID behavior.
|
mappings use `fullwidth-u<lowercase-codepoint>`; mappings shipped previously
|
||||||
|
retain their `uts39-*` or `fullwidth-hyphen-minus` IDs for compatibility, while
|
||||||
|
ideographic space uses `ideographic-space`. Every entry can be disabled or
|
||||||
|
overridden through normal merge-by-ID behavior.
|
||||||
Maintain the embedded list by editing
|
Maintain the embedded list by editing
|
||||||
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:
|
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,11 @@ locale-aware lookalike table.
|
||||||
|
|
||||||
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
|
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
|
||||||
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
|
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
|
||||||
scalar whose single-character skeleton is non-empty printable ASCII, plus ten
|
scalar whose single-character skeleton is non-empty printable ASCII, plus nine
|
||||||
reviewed punctuation mappings. That is 1,829 mappings in the current profile.
|
reviewed punctuation mappings and 95 compatibility width folds. Width folding
|
||||||
|
takes precedence when a UTS #39 visual skeleton would produce a different
|
||||||
|
target, such as fullwidth `I` mapping to `I` rather than `l`. That is 1,884
|
||||||
|
mappings in the current profile.
|
||||||
Each entry has a stable ID, source, target, rationale, and enabled switch, and
|
Each entry has a stable ID, source, target, rationale, and enabled switch, and
|
||||||
the same asset owns the diagnostic metadata.
|
the same asset owns the diagnostic metadata.
|
||||||
|
|
||||||
|
|
@ -40,8 +43,8 @@ The default `[unicode] confusables = "replace"` policy emits
|
||||||
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
|
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
|
||||||
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
|
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
|
||||||
`review` for identical findings without edits, or `off` to suppress generated
|
`review` for identical findings without edits, or `off` to suppress generated
|
||||||
lookalikes. The ten curated punctuation mappings remain active under all three
|
lookalikes. The reviewed punctuation and width-folding mappings remain active
|
||||||
settings and can still be disabled individually.
|
under all three settings and can still be disabled individually.
|
||||||
|
|
||||||
This broad mapping is intentionally more aggressive than Gitea's locale-aware
|
This broad mapping is intentionally more aggressive than Gitea's locale-aware
|
||||||
display warning. Greek, Cyrillic, mathematical, and other characters may be
|
display warning. Greek, Cyrillic, mathematical, and other characters may be
|
||||||
|
|
|
||||||
|
|
@ -16,8 +16,8 @@ cargo run --example ascii_replacements
|
||||||
optional path defaults to `examples/fixtures`.
|
optional path defaults to `examples/fixtures`.
|
||||||
- `plan_replacements` prints the exact UTF-8 byte edits shared by the formatter
|
- `plan_replacements` prints the exact UTF-8 byte edits shared by the formatter
|
||||||
and LSP quick fixes.
|
and LSP quick fixes.
|
||||||
- `ascii_replacements` builds an opt-in catalog through the public API and maps
|
- `ascii_replacements` demonstrates the embedded width-folding catalog across
|
||||||
fullwidth characters to all 95 printable ASCII targets.
|
all 95 printable ASCII targets.
|
||||||
|
|
||||||
These examples use embedded defaults so their behavior does not depend on a
|
These examples use embedded defaults so their behavior does not depend on a
|
||||||
developer's user configuration. Applications that want normal configuration
|
developer's user configuration. Applications that want normal configuration
|
||||||
|
|
@ -52,9 +52,9 @@ as the formatter.
|
||||||
|
|
||||||
## Printable ASCII replacement showcase
|
## Printable ASCII replacement showcase
|
||||||
|
|
||||||
[`ascii_replacements.rs`](ascii_replacements.rs) extends the embedded defaults
|
[`ascii_replacements.rs`](ascii_replacements.rs) applies the embedded fullwidth
|
||||||
in memory with fullwidth mappings for ASCII `!` through `~` and maps ideographic
|
mappings for ASCII `!` through `~` and maps ideographic space to ASCII space.
|
||||||
space to ASCII space. Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
|
Its [Unicode input](ascii-replacement-showcase/fullwidth.txt)
|
||||||
and [expected output](ascii-replacement-showcase/ascii.txt) display all 95
|
and [expected output](ascii-replacement-showcase/ascii.txt) display all 95
|
||||||
printable ASCII targets in one place:
|
printable ASCII targets in one place:
|
||||||
|
|
||||||
|
|
@ -63,17 +63,16 @@ cargo run --example ascii_replacements
|
||||||
```
|
```
|
||||||
|
|
||||||
The example reports 95 planned edits on stderr and writes only the transformed
|
The example reports 95 planned edits on stderr and writes only the transformed
|
||||||
fixture to stdout. These broad letter, digit, punctuation, and space mappings
|
fixture to stdout. These compatibility width folds are production defaults;
|
||||||
are an API/configuration demonstration, not additions to the production
|
broader script-confusable replacements remain controlled separately by the
|
||||||
defaults or a recommendation to rewrite arbitrary multilingual text. The
|
`[unicode] confusables` policy.
|
||||||
embedded catalog remains limited to individually reviewed conservative rules.
|
|
||||||
|
|
||||||
## Expected-failure diagnostic showcase
|
## Expected-failure diagnostic showcase
|
||||||
|
|
||||||
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
|
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
|
||||||
shipped prose signal, while
|
shipped prose signal, while
|
||||||
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
|
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
|
||||||
source-placeholder and tutorial-narration rules plus the ten curated
|
source-placeholder and tutorial-narration rules plus the reviewed
|
||||||
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
|
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
|
||||||
right-to-left override. The adjacent example configuration promotes assistant
|
right-to-left override. The adjacent example configuration promotes assistant
|
||||||
self-identification from warning to error and lifts the typography occurrence
|
self-identification from warning to error and lifts the typography occurrence
|
||||||
|
|
|
||||||
|
|
@ -1,59 +1,22 @@
|
||||||
//! Showcase opt-in replacements for every printable ASCII target.
|
//! Showcase embedded width folding for every printable ASCII target.
|
||||||
|
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
|
||||||
use john_slop::{
|
use john_slop::{Config, apply_replacements, plan_replacements};
|
||||||
Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements,
|
|
||||||
};
|
|
||||||
|
|
||||||
const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
|
const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
|
||||||
|
|
||||||
fn showcase_config() -> Result<Config, ConfigValidationError> {
|
fn showcase_config() -> Config {
|
||||||
let mut config = Config::embedded();
|
Config::embedded()
|
||||||
let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| {
|
|
||||||
// U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values.
|
|
||||||
let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0)
|
|
||||||
.expect("the fullwidth ASCII range contains valid scalar values");
|
|
||||||
(fullwidth, char::from(ascii))
|
|
||||||
}));
|
|
||||||
|
|
||||||
for (source, target) in mappings {
|
|
||||||
let source = source.to_string();
|
|
||||||
if let Some(rule) = config
|
|
||||||
.replacements
|
|
||||||
.iter_mut()
|
|
||||||
.find(|rule| rule.source == source)
|
|
||||||
{
|
|
||||||
rule.replacement = target.to_string();
|
|
||||||
rule.rationale = format!(
|
|
||||||
"Override U+{:04X} for the complete printable-ASCII showcase.",
|
|
||||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
config.replacements.push(ReplacementRule {
|
|
||||||
id: format!(
|
|
||||||
"showcase-fullwidth-{:04x}",
|
|
||||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
|
||||||
),
|
|
||||||
source,
|
|
||||||
replacement: target.to_string(),
|
|
||||||
rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."),
|
|
||||||
enabled: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
config.validate()?;
|
|
||||||
Ok(config)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let config = showcase_config()?;
|
let config = showcase_config();
|
||||||
let edits = plan_replacements(&config, SOURCE);
|
let edits = plan_replacements(&config, SOURCE);
|
||||||
let formatted = apply_replacements(SOURCE, &edits)?;
|
let formatted = apply_replacements(SOURCE, &edits)?;
|
||||||
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"planned {} opt-in replacements covering ASCII space through tilde",
|
"planned {} embedded replacements covering ASCII space through tilde",
|
||||||
edits.len()
|
edits.len()
|
||||||
);
|
);
|
||||||
std::io::stdout().write_all(formatted.as_bytes())?;
|
std::io::stdout().write_all(formatted.as_bytes())?;
|
||||||
|
|
@ -70,7 +33,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn showcase_maps_each_printable_ascii_target_once() {
|
fn showcase_maps_each_printable_ascii_target_once() {
|
||||||
let config = showcase_config().expect("build valid showcase configuration");
|
let config = showcase_config();
|
||||||
let edits = plan_replacements(&config, SOURCE);
|
let edits = plan_replacements(&config, SOURCE);
|
||||||
let targets = edits
|
let targets = edits
|
||||||
.iter()
|
.iter()
|
||||||
|
|
|
||||||
23
tasks.md
23
tasks.md
|
|
@ -636,6 +636,29 @@ checks, Unicode-catalog reproducibility, and workspace package assembly pass;
|
||||||
coverage is 92.83% (1,579/1,701 lines), and all release metadata and binary
|
coverage is 92.83% (1,579/1,701 lines), and all release metadata and binary
|
||||||
versions agree on `0.1.24`.
|
versions agree on `0.1.24`.
|
||||||
|
|
||||||
|
## Milestone 25 — Deterministic fullwidth ASCII folding
|
||||||
|
|
||||||
|
- [x] Give ideographic space and every fullwidth form in U+FF01..=U+FF5E an
|
||||||
|
explicit one-to-one printable ASCII replacement before generating broader
|
||||||
|
UTS #39 skeleton mappings. Preserve all previously shipped fullwidth IDs and
|
||||||
|
cover the complete sequence through core, formatter-example, and real LSP
|
||||||
|
formatting tests.
|
||||||
|
- [ ] Update Unicode/configuration guidance, changelog, release metadata,
|
||||||
|
coverage, badges, and recap; run all quality, audit, regeneration, and
|
||||||
|
package gates.
|
||||||
|
|
||||||
|
Decision: compatibility width folding takes precedence over visual-confusable
|
||||||
|
skeletons for this contiguous Unicode block. These 95 reviewed mappings remain
|
||||||
|
active under `replace`, `review`, and `off`; the bulk policy continues to govern
|
||||||
|
only `uts39-*` mappings. This prevents visually motivated results such as
|
||||||
|
fullwidth `I` becoming lowercase `l`, while preserving policy compatibility
|
||||||
|
and the shared planner used by the CLI, formatter, and LSP.
|
||||||
|
|
||||||
|
Acceptance: formatting ideographic space followed by U+FF01..=U+FF5E produces
|
||||||
|
ASCII space through tilde exactly, with 95 source-ordered edits and identical
|
||||||
|
standalone/LSP output; the generated catalog remains deterministic; all release
|
||||||
|
artifacts agree on `0.1.25` before the immutable `v0.1.25` tag is created.
|
||||||
|
|
||||||
## Deferred by design
|
## Deferred by design
|
||||||
|
|
||||||
- Statistical or model-based authorship classification.
|
- Statistical or model-based authorship classification.
|
||||||
|
|
|
||||||
46
tests/lsp.rs
46
tests/lsp.rs
|
|
@ -447,6 +447,52 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
|
||||||
assert!(closed["version"].is_null());
|
assert!(closed["version"].is_null());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn formatting_width_folds_the_complete_fullwidth_ascii_block() {
|
||||||
|
let mut harness = Harness::new(None).await;
|
||||||
|
let uri = "file:///tmp/fullwidth.txt";
|
||||||
|
let source = std::iter::once('\u{3000}')
|
||||||
|
.chain((b'!'..=b'~').filter_map(|ascii| char::from_u32(u32::from(ascii) + 0xfee0)))
|
||||||
|
.collect::<String>();
|
||||||
|
let expected = (b' '..=b'~').map(char::from).collect::<String>();
|
||||||
|
harness
|
||||||
|
.notify("textDocument/didOpen", open_params(uri, 1, &source))
|
||||||
|
.await;
|
||||||
|
let _published = harness
|
||||||
|
.notification("textDocument/publishDiagnostics")
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let formatting = harness
|
||||||
|
.request(
|
||||||
|
"textDocument/formatting",
|
||||||
|
json!({
|
||||||
|
"textDocument": { "uri": uri },
|
||||||
|
"options": { "tabSize": 4, "insertSpaces": true }
|
||||||
|
}),
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut formatted = source.clone();
|
||||||
|
for edit in formatting
|
||||||
|
.result()
|
||||||
|
.unwrap()
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
{
|
||||||
|
let start = utf16_position_to_byte(&source, &edit["range"]["start"]);
|
||||||
|
let end = utf16_position_to_byte(&source, &edit["range"]["end"]);
|
||||||
|
formatted.replace_range(start..end, edit["newText"].as_str().unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(formatted, expected);
|
||||||
|
assert_eq!(
|
||||||
|
apply_replacements(&source, &plan_replacements(&Config::embedded(), &source)).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn configuration_reload_and_concurrent_documents_remain_independent() {
|
async fn configuration_reload_and_concurrent_documents_remain_independent() {
|
||||||
let disabled = r#"
|
let disabled = r#"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue