feat: generate maintainable unicode replacement catalog
This commit is contained in:
parent
893fbfdd67
commit
82afdc7a3f
|
|
@ -1,6 +1,7 @@
|
|||
# Embedded replacement and input defaults. User configuration layers may
|
||||
# override these values, but installed binaries never require this source file.
|
||||
# Embedded input defaults. Unicode replacements live in the generated
|
||||
# `unicode.toml` policy asset so detection, diagnostics, and fixes stay aligned.
|
||||
schema_version = 1
|
||||
replacements = []
|
||||
|
||||
[input]
|
||||
max_document_bytes = 8388608
|
||||
|
|
@ -9,83 +10,3 @@ max_expanded_bytes = 268435456
|
|||
max_archive_entries = 10000
|
||||
max_compression_ratio = 100
|
||||
max_walk_depth = 64
|
||||
|
||||
# A replacement entry has a stable lowercase kebab-case ID, non-empty Unicode
|
||||
# source text, an ASCII target, a rationale, and an enabled switch. When enabled
|
||||
# sources share a prefix, the longest source wins. Example:
|
||||
# [[replacements]]
|
||||
# id = "example-symbol"
|
||||
# source = "→"
|
||||
# replacement = "->"
|
||||
# rationale = "Prefer an ASCII arrow in plain-text output."
|
||||
# enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "hyphen"
|
||||
source = "‐"
|
||||
replacement = "-"
|
||||
rationale = "Replace the Unicode hyphen with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "non-breaking-hyphen"
|
||||
source = "‑"
|
||||
replacement = "-"
|
||||
rationale = "Replace the non-breaking hyphen with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "figure-dash"
|
||||
source = "‒"
|
||||
replacement = "-"
|
||||
rationale = "Replace the figure dash with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "en-dash"
|
||||
source = "–"
|
||||
replacement = "-"
|
||||
rationale = "Replace the en dash with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "em-dash"
|
||||
source = "—"
|
||||
replacement = "-"
|
||||
rationale = "Replace the em dash with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "horizontal-bar"
|
||||
source = "―"
|
||||
replacement = "-"
|
||||
rationale = "Replace the horizontal bar with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "minus-sign"
|
||||
source = "−"
|
||||
replacement = "-"
|
||||
rationale = "Replace the mathematical minus sign with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "small-em-dash"
|
||||
source = "﹘"
|
||||
replacement = "-"
|
||||
rationale = "Replace the small em dash with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "small-hyphen-minus"
|
||||
source = "﹣"
|
||||
replacement = "-"
|
||||
rationale = "Replace the small hyphen-minus with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
||||
[[replacements]]
|
||||
id = "fullwidth-hyphen-minus"
|
||||
source = "-"
|
||||
replacement = "-"
|
||||
rationale = "Replace the fullwidth hyphen-minus with ASCII hyphen-minus."
|
||||
enabled = true
|
||||
|
|
|
|||
|
|
@ -166,51 +166,3 @@ enabled = true
|
|||
[rules.scope]
|
||||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
|
||||
|
||||
[[rules]]
|
||||
id = "typography-non-ascii-hyphen"
|
||||
category = "typography"
|
||||
severity = "advice"
|
||||
matcher = "replacement"
|
||||
pattern = "configured-replacements"
|
||||
weight = 10.0
|
||||
message = "a configured non-ASCII character has an ASCII replacement"
|
||||
help = "Review the punctuation or apply the configured safe replacement."
|
||||
occurrence_cap = 5
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose", "source-code"]
|
||||
extensions = []
|
||||
|
||||
[[rules]]
|
||||
id = "unicode-ascii-confusable"
|
||||
category = "typography"
|
||||
severity = "advice"
|
||||
matcher = "confusable"
|
||||
pattern = "uts39-ascii-skeleton-v16"
|
||||
weight = 4.0
|
||||
message = "a Unicode character visually resembles ASCII under UTS #39"
|
||||
help = "Review whether the lookalike character is intentional; it is not automatically rewritten."
|
||||
occurrence_cap = 5
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose", "source-code"]
|
||||
extensions = []
|
||||
|
||||
[[rules]]
|
||||
id = "unicode-invisible-control"
|
||||
category = "typography"
|
||||
severity = "warning"
|
||||
matcher = "invisible"
|
||||
pattern = "gitea-invisible-9c715c0"
|
||||
weight = 12.0
|
||||
message = "an invisible or bidirectional Unicode control appears in the source"
|
||||
help = "Review the hidden or direction-changing character; it is not removed automatically."
|
||||
occurrence_cap = 5
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose", "source-code"]
|
||||
extensions = []
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,163 @@
|
|||
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
const CURATED: &[(&str, &str, &str, &str)] = &[
|
||||
("hyphen", "‐", "-", "Unicode hyphen"),
|
||||
("non-breaking-hyphen", "‑", "-", "non-breaking hyphen"),
|
||||
("figure-dash", "‒", "-", "figure dash"),
|
||||
("en-dash", "–", "-", "en dash"),
|
||||
("em-dash", "—", "-", "em dash"),
|
||||
("horizontal-bar", "―", "-", "horizontal bar"),
|
||||
("minus-sign", "−", "-", "mathematical minus sign"),
|
||||
("small-em-dash", "﹘", "-", "small em dash"),
|
||||
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
|
||||
(
|
||||
"fullwidth-hyphen-minus",
|
||||
"-",
|
||||
"-",
|
||||
"fullwidth hyphen-minus",
|
||||
),
|
||||
];
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Catalog {
|
||||
schema_version: u32,
|
||||
replacements: Vec<Replacement>,
|
||||
rules: Vec<Rule>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Replacement {
|
||||
id: String,
|
||||
source: String,
|
||||
replacement: String,
|
||||
rationale: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Rule {
|
||||
id: &'static str,
|
||||
category: &'static str,
|
||||
severity: &'static str,
|
||||
matcher: &'static str,
|
||||
pattern: &'static str,
|
||||
weight: f64,
|
||||
message: &'static str,
|
||||
help: &'static str,
|
||||
occurrence_cap: usize,
|
||||
enabled: bool,
|
||||
scope: Scope,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Scope {
|
||||
content: [&'static str; 2],
|
||||
extensions: [String; 0],
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut replacements = CURATED
|
||||
.iter()
|
||||
.map(|(id, source, replacement, name)| Replacement {
|
||||
id: (*id).to_owned(),
|
||||
source: (*source).to_owned(),
|
||||
replacement: (*replacement).to_owned(),
|
||||
rationale: format!("Replace the {name} with reviewed ASCII `{replacement}`."),
|
||||
enabled: true,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for value in 0..=char::MAX as u32 {
|
||||
let Some(character) = char::from_u32(value) else {
|
||||
continue;
|
||||
};
|
||||
if character.is_ascii()
|
||||
|| CURATED
|
||||
.iter()
|
||||
.any(|(_, source, _, _)| source.starts_with(character))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let source = character.to_string();
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
if skeleton.is_empty()
|
||||
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||
|| skeleton == source
|
||||
{
|
||||
continue;
|
||||
}
|
||||
replacements.push(Replacement {
|
||||
id: format!("uts39-u{value:04x}"),
|
||||
source,
|
||||
replacement: skeleton.clone(),
|
||||
rationale: format!(
|
||||
"Replace Unicode 16.0 UTS #39 lookalike U+{value:04X} with its printable ASCII skeleton {skeleton:?}."
|
||||
),
|
||||
enabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
let catalog = Catalog {
|
||||
schema_version: 1,
|
||||
replacements,
|
||||
rules: vec![
|
||||
Rule {
|
||||
id: "unicode-ascii-replacement",
|
||||
category: "typography",
|
||||
severity: "advice",
|
||||
matcher: "replacement",
|
||||
pattern: "configured-replacements",
|
||||
weight: 10.0,
|
||||
message: "a Unicode character or sequence has a configured ASCII replacement",
|
||||
help: "Review the character or apply the configured ASCII replacement.",
|
||||
occurrence_cap: 5,
|
||||
enabled: true,
|
||||
scope: Scope {
|
||||
content: ["prose", "source-code"],
|
||||
extensions: [],
|
||||
},
|
||||
},
|
||||
Rule {
|
||||
id: "unicode-ascii-confusable",
|
||||
category: "typography",
|
||||
severity: "advice",
|
||||
matcher: "confusable",
|
||||
pattern: "uts39-ascii-skeleton-v16",
|
||||
weight: 4.0,
|
||||
message: "a Unicode character visually resembles ASCII under UTS #39",
|
||||
help: "Review whether the lookalike character is intentional.",
|
||||
occurrence_cap: 5,
|
||||
enabled: true,
|
||||
scope: Scope {
|
||||
content: ["prose", "source-code"],
|
||||
extensions: [],
|
||||
},
|
||||
},
|
||||
Rule {
|
||||
id: "unicode-invisible-control",
|
||||
category: "typography",
|
||||
severity: "warning",
|
||||
matcher: "invisible",
|
||||
pattern: "gitea-invisible-9c715c0",
|
||||
weight: 12.0,
|
||||
message: "an invisible or bidirectional Unicode control appears in the source",
|
||||
help: "Review the hidden or direction-changing character; it is not removed automatically.",
|
||||
occurrence_cap: 5,
|
||||
enabled: true,
|
||||
scope: Scope {
|
||||
content: ["prose", "source-code"],
|
||||
extensions: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
println!(
|
||||
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
|
||||
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
|
||||
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
|
||||
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
|
||||
);
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ use crate::{Category, FindingSeverity};
|
|||
|
||||
const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml");
|
||||
const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml");
|
||||
const EMBEDDED_UNICODE: &str = include_str!("../assets/unicode.toml");
|
||||
|
||||
/// A validated named configuration without path syntax or a file extension.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
|
|
@ -916,11 +917,20 @@ impl Default for InputLimits {
|
|||
}
|
||||
|
||||
impl Config {
|
||||
/// Load the replacement defaults embedded in this build.
|
||||
/// Load all defaults embedded in this build.
|
||||
pub fn embedded() -> Self {
|
||||
let config: Self = embedded_figment()
|
||||
let mut config: Self = embedded_figment()
|
||||
.extract()
|
||||
.expect("embedded configuration must deserialize");
|
||||
let unicode_layer = Figment::from(Toml::string(EMBEDDED_UNICODE))
|
||||
.extract::<ConfigLayer>()
|
||||
.expect("embedded Unicode policy must deserialize");
|
||||
apply_layer(
|
||||
&mut config,
|
||||
unicode_layer,
|
||||
std::path::Path::new("<embedded unicode.toml>"),
|
||||
)
|
||||
.expect("embedded Unicode policy must merge");
|
||||
config
|
||||
.validate()
|
||||
.expect("embedded configuration must be semantically valid");
|
||||
|
|
@ -1151,7 +1161,7 @@ mod tests {
|
|||
.map(|rule| rule.source.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(config.replacements.len(), 10);
|
||||
assert_eq!(config.replacements.len(), 1_829);
|
||||
assert_eq!(config.schema_version, 1);
|
||||
assert_eq!(ids.len(), config.replacements.len());
|
||||
assert_eq!(sources.len(), config.replacements.len());
|
||||
|
|
@ -1161,6 +1171,37 @@ mod tests {
|
|||
&& !rule.rationale.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_unicode_catalog_covers_every_printable_ascii_skeleton() {
|
||||
let config = Config::embedded();
|
||||
let replacements = config
|
||||
.replacements
|
||||
.iter()
|
||||
.map(|rule| (rule.source.as_str(), rule.replacement.as_str()))
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
|
||||
for value in 0..=char::MAX as u32 {
|
||||
let Some(character) = char::from_u32(value) else {
|
||||
continue;
|
||||
};
|
||||
if character.is_ascii() {
|
||||
continue;
|
||||
}
|
||||
let source = character.to_string();
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
if !skeleton.is_empty()
|
||||
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|
||||
&& skeleton != source
|
||||
{
|
||||
assert_eq!(
|
||||
replacements.get(source.as_str()),
|
||||
Some(&skeleton.as_str()),
|
||||
"catalog drift for U+{value:04X}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_input_limits_match_the_accepted_policy() {
|
||||
let limits = InputLimits::default();
|
||||
|
|
@ -1311,7 +1352,7 @@ weight = 9.5
|
|||
|
||||
let config = ConfigLoader::from_paths(paths).load().unwrap();
|
||||
|
||||
assert_eq!(config.replacements.len(), 11);
|
||||
assert_eq!(config.replacements.len(), 1_830);
|
||||
let replacement = config
|
||||
.replacements
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -566,7 +566,7 @@ mod tests {
|
|||
assert_eq!(analysis.findings.len(), 1);
|
||||
assert_eq!(
|
||||
analysis.findings[0].rule_id.as_str(),
|
||||
"typography-non-ascii-hyphen"
|
||||
"unicode-ascii-replacement"
|
||||
);
|
||||
assert_eq!(analysis.findings[0].matched_text, "—");
|
||||
assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-");
|
||||
|
|
@ -575,17 +575,16 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_review_findings_have_exact_spans_and_never_become_fixes() {
|
||||
fn unicode_review_findings_have_exact_spans_and_safe_fixes() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let source = "é pаss\u{200b}word\u{202e}\r\nnext";
|
||||
let findings = detector.analyze(source).findings;
|
||||
|
||||
assert_eq!(findings.len(), 3);
|
||||
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-confusable");
|
||||
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
|
||||
assert_eq!(findings[0].matched_text, "а");
|
||||
assert_eq!(&source[findings[0].byte_span.clone()], "а");
|
||||
assert!(findings[0].help.as_deref().unwrap().contains("\"a\""));
|
||||
assert!(findings.iter().all(|finding| finding.fix.is_none()));
|
||||
assert_eq!(findings[0].fix.as_ref().unwrap().replacement, "a");
|
||||
assert_eq!(
|
||||
findings
|
||||
.iter()
|
||||
|
|
@ -593,6 +592,7 @@ mod tests {
|
|||
.count(),
|
||||
2
|
||||
);
|
||||
assert!(findings[1..].iter().all(|finding| finding.fix.is_none()));
|
||||
assert!(findings.iter().all(|finding| finding.matched_text != "\r"));
|
||||
}
|
||||
|
||||
|
|
@ -604,7 +604,7 @@ mod tests {
|
|||
.findings;
|
||||
|
||||
assert_eq!(findings.len(), 1);
|
||||
assert_eq!(findings[0].rule_id.as_str(), "typography-non-ascii-hyphen");
|
||||
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -675,7 +675,7 @@ mod tests {
|
|||
"repeated-transition",
|
||||
"source-placeholder-todo",
|
||||
"source-tutorial-narration",
|
||||
"typography-non-ascii-hyphen",
|
||||
"unicode-ascii-replacement",
|
||||
] {
|
||||
assert!(ids.contains(expected), "missing {expected}");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
//! Pure planning and application of configured text replacements.
|
||||
|
||||
use std::{borrow::Cow, ops::Range};
|
||||
use std::{borrow::Cow, collections::BTreeMap, ops::Range};
|
||||
|
||||
use miette::Diagnostic;
|
||||
|
||||
|
|
@ -36,12 +36,18 @@ pub enum ReplacementError {
|
|||
|
||||
/// Plan deterministic non-overlapping edits without modifying the source.
|
||||
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||
let mut rules = config
|
||||
.replacements
|
||||
.iter()
|
||||
.filter(|rule| rule.enabled)
|
||||
.collect::<Vec<_>>();
|
||||
rules.sort_by(|left, right| {
|
||||
let mut single_character_rules = BTreeMap::new();
|
||||
let mut sequence_rules = Vec::new();
|
||||
for rule in config.replacements.iter().filter(|rule| rule.enabled) {
|
||||
let mut characters = rule.source.chars();
|
||||
let first = characters.next();
|
||||
if let (Some(character), None) = (first, characters.next()) {
|
||||
single_character_rules.insert(character, rule);
|
||||
} else {
|
||||
sequence_rules.push(rule);
|
||||
}
|
||||
}
|
||||
sequence_rules.sort_by(|left, right| {
|
||||
right
|
||||
.source
|
||||
.len()
|
||||
|
|
@ -52,7 +58,7 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
|||
let mut edits = Vec::new();
|
||||
let mut offset = 0;
|
||||
while offset < source.len() {
|
||||
if let Some(rule) = rules
|
||||
if let Some(rule) = sequence_rules
|
||||
.iter()
|
||||
.find(|rule| source[offset..].starts_with(&rule.source))
|
||||
{
|
||||
|
|
@ -63,7 +69,17 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
|||
});
|
||||
offset = end;
|
||||
} else {
|
||||
offset += source[offset..].chars().next().map_or(0, char::len_utf8);
|
||||
let Some(character) = source[offset..].chars().next() else {
|
||||
break;
|
||||
};
|
||||
let end = offset + character.len_utf8();
|
||||
if let Some(rule) = single_character_rules.get(&character) {
|
||||
edits.push(TextEdit {
|
||||
byte_span: offset..end,
|
||||
replacement: rule.replacement.clone(),
|
||||
});
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
edits
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -549,7 +549,7 @@ changelog, and the generated badge agree on `0.1.21`.
|
|||
|
||||
## Milestone 22 — Maintainable ambiguous-Unicode replacement catalog
|
||||
|
||||
- [ ] Define one versioned Unicode policy asset that owns the replacement
|
||||
- [x] Define one versioned Unicode policy asset that owns the replacement
|
||||
diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39
|
||||
ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a
|
||||
deterministic generator with pinned Unicode provenance and drift checks.
|
||||
|
|
|
|||
Loading…
Reference in New Issue