feat: generate maintainable unicode replacement catalog

This commit is contained in:
Dandaedre 2026-09-30 00:24:24 +02:00
parent 893fbfdd67
commit 82afdc7a3f
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
8 changed files with 13109 additions and 151 deletions

View File

@ -1,6 +1,7 @@
# Embedded replacement and input defaults. User configuration layers may
# override these values, but installed binaries never require this source file.
# Embedded input defaults. Unicode replacements live in the generated
# `unicode.toml` policy asset so detection, diagnostics, and fixes stay aligned.
schema_version = 1
replacements = []
[input]
max_document_bytes = 8388608
@ -9,83 +10,3 @@ max_expanded_bytes = 268435456
max_archive_entries = 10000
max_compression_ratio = 100
max_walk_depth = 64
# A replacement entry has a stable lowercase kebab-case ID, non-empty Unicode
# source text, an ASCII target, a rationale, and an enabled switch. When enabled
# sources share a prefix, the longest source wins. Example:
# [[replacements]]
# id = "example-symbol"
# source = "→"
# replacement = "->"
# rationale = "Prefer an ASCII arrow in plain-text output."
# enabled = true
[[replacements]]
id = "hyphen"
source = "‐"
replacement = "-"
rationale = "Replace the Unicode hyphen with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "non-breaking-hyphen"
source = "‑"
replacement = "-"
rationale = "Replace the non-breaking hyphen with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "figure-dash"
source = "‒"
replacement = "-"
rationale = "Replace the figure dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "en-dash"
source = "–"
replacement = "-"
rationale = "Replace the en dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "em-dash"
source = "—"
replacement = "-"
rationale = "Replace the em dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "horizontal-bar"
source = "―"
replacement = "-"
rationale = "Replace the horizontal bar with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "minus-sign"
source = "−"
replacement = "-"
rationale = "Replace the mathematical minus sign with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "small-em-dash"
source = "﹘"
replacement = "-"
rationale = "Replace the small em dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "small-hyphen-minus"
source = "﹣"
replacement = "-"
rationale = "Replace the small hyphen-minus with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "fullwidth-hyphen-minus"
source = "-"
replacement = "-"
rationale = "Replace the fullwidth hyphen-minus with ASCII hyphen-minus."
enabled = true

View File

@ -166,51 +166,3 @@ enabled = true
[rules.scope]
content = ["source-code"]
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
[[rules]]
id = "typography-non-ascii-hyphen"
category = "typography"
severity = "advice"
matcher = "replacement"
pattern = "configured-replacements"
weight = 10.0
message = "a configured non-ASCII character has an ASCII replacement"
help = "Review the punctuation or apply the configured safe replacement."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-ascii-confusable"
category = "typography"
severity = "advice"
matcher = "confusable"
pattern = "uts39-ascii-skeleton-v16"
weight = 4.0
message = "a Unicode character visually resembles ASCII under UTS #39"
help = "Review whether the lookalike character is intentional; it is not automatically rewritten."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-invisible-control"
category = "typography"
severity = "warning"
matcher = "invisible"
pattern = "gitea-invisible-9c715c0"
weight = 12.0
message = "an invisible or bidirectional Unicode control appears in the source"
help = "Review the hidden or direction-changing character; it is not removed automatically."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,163 @@
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
use serde::Serialize;
const CURATED: &[(&str, &str, &str, &str)] = &[
("hyphen", "‐", "-", "Unicode hyphen"),
("non-breaking-hyphen", "‑", "-", "non-breaking hyphen"),
("figure-dash", "‒", "-", "figure dash"),
("en-dash", "–", "-", "en dash"),
("em-dash", "—", "-", "em dash"),
("horizontal-bar", "―", "-", "horizontal bar"),
("minus-sign", "−", "-", "mathematical minus sign"),
("small-em-dash", "﹘", "-", "small em dash"),
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
(
"fullwidth-hyphen-minus",
"-",
"-",
"fullwidth hyphen-minus",
),
];
#[derive(Serialize)]
struct Catalog {
schema_version: u32,
replacements: Vec<Replacement>,
rules: Vec<Rule>,
}
#[derive(Serialize)]
struct Replacement {
id: String,
source: String,
replacement: String,
rationale: String,
enabled: bool,
}
#[derive(Serialize)]
struct Rule {
id: &'static str,
category: &'static str,
severity: &'static str,
matcher: &'static str,
pattern: &'static str,
weight: f64,
message: &'static str,
help: &'static str,
occurrence_cap: usize,
enabled: bool,
scope: Scope,
}
#[derive(Serialize)]
struct Scope {
content: [&'static str; 2],
extensions: [String; 0],
}
fn main() {
let mut replacements = CURATED
.iter()
.map(|(id, source, replacement, name)| Replacement {
id: (*id).to_owned(),
source: (*source).to_owned(),
replacement: (*replacement).to_owned(),
rationale: format!("Replace the {name} with reviewed ASCII `{replacement}`."),
enabled: true,
})
.collect::<Vec<_>>();
for value in 0..=char::MAX as u32 {
let Some(character) = char::from_u32(value) else {
continue;
};
if character.is_ascii()
|| CURATED
.iter()
.any(|(_, source, _, _)| source.starts_with(character))
{
continue;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if skeleton.is_empty()
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|| skeleton == source
{
continue;
}
replacements.push(Replacement {
id: format!("uts39-u{value:04x}"),
source,
replacement: skeleton.clone(),
rationale: format!(
"Replace Unicode 16.0 UTS #39 lookalike U+{value:04X} with its printable ASCII skeleton {skeleton:?}."
),
enabled: true,
});
}
let catalog = Catalog {
schema_version: 1,
replacements,
rules: vec![
Rule {
id: "unicode-ascii-replacement",
category: "typography",
severity: "advice",
matcher: "replacement",
pattern: "configured-replacements",
weight: 10.0,
message: "a Unicode character or sequence has a configured ASCII replacement",
help: "Review the character or apply the configured ASCII replacement.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
Rule {
id: "unicode-ascii-confusable",
category: "typography",
severity: "advice",
matcher: "confusable",
pattern: "uts39-ascii-skeleton-v16",
weight: 4.0,
message: "a Unicode character visually resembles ASCII under UTS #39",
help: "Review whether the lookalike character is intentional.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
Rule {
id: "unicode-invisible-control",
category: "typography",
severity: "warning",
matcher: "invisible",
pattern: "gitea-invisible-9c715c0",
weight: 12.0,
message: "an invisible or bidirectional Unicode control appears in the source",
help: "Review the hidden or direction-changing character; it is not removed automatically.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
],
};
println!(
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
);
}

View File

@ -15,6 +15,7 @@ use crate::{Category, FindingSeverity};
const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml");
const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml");
const EMBEDDED_UNICODE: &str = include_str!("../assets/unicode.toml");
/// A validated named configuration without path syntax or a file extension.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@ -916,11 +917,20 @@ impl Default for InputLimits {
}
impl Config {
/// Load the replacement defaults embedded in this build.
/// Load all defaults embedded in this build.
pub fn embedded() -> Self {
let config: Self = embedded_figment()
let mut config: Self = embedded_figment()
.extract()
.expect("embedded configuration must deserialize");
let unicode_layer = Figment::from(Toml::string(EMBEDDED_UNICODE))
.extract::<ConfigLayer>()
.expect("embedded Unicode policy must deserialize");
apply_layer(
&mut config,
unicode_layer,
std::path::Path::new("<embedded unicode.toml>"),
)
.expect("embedded Unicode policy must merge");
config
.validate()
.expect("embedded configuration must be semantically valid");
@ -1151,7 +1161,7 @@ mod tests {
.map(|rule| rule.source.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(config.replacements.len(), 10);
assert_eq!(config.replacements.len(), 1_829);
assert_eq!(config.schema_version, 1);
assert_eq!(ids.len(), config.replacements.len());
assert_eq!(sources.len(), config.replacements.len());
@ -1161,6 +1171,37 @@ mod tests {
&& !rule.rationale.is_empty()));
}
#[test]
fn embedded_unicode_catalog_covers_every_printable_ascii_skeleton() {
let config = Config::embedded();
let replacements = config
.replacements
.iter()
.map(|rule| (rule.source.as_str(), rule.replacement.as_str()))
.collect::<std::collections::BTreeMap<_, _>>();
for value in 0..=char::MAX as u32 {
let Some(character) = char::from_u32(value) else {
continue;
};
if character.is_ascii() {
continue;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if !skeleton.is_empty()
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
&& skeleton != source
{
assert_eq!(
replacements.get(source.as_str()),
Some(&skeleton.as_str()),
"catalog drift for U+{value:04X}"
);
}
}
}
#[test]
fn default_input_limits_match_the_accepted_policy() {
let limits = InputLimits::default();
@ -1311,7 +1352,7 @@ weight = 9.5
let config = ConfigLoader::from_paths(paths).load().unwrap();
assert_eq!(config.replacements.len(), 11);
assert_eq!(config.replacements.len(), 1_830);
let replacement = config
.replacements
.iter()

View File

@ -566,7 +566,7 @@ mod tests {
assert_eq!(analysis.findings.len(), 1);
assert_eq!(
analysis.findings[0].rule_id.as_str(),
"typography-non-ascii-hyphen"
"unicode-ascii-replacement"
);
assert_eq!(analysis.findings[0].matched_text, "—");
assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-");
@ -575,17 +575,16 @@ mod tests {
}
#[test]
fn unicode_review_findings_have_exact_spans_and_never_become_fixes() {
fn unicode_review_findings_have_exact_spans_and_safe_fixes() {
let detector = Detector::new(&Config::embedded()).unwrap();
let source = "é pаss\u{200b}word\u{202e}\r\nnext";
let findings = detector.analyze(source).findings;
assert_eq!(findings.len(), 3);
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-confusable");
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
assert_eq!(findings[0].matched_text, "а");
assert_eq!(&source[findings[0].byte_span.clone()], "а");
assert!(findings[0].help.as_deref().unwrap().contains("\"a\""));
assert!(findings.iter().all(|finding| finding.fix.is_none()));
assert_eq!(findings[0].fix.as_ref().unwrap().replacement, "a");
assert_eq!(
findings
.iter()
@ -593,6 +592,7 @@ mod tests {
.count(),
2
);
assert!(findings[1..].iter().all(|finding| finding.fix.is_none()));
assert!(findings.iter().all(|finding| finding.matched_text != "\r"));
}
@ -604,7 +604,7 @@ mod tests {
.findings;
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].rule_id.as_str(), "typography-non-ascii-hyphen");
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
}
#[test]
@ -675,7 +675,7 @@ mod tests {
"repeated-transition",
"source-placeholder-todo",
"source-tutorial-narration",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
] {
assert!(ids.contains(expected), "missing {expected}");
}

View File

@ -1,6 +1,6 @@
//! Pure planning and application of configured text replacements.
use std::{borrow::Cow, ops::Range};
use std::{borrow::Cow, collections::BTreeMap, ops::Range};
use miette::Diagnostic;
@ -36,12 +36,18 @@ pub enum ReplacementError {
/// Plan deterministic non-overlapping edits without modifying the source.
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
let mut rules = config
.replacements
.iter()
.filter(|rule| rule.enabled)
.collect::<Vec<_>>();
rules.sort_by(|left, right| {
let mut single_character_rules = BTreeMap::new();
let mut sequence_rules = Vec::new();
for rule in config.replacements.iter().filter(|rule| rule.enabled) {
let mut characters = rule.source.chars();
let first = characters.next();
if let (Some(character), None) = (first, characters.next()) {
single_character_rules.insert(character, rule);
} else {
sequence_rules.push(rule);
}
}
sequence_rules.sort_by(|left, right| {
right
.source
.len()
@ -52,7 +58,7 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
let mut edits = Vec::new();
let mut offset = 0;
while offset < source.len() {
if let Some(rule) = rules
if let Some(rule) = sequence_rules
.iter()
.find(|rule| source[offset..].starts_with(&rule.source))
{
@ -63,7 +69,17 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
});
offset = end;
} else {
offset += source[offset..].chars().next().map_or(0, char::len_utf8);
let Some(character) = source[offset..].chars().next() else {
break;
};
let end = offset + character.len_utf8();
if let Some(rule) = single_character_rules.get(&character) {
edits.push(TextEdit {
byte_span: offset..end,
replacement: rule.replacement.clone(),
});
}
offset = end;
}
}
edits

View File

@ -549,7 +549,7 @@ changelog, and the generated badge agree on `0.1.21`.
## Milestone 22 — Maintainable ambiguous-Unicode replacement catalog
- [ ] Define one versioned Unicode policy asset that owns the replacement
- [x] Define one versioned Unicode policy asset that owns the replacement
diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39
ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a
deterministic generator with pinned Unicode provenance and drift checks.