feat: replace printable ascii confusables consistently

This commit is contained in:
Dandaedre 2026-09-30 00:27:36 +02:00
parent 82afdc7a3f
commit f229c1445c
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
16 changed files with 68 additions and 74 deletions

View File

@ -12825,25 +12825,6 @@ content = [
]
extensions = []
[[rules]]
id = "unicode-ascii-confusable"
category = "typography"
severity = "advice"
matcher = "confusable"
pattern = "uts39-ascii-skeleton-v16"
weight = 4.0
message = "a Unicode character visually resembles ASCII under UTS #39"
help = "Review whether the lookalike character is intentional."
occurrence_cap = 5
enabled = true
[rules.scope]
content = [
"prose",
"source-code",
]
extensions = []
[[rules]]
id = "unicode-invisible-control"
category = "typography"

View File

@ -119,22 +119,6 @@ fn main() {
extensions: [],
},
},
Rule {
id: "unicode-ascii-confusable",
category: "typography",
severity: "advice",
matcher: "confusable",
pattern: "uts39-ascii-skeleton-v16",
weight: 4.0,
message: "a Unicode character visually resembles ASCII under UTS #39",
help: "Review whether the lookalike character is intentional.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
Rule {
id: "unicode-invisible-control",
category: "typography",

View File

@ -1218,7 +1218,7 @@ mod tests {
fn embedded_heuristics_have_complete_metadata() {
let config = Config::embedded();
assert_eq!(config.rules.len(), 13);
assert_eq!(config.rules.len(), 12);
assert_eq!(config.thresholds.low, 20);
assert_eq!(config.thresholds.medium, 50);
assert_eq!(config.thresholds.high, 75);
@ -1335,6 +1335,10 @@ mod tests {
id = "em-dash"
enabled = false
[[replacements]]
id = "uts39-u0430"
enabled = false
[[replacements]]
id = "right-arrow"
source = "→"
@ -1361,8 +1365,16 @@ weight = 9.5
assert!(!replacement.enabled);
assert_eq!(replacement.source, "—");
assert_eq!(replacement.replacement, "-");
assert!(
!config
.replacements
.iter()
.find(|rule| rule.id == "uts39-u0430")
.unwrap()
.enabled
);
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
assert_eq!(config.rules.len(), 13);
assert_eq!(config.rules.len(), 12);
let heuristic = config
.rules
.iter()

View File

@ -165,6 +165,16 @@ mod tests {
);
}
#[test]
fn generated_catalog_replaces_letters_digits_punctuation_and_multi_byte_skeletons() {
let source = "а𝐀!𝟘ⅿ🯹";
let edits = plan(&Config::embedded(), source);
assert_eq!(edits.len(), 6);
assert_eq!(apply(source, &edits).unwrap(), "aA!Orn9");
assert_eq!(edits[4].replacement, "rn");
}
#[test]
fn empty_or_ascii_sources_need_no_edits() {
let config = Config::embedded();

View File

@ -790,7 +790,7 @@ mod tests {
assert_eq!(
diagnostics[1].code,
Some(NumberOrString::String(
"typography-non-ascii-hyphen".to_owned()
"unicode-ascii-replacement".to_owned()
))
);
assert_eq!(diagnostics[1].data.as_ref().unwrap()["fixable"], true);

View File

@ -7,5 +7,5 @@ severity = "error"
# Let the companion source fixture display every embedded replacement character
# instead of stopping at the production catalog's normal per-document cap.
[[rules]]
id = "typography-non-ascii-hyphen"
id = "unicode-ascii-replacement"
occurrence_cap = 10

View File

@ -14,7 +14,7 @@ expected_rules = [
"assistant-answer-preface",
"assistant-capability-disclaimer",
"assistant-self-identification",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
]
expected_replacements = 1
@ -31,14 +31,14 @@ expected_replacements = 0
[[cases]]
path = "typography/example.txt"
formatted = "typography/formatted.txt"
expected_rules = ["typography-non-ascii-hyphen"]
expected_rules = ["unicode-ascii-replacement"]
expected_replacements = 3
[[cases]]
path = "unicode-review/example.txt"
formatted = "unicode-review/formatted.txt"
expected_rules = ["unicode-ascii-confusable", "unicode-invisible-control"]
expected_replacements = 0
expected_rules = ["unicode-ascii-replacement", "unicode-invisible-control"]
expected_replacements = 1
[[cases]]
path = "source/example.rs"
@ -46,6 +46,6 @@ formatted = "source/formatted.rs"
expected_rules = [
"source-placeholder-todo",
"source-tutorial-narration",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
]
expected_replacements = 1

View File

@ -1 +1 @@
The pаss​word contains lookalike and invisible characters.
The pass​word contains lookalike and invisible characters.

View File

@ -322,7 +322,7 @@ mod tests {
)
.unwrap();
assert!(output.contains("typography-non-ascii-hyphen"));
assert!(output.contains("unicode-ascii-replacement"));
assert!(output.contains("article.md"));
assert!(output.contains('╭'));
assert!(output.contains('╰'));
@ -332,7 +332,7 @@ mod tests {
let lines = output.lines().collect::<Vec<_>>();
assert!(lines[0].starts_with("◆ "));
assert!(lines[0].contains("typography-non-ascii-hyphen"));
assert!(lines[0].contains("unicode-ascii-replacement"));
assert!(!lines[1].is_empty(), "title and message must stay adjacent");
let help_line = lines
.iter()
@ -382,7 +382,7 @@ mod tests {
let output =
render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap();
assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]"));
assert!(output.contains("article.md:8:11: advice[unicode-ascii-replacement]"));
assert!(output.contains("article.md: summary: evidence_score="));
assert!(output.contains("advice=1"));
assert_eq!(

View File

@ -553,7 +553,7 @@ changelog, and the generated badge agree on `0.1.21`.
diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39
ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a
deterministic generator with pinned Unicode provenance and drift checks.
- [ ] Make the shared replacement planner efficient for the expanded catalog,
- [x] Make the shared replacement planner efficient for the expanded catalog,
load the catalog through every configuration path, and use its mappings for
CLI fixes, formatter output, LSP quick fixes, and LSP document formatting.
Preserve review-only behavior for invisible and bidirectional controls.

View File

@ -65,7 +65,7 @@ fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() {
assert!(output.status.success());
assert!(output.stderr.is_empty());
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("unicode-ascii-replacement"));
#[cfg(feature = "fancy-diagnostics")]
{
assert!(stdout.contains('\u{1b}'));
@ -121,7 +121,7 @@ fn config_show_emits_resolved_toml_with_explicit_overrides() {
let resolved = toml::from_str::<toml::Value>(&stdout).expect("valid TOML");
assert_eq!(resolved["schema_version"].as_integer(), Some(1));
assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23));
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 10);
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_829);
}
#[test]
@ -196,16 +196,19 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
}
#[test]
fn unicode_review_diagnostics_are_non_fixing_and_exact() {
fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() {
let source = "pаss\u{200b}word";
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
assert!(output.status.success());
let record = &json_records(&output)[0];
assert_eq!(record["summary"]["total"], 2);
assert_eq!(record["findings"][0]["rule_id"], "unicode-ascii-confusable");
assert_eq!(
record["findings"][0]["rule_id"],
"unicode-ascii-replacement"
);
assert_eq!(record["findings"][0]["matched_text"], "а");
assert!(record["findings"][0]["fix"].is_null());
assert_eq!(record["findings"][0]["fix"]["replacement"], "a");
assert_eq!(
record["findings"][1]["rule_id"],
"unicode-invisible-control"

View File

@ -88,7 +88,12 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
tp,
tp + false_negatives
);
assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}");
let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1;
assert_eq!(
(tp, fp, false_negatives),
(expected_true_positives, 0, 0),
"metrics for {id}"
);
total_tp += tp;
total_fp += fp;
total_fn += false_negatives;

View File

@ -159,8 +159,7 @@ fn diagnostic_showcase_spans_prose_source_and_severities() {
assert!(output.stderr.is_empty());
assert!(stdout.contains("assistant-self-identification"));
assert!(stdout.contains("prompt-response-scaffolding"));
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("unicode-ascii-confusable"));
assert!(stdout.contains("unicode-ascii-replacement"));
assert!(stdout.contains("unicode-invisible-control"));
assert!(stdout.contains("source-placeholder-todo"));
assert!(stdout.contains("source-tutorial-narration"));

View File

@ -34,13 +34,13 @@ expected = ["source-placeholder-todo", "source-tutorial-narration"]
path = "typography.md"
kind = "prose"
expected_score = 17
expected = ["typography-non-ascii-hyphen"]
expected = ["unicode-ascii-replacement"]
[[cases]]
path = "unicode.md"
kind = "prose"
expected_score = 21
expected = ["unicode-ascii-confusable", "unicode-invisible-control"]
expected_score = 28
expected = ["unicode-ascii-replacement", "unicode-invisible-control"]
[[cases]]
path = "negative.md"
@ -48,8 +48,7 @@ kind = "prose"
expected_score = 0
expected = []
near_miss_for = [
"typography-non-ascii-hyphen",
"unicode-ascii-confusable",
"unicode-ascii-replacement",
"unicode-invisible-control",
]

View File

@ -93,17 +93,11 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
kind: FixtureKind::Rust,
},
RuleCase {
id: "typography-non-ascii-hyphen",
id: "unicode-ascii-replacement",
positive: "Review this—then continue.",
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-ascii-confusable",
positive: "The pаssword contains a Cyrillic lookalike.",
near_miss: "Résumé précis en français.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-invisible-control",
positive: "A zero-width\u{200b}space is hidden here.",

View File

@ -218,7 +218,7 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() {
}
#[tokio::test]
async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
async fn unicode_review_findings_separate_safe_fixes_from_controls() {
let mut harness = Harness::new(None).await;
let uri = "file:///tmp/unicode-review.txt";
let source = "pаss\u{200b}word\u{202e}";
@ -230,8 +230,8 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
.await;
let diagnostics = published["diagnostics"].as_array().unwrap();
assert_eq!(diagnostics.len(), 3);
assert_eq!(diagnostics[0]["code"], "unicode-ascii-confusable");
assert_eq!(diagnostics[0]["data"]["fixable"], false);
assert_eq!(diagnostics[0]["code"], "unicode-ascii-replacement");
assert_eq!(diagnostics[0]["data"]["fixable"], true);
assert_eq!(diagnostics[1]["code"], "unicode-invisible-control");
assert_eq!(diagnostics[2]["code"], "unicode-invisible-control");
@ -245,7 +245,9 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
40,
)
.await;
assert!(formatting.result().unwrap().as_array().unwrap().is_empty());
let formatting = formatting.result().unwrap().as_array().unwrap();
assert_eq!(formatting.len(), 1);
assert_eq!(formatting[0]["newText"], "a");
let actions = harness
.request(
@ -261,7 +263,12 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
41,
)
.await;
assert!(actions.result().unwrap().as_array().unwrap().is_empty());
let actions = actions.result().unwrap().as_array().unwrap();
assert_eq!(actions.len(), 1);
assert_eq!(
actions[0]["edit"]["documentChanges"][0]["edits"][0]["newText"],
"a"
);
}
fn utf16_position_to_byte(source: &str, position: &Value) -> usize {