feat: replace printable ascii confusables consistently
This commit is contained in:
parent
82afdc7a3f
commit
f229c1445c
|
|
@ -12825,25 +12825,6 @@ content = [
|
|||
]
|
||||
extensions = []
|
||||
|
||||
[[rules]]
|
||||
id = "unicode-ascii-confusable"
|
||||
category = "typography"
|
||||
severity = "advice"
|
||||
matcher = "confusable"
|
||||
pattern = "uts39-ascii-skeleton-v16"
|
||||
weight = 4.0
|
||||
message = "a Unicode character visually resembles ASCII under UTS #39"
|
||||
help = "Review whether the lookalike character is intentional."
|
||||
occurrence_cap = 5
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = [
|
||||
"prose",
|
||||
"source-code",
|
||||
]
|
||||
extensions = []
|
||||
|
||||
[[rules]]
|
||||
id = "unicode-invisible-control"
|
||||
category = "typography"
|
||||
|
|
|
|||
|
|
@ -119,22 +119,6 @@ fn main() {
|
|||
extensions: [],
|
||||
},
|
||||
},
|
||||
Rule {
|
||||
id: "unicode-ascii-confusable",
|
||||
category: "typography",
|
||||
severity: "advice",
|
||||
matcher: "confusable",
|
||||
pattern: "uts39-ascii-skeleton-v16",
|
||||
weight: 4.0,
|
||||
message: "a Unicode character visually resembles ASCII under UTS #39",
|
||||
help: "Review whether the lookalike character is intentional.",
|
||||
occurrence_cap: 5,
|
||||
enabled: true,
|
||||
scope: Scope {
|
||||
content: ["prose", "source-code"],
|
||||
extensions: [],
|
||||
},
|
||||
},
|
||||
Rule {
|
||||
id: "unicode-invisible-control",
|
||||
category: "typography",
|
||||
|
|
|
|||
|
|
@ -1218,7 +1218,7 @@ mod tests {
|
|||
fn embedded_heuristics_have_complete_metadata() {
|
||||
let config = Config::embedded();
|
||||
|
||||
assert_eq!(config.rules.len(), 13);
|
||||
assert_eq!(config.rules.len(), 12);
|
||||
assert_eq!(config.thresholds.low, 20);
|
||||
assert_eq!(config.thresholds.medium, 50);
|
||||
assert_eq!(config.thresholds.high, 75);
|
||||
|
|
@ -1335,6 +1335,10 @@ mod tests {
|
|||
id = "em-dash"
|
||||
enabled = false
|
||||
|
||||
[[replacements]]
|
||||
id = "uts39-u0430"
|
||||
enabled = false
|
||||
|
||||
[[replacements]]
|
||||
id = "right-arrow"
|
||||
source = "→"
|
||||
|
|
@ -1361,8 +1365,16 @@ weight = 9.5
|
|||
assert!(!replacement.enabled);
|
||||
assert_eq!(replacement.source, "—");
|
||||
assert_eq!(replacement.replacement, "-");
|
||||
assert!(
|
||||
!config
|
||||
.replacements
|
||||
.iter()
|
||||
.find(|rule| rule.id == "uts39-u0430")
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
|
||||
assert_eq!(config.rules.len(), 13);
|
||||
assert_eq!(config.rules.len(), 12);
|
||||
let heuristic = config
|
||||
.rules
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -165,6 +165,16 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_catalog_replaces_letters_digits_punctuation_and_multi_byte_skeletons() {
|
||||
let source = "а𝐀!𝟘ⅿ🯹";
|
||||
let edits = plan(&Config::embedded(), source);
|
||||
|
||||
assert_eq!(edits.len(), 6);
|
||||
assert_eq!(apply(source, &edits).unwrap(), "aA!Orn9");
|
||||
assert_eq!(edits[4].replacement, "rn");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_or_ascii_sources_need_no_edits() {
|
||||
let config = Config::embedded();
|
||||
|
|
|
|||
|
|
@ -790,7 +790,7 @@ mod tests {
|
|||
assert_eq!(
|
||||
diagnostics[1].code,
|
||||
Some(NumberOrString::String(
|
||||
"typography-non-ascii-hyphen".to_owned()
|
||||
"unicode-ascii-replacement".to_owned()
|
||||
))
|
||||
);
|
||||
assert_eq!(diagnostics[1].data.as_ref().unwrap()["fixable"], true);
|
||||
|
|
|
|||
|
|
@ -7,5 +7,5 @@ severity = "error"
|
|||
# Let the companion source fixture display every embedded replacement character
|
||||
# instead of stopping at the production catalog's normal per-document cap.
|
||||
[[rules]]
|
||||
id = "typography-non-ascii-hyphen"
|
||||
id = "unicode-ascii-replacement"
|
||||
occurrence_cap = 10
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ expected_rules = [
|
|||
"assistant-answer-preface",
|
||||
"assistant-capability-disclaimer",
|
||||
"assistant-self-identification",
|
||||
"typography-non-ascii-hyphen",
|
||||
"unicode-ascii-replacement",
|
||||
]
|
||||
expected_replacements = 1
|
||||
|
||||
|
|
@ -31,14 +31,14 @@ expected_replacements = 0
|
|||
[[cases]]
|
||||
path = "typography/example.txt"
|
||||
formatted = "typography/formatted.txt"
|
||||
expected_rules = ["typography-non-ascii-hyphen"]
|
||||
expected_rules = ["unicode-ascii-replacement"]
|
||||
expected_replacements = 3
|
||||
|
||||
[[cases]]
|
||||
path = "unicode-review/example.txt"
|
||||
formatted = "unicode-review/formatted.txt"
|
||||
expected_rules = ["unicode-ascii-confusable", "unicode-invisible-control"]
|
||||
expected_replacements = 0
|
||||
expected_rules = ["unicode-ascii-replacement", "unicode-invisible-control"]
|
||||
expected_replacements = 1
|
||||
|
||||
[[cases]]
|
||||
path = "source/example.rs"
|
||||
|
|
@ -46,6 +46,6 @@ formatted = "source/formatted.rs"
|
|||
expected_rules = [
|
||||
"source-placeholder-todo",
|
||||
"source-tutorial-narration",
|
||||
"typography-non-ascii-hyphen",
|
||||
"unicode-ascii-replacement",
|
||||
]
|
||||
expected_replacements = 1
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
The pаssword contains lookalike and invisible characters.
|
||||
The password contains lookalike and invisible characters.
|
||||
|
|
|
|||
|
|
@ -322,7 +322,7 @@ mod tests {
|
|||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(output.contains("typography-non-ascii-hyphen"));
|
||||
assert!(output.contains("unicode-ascii-replacement"));
|
||||
assert!(output.contains("article.md"));
|
||||
assert!(output.contains('╭'));
|
||||
assert!(output.contains('╰'));
|
||||
|
|
@ -332,7 +332,7 @@ mod tests {
|
|||
|
||||
let lines = output.lines().collect::<Vec<_>>();
|
||||
assert!(lines[0].starts_with("◆ "));
|
||||
assert!(lines[0].contains("typography-non-ascii-hyphen"));
|
||||
assert!(lines[0].contains("unicode-ascii-replacement"));
|
||||
assert!(!lines[1].is_empty(), "title and message must stay adjacent");
|
||||
let help_line = lines
|
||||
.iter()
|
||||
|
|
@ -382,7 +382,7 @@ mod tests {
|
|||
let output =
|
||||
render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap();
|
||||
|
||||
assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]"));
|
||||
assert!(output.contains("article.md:8:11: advice[unicode-ascii-replacement]"));
|
||||
assert!(output.contains("article.md: summary: evidence_score="));
|
||||
assert!(output.contains("advice=1"));
|
||||
assert_eq!(
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -553,7 +553,7 @@ changelog, and the generated badge agree on `0.1.21`.
|
|||
diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39
|
||||
ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a
|
||||
deterministic generator with pinned Unicode provenance and drift checks.
|
||||
- [ ] Make the shared replacement planner efficient for the expanded catalog,
|
||||
- [x] Make the shared replacement planner efficient for the expanded catalog,
|
||||
load the catalog through every configuration path, and use its mappings for
|
||||
CLI fixes, formatter output, LSP quick fixes, and LSP document formatting.
|
||||
Preserve review-only behavior for invisible and bidirectional controls.
|
||||
|
|
|
|||
13
tests/cli.rs
13
tests/cli.rs
|
|
@ -65,7 +65,7 @@ fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() {
|
|||
|
||||
assert!(output.status.success());
|
||||
assert!(output.stderr.is_empty());
|
||||
assert!(stdout.contains("typography-non-ascii-hyphen"));
|
||||
assert!(stdout.contains("unicode-ascii-replacement"));
|
||||
#[cfg(feature = "fancy-diagnostics")]
|
||||
{
|
||||
assert!(stdout.contains('\u{1b}'));
|
||||
|
|
@ -121,7 +121,7 @@ fn config_show_emits_resolved_toml_with_explicit_overrides() {
|
|||
let resolved = toml::from_str::<toml::Value>(&stdout).expect("valid TOML");
|
||||
assert_eq!(resolved["schema_version"].as_integer(), Some(1));
|
||||
assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23));
|
||||
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 10);
|
||||
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_829);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -196,16 +196,19 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_review_diagnostics_are_non_fixing_and_exact() {
|
||||
fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() {
|
||||
let source = "pаss\u{200b}word";
|
||||
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
|
||||
|
||||
assert!(output.status.success());
|
||||
let record = &json_records(&output)[0];
|
||||
assert_eq!(record["summary"]["total"], 2);
|
||||
assert_eq!(record["findings"][0]["rule_id"], "unicode-ascii-confusable");
|
||||
assert_eq!(
|
||||
record["findings"][0]["rule_id"],
|
||||
"unicode-ascii-replacement"
|
||||
);
|
||||
assert_eq!(record["findings"][0]["matched_text"], "а");
|
||||
assert!(record["findings"][0]["fix"].is_null());
|
||||
assert_eq!(record["findings"][0]["fix"]["replacement"], "a");
|
||||
assert_eq!(
|
||||
record["findings"][1]["rule_id"],
|
||||
"unicode-invisible-control"
|
||||
|
|
|
|||
|
|
@ -88,7 +88,12 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
|||
tp,
|
||||
tp + false_negatives
|
||||
);
|
||||
assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}");
|
||||
let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1;
|
||||
assert_eq!(
|
||||
(tp, fp, false_negatives),
|
||||
(expected_true_positives, 0, 0),
|
||||
"metrics for {id}"
|
||||
);
|
||||
total_tp += tp;
|
||||
total_fp += fp;
|
||||
total_fn += false_negatives;
|
||||
|
|
|
|||
|
|
@ -159,8 +159,7 @@ fn diagnostic_showcase_spans_prose_source_and_severities() {
|
|||
assert!(output.stderr.is_empty());
|
||||
assert!(stdout.contains("assistant-self-identification"));
|
||||
assert!(stdout.contains("prompt-response-scaffolding"));
|
||||
assert!(stdout.contains("typography-non-ascii-hyphen"));
|
||||
assert!(stdout.contains("unicode-ascii-confusable"));
|
||||
assert!(stdout.contains("unicode-ascii-replacement"));
|
||||
assert!(stdout.contains("unicode-invisible-control"));
|
||||
assert!(stdout.contains("source-placeholder-todo"));
|
||||
assert!(stdout.contains("source-tutorial-narration"));
|
||||
|
|
|
|||
|
|
@ -34,13 +34,13 @@ expected = ["source-placeholder-todo", "source-tutorial-narration"]
|
|||
path = "typography.md"
|
||||
kind = "prose"
|
||||
expected_score = 17
|
||||
expected = ["typography-non-ascii-hyphen"]
|
||||
expected = ["unicode-ascii-replacement"]
|
||||
|
||||
[[cases]]
|
||||
path = "unicode.md"
|
||||
kind = "prose"
|
||||
expected_score = 21
|
||||
expected = ["unicode-ascii-confusable", "unicode-invisible-control"]
|
||||
expected_score = 28
|
||||
expected = ["unicode-ascii-replacement", "unicode-invisible-control"]
|
||||
|
||||
[[cases]]
|
||||
path = "negative.md"
|
||||
|
|
@ -48,8 +48,7 @@ kind = "prose"
|
|||
expected_score = 0
|
||||
expected = []
|
||||
near_miss_for = [
|
||||
"typography-non-ascii-hyphen",
|
||||
"unicode-ascii-confusable",
|
||||
"unicode-ascii-replacement",
|
||||
"unicode-invisible-control",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -93,17 +93,11 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
|||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "typography-non-ascii-hyphen",
|
||||
id: "unicode-ascii-replacement",
|
||||
positive: "Review this—then continue.",
|
||||
near_miss: "Review this-then continue.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "unicode-ascii-confusable",
|
||||
positive: "The pаssword contains a Cyrillic lookalike.",
|
||||
near_miss: "Résumé précis en français.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "unicode-invisible-control",
|
||||
positive: "A zero-width\u{200b}space is hidden here.",
|
||||
|
|
|
|||
17
tests/lsp.rs
17
tests/lsp.rs
|
|
@ -218,7 +218,7 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
|
||||
async fn unicode_review_findings_separate_safe_fixes_from_controls() {
|
||||
let mut harness = Harness::new(None).await;
|
||||
let uri = "file:///tmp/unicode-review.txt";
|
||||
let source = "pаss\u{200b}word\u{202e}";
|
||||
|
|
@ -230,8 +230,8 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
|
|||
.await;
|
||||
let diagnostics = published["diagnostics"].as_array().unwrap();
|
||||
assert_eq!(diagnostics.len(), 3);
|
||||
assert_eq!(diagnostics[0]["code"], "unicode-ascii-confusable");
|
||||
assert_eq!(diagnostics[0]["data"]["fixable"], false);
|
||||
assert_eq!(diagnostics[0]["code"], "unicode-ascii-replacement");
|
||||
assert_eq!(diagnostics[0]["data"]["fixable"], true);
|
||||
assert_eq!(diagnostics[1]["code"], "unicode-invisible-control");
|
||||
assert_eq!(diagnostics[2]["code"], "unicode-invisible-control");
|
||||
|
||||
|
|
@ -245,7 +245,9 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
|
|||
40,
|
||||
)
|
||||
.await;
|
||||
assert!(formatting.result().unwrap().as_array().unwrap().is_empty());
|
||||
let formatting = formatting.result().unwrap().as_array().unwrap();
|
||||
assert_eq!(formatting.len(), 1);
|
||||
assert_eq!(formatting[0]["newText"], "a");
|
||||
|
||||
let actions = harness
|
||||
.request(
|
||||
|
|
@ -261,7 +263,12 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
|
|||
41,
|
||||
)
|
||||
.await;
|
||||
assert!(actions.result().unwrap().as_array().unwrap().is_empty());
|
||||
let actions = actions.result().unwrap().as_array().unwrap();
|
||||
assert_eq!(actions.len(), 1);
|
||||
assert_eq!(
|
||||
actions[0]["edit"]["documentChanges"][0]["edits"][0]["newText"],
|
||||
"a"
|
||||
);
|
||||
}
|
||||
|
||||
fn utf16_position_to_byte(source: &str, position: &Value) -> usize {
|
||||
|
|
|
|||
Loading…
Reference in New Issue