diff --git a/crates/john-slop-core/assets/unicode.toml b/crates/john-slop-core/assets/unicode.toml index 04e8d5f..2653f4d 100644 --- a/crates/john-slop-core/assets/unicode.toml +++ b/crates/john-slop-core/assets/unicode.toml @@ -12825,25 +12825,6 @@ content = [ ] extensions = [] -[[rules]] -id = "unicode-ascii-confusable" -category = "typography" -severity = "advice" -matcher = "confusable" -pattern = "uts39-ascii-skeleton-v16" -weight = 4.0 -message = "a Unicode character visually resembles ASCII under UTS #39" -help = "Review whether the lookalike character is intentional." -occurrence_cap = 5 -enabled = true - -[rules.scope] -content = [ - "prose", - "source-code", -] -extensions = [] - [[rules]] id = "unicode-invisible-control" category = "typography" diff --git a/crates/john-slop-core/examples/generate_unicode_catalog.rs b/crates/john-slop-core/examples/generate_unicode_catalog.rs index 0a0bb60..1c82c29 100644 --- a/crates/john-slop-core/examples/generate_unicode_catalog.rs +++ b/crates/john-slop-core/examples/generate_unicode_catalog.rs @@ -119,22 +119,6 @@ fn main() { extensions: [], }, }, - Rule { - id: "unicode-ascii-confusable", - category: "typography", - severity: "advice", - matcher: "confusable", - pattern: "uts39-ascii-skeleton-v16", - weight: 4.0, - message: "a Unicode character visually resembles ASCII under UTS #39", - help: "Review whether the lookalike character is intentional.", - occurrence_cap: 5, - enabled: true, - scope: Scope { - content: ["prose", "source-code"], - extensions: [], - }, - }, Rule { id: "unicode-invisible-control", category: "typography", diff --git a/crates/john-slop-core/src/config.rs b/crates/john-slop-core/src/config.rs index d293299..12c17fa 100644 --- a/crates/john-slop-core/src/config.rs +++ b/crates/john-slop-core/src/config.rs @@ -1218,7 +1218,7 @@ mod tests { fn embedded_heuristics_have_complete_metadata() { let config = Config::embedded(); - assert_eq!(config.rules.len(), 13); + assert_eq!(config.rules.len(), 12); assert_eq!(config.thresholds.low, 20); assert_eq!(config.thresholds.medium, 50); assert_eq!(config.thresholds.high, 75); @@ -1335,6 +1335,10 @@ mod tests { id = "em-dash" enabled = false +[[replacements]] +id = "uts39-u0430" +enabled = false + [[replacements]] id = "right-arrow" source = "→" @@ -1361,8 +1365,16 @@ weight = 9.5 assert!(!replacement.enabled); assert_eq!(replacement.source, "—"); assert_eq!(replacement.replacement, "-"); + assert!( + !config + .replacements + .iter() + .find(|rule| rule.id == "uts39-u0430") + .unwrap() + .enabled + ); assert_eq!(config.replacements.last().unwrap().id, "right-arrow"); - assert_eq!(config.rules.len(), 13); + assert_eq!(config.rules.len(), 12); let heuristic = config .rules .iter() diff --git a/crates/john-slop-core/src/replacement.rs b/crates/john-slop-core/src/replacement.rs index 7e7ee6e..410a945 100644 --- a/crates/john-slop-core/src/replacement.rs +++ b/crates/john-slop-core/src/replacement.rs @@ -165,6 +165,16 @@ mod tests { ); } + #[test] + fn generated_catalog_replaces_letters_digits_punctuation_and_multi_byte_skeletons() { + let source = "а𝐀!𝟘ⅿ🯹"; + let edits = plan(&Config::embedded(), source); + + assert_eq!(edits.len(), 6); + assert_eq!(apply(source, &edits).unwrap(), "aA!Orn9"); + assert_eq!(edits[4].replacement, "rn"); + } + #[test] fn empty_or_ascii_sources_need_no_edits() { let config = Config::embedded(); diff --git a/crates/john-slop-lsp/src/lib.rs b/crates/john-slop-lsp/src/lib.rs index aa70979..8b6a5a8 100644 --- a/crates/john-slop-lsp/src/lib.rs +++ b/crates/john-slop-lsp/src/lib.rs @@ -790,7 +790,7 @@ mod tests { assert_eq!( diagnostics[1].code, Some(NumberOrString::String( - "typography-non-ascii-hyphen".to_owned() + "unicode-ascii-replacement".to_owned() )) ); assert_eq!(diagnostics[1].data.as_ref().unwrap()["fixable"], true); diff --git a/examples/diagnostic-showcase.toml b/examples/diagnostic-showcase.toml index bfe0899..678272a 100644 --- a/examples/diagnostic-showcase.toml +++ b/examples/diagnostic-showcase.toml @@ -7,5 +7,5 @@ severity = "error" # Let the companion source fixture display every embedded replacement character # instead of stopping at the production catalog's normal per-document cap. [[rules]] -id = "typography-non-ascii-hyphen" +id = "unicode-ascii-replacement" occurrence_cap = 10 diff --git a/examples/fixtures/manifest.toml b/examples/fixtures/manifest.toml index f6ba57a..befc8db 100644 --- a/examples/fixtures/manifest.toml +++ b/examples/fixtures/manifest.toml @@ -14,7 +14,7 @@ expected_rules = [ "assistant-answer-preface", "assistant-capability-disclaimer", "assistant-self-identification", - "typography-non-ascii-hyphen", + "unicode-ascii-replacement", ] expected_replacements = 1 @@ -31,14 +31,14 @@ expected_replacements = 0 [[cases]] path = "typography/example.txt" formatted = "typography/formatted.txt" -expected_rules = ["typography-non-ascii-hyphen"] +expected_rules = ["unicode-ascii-replacement"] expected_replacements = 3 [[cases]] path = "unicode-review/example.txt" formatted = "unicode-review/formatted.txt" -expected_rules = ["unicode-ascii-confusable", "unicode-invisible-control"] -expected_replacements = 0 +expected_rules = ["unicode-ascii-replacement", "unicode-invisible-control"] +expected_replacements = 1 [[cases]] path = "source/example.rs" @@ -46,6 +46,6 @@ formatted = "source/formatted.rs" expected_rules = [ "source-placeholder-todo", "source-tutorial-narration", - "typography-non-ascii-hyphen", + "unicode-ascii-replacement", ] expected_replacements = 1 diff --git a/examples/fixtures/unicode-review/formatted.txt b/examples/fixtures/unicode-review/formatted.txt index 3a480ba..82b8b15 100644 --- a/examples/fixtures/unicode-review/formatted.txt +++ b/examples/fixtures/unicode-review/formatted.txt @@ -1 +1 @@ -The pаss​word contains lookalike and invisible characters. +The pass​word contains lookalike and invisible characters. diff --git a/src/bin/john-slop/output.rs b/src/bin/john-slop/output.rs index e3e32c0..28cbb87 100644 --- a/src/bin/john-slop/output.rs +++ b/src/bin/john-slop/output.rs @@ -322,7 +322,7 @@ mod tests { ) .unwrap(); - assert!(output.contains("typography-non-ascii-hyphen")); + assert!(output.contains("unicode-ascii-replacement")); assert!(output.contains("article.md")); assert!(output.contains('╭')); assert!(output.contains('╰')); @@ -332,7 +332,7 @@ mod tests { let lines = output.lines().collect::>(); assert!(lines[0].starts_with("◆ ")); - assert!(lines[0].contains("typography-non-ascii-hyphen")); + assert!(lines[0].contains("unicode-ascii-replacement")); assert!(!lines[1].is_empty(), "title and message must stay adjacent"); let help_line = lines .iter() @@ -382,7 +382,7 @@ mod tests { let output = render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap(); - assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]")); + assert!(output.contains("article.md:8:11: advice[unicode-ascii-replacement]")); assert!(output.contains("article.md: summary: evidence_score=")); assert!(output.contains("advice=1")); assert_eq!( diff --git a/tasks.md b/tasks.md index f74ea8e..70ae3b7 100644 --- a/tasks.md +++ b/tasks.md @@ -553,7 +553,7 @@ changelog, and the generated badge agree on `0.1.21`. diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39 ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a deterministic generator with pinned Unicode provenance and drift checks. -- [ ] Make the shared replacement planner efficient for the expanded catalog, +- [x] Make the shared replacement planner efficient for the expanded catalog, load the catalog through every configuration path, and use its mappings for CLI fixes, formatter output, LSP quick fixes, and LSP document formatting. Preserve review-only behavior for invisible and bidirectional controls. diff --git a/tests/cli.rs b/tests/cli.rs index 0a17605..78722ae 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -65,7 +65,7 @@ fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() { assert!(output.status.success()); assert!(output.stderr.is_empty()); - assert!(stdout.contains("typography-non-ascii-hyphen")); + assert!(stdout.contains("unicode-ascii-replacement")); #[cfg(feature = "fancy-diagnostics")] { assert!(stdout.contains('\u{1b}')); @@ -121,7 +121,7 @@ fn config_show_emits_resolved_toml_with_explicit_overrides() { let resolved = toml::from_str::(&stdout).expect("valid TOML"); assert_eq!(resolved["schema_version"].as_integer(), Some(1)); assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23)); - assert_eq!(resolved["replacements"].as_array().unwrap().len(), 10); + assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_829); } #[test] @@ -196,16 +196,19 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() { } #[test] -fn unicode_review_diagnostics_are_non_fixing_and_exact() { +fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() { let source = "pаss\u{200b}word"; let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes()); assert!(output.status.success()); let record = &json_records(&output)[0]; assert_eq!(record["summary"]["total"], 2); - assert_eq!(record["findings"][0]["rule_id"], "unicode-ascii-confusable"); + assert_eq!( + record["findings"][0]["rule_id"], + "unicode-ascii-replacement" + ); assert_eq!(record["findings"][0]["matched_text"], "а"); - assert!(record["findings"][0]["fix"].is_null()); + assert_eq!(record["findings"][0]["fix"]["replacement"], "a"); assert_eq!( record["findings"][1]["rule_id"], "unicode-invisible-control" diff --git a/tests/evaluation.rs b/tests/evaluation.rs index 7053696..95ba943 100644 --- a/tests/evaluation.rs +++ b/tests/evaluation.rs @@ -88,7 +88,12 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() { tp, tp + false_negatives ); - assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}"); + let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1; + assert_eq!( + (tp, fp, false_negatives), + (expected_true_positives, 0, 0), + "metrics for {id}" + ); total_tp += tp; total_fp += fp; total_fn += false_negatives; diff --git a/tests/example_fixtures.rs b/tests/example_fixtures.rs index fc65658..dde3b8b 100644 --- a/tests/example_fixtures.rs +++ b/tests/example_fixtures.rs @@ -159,8 +159,7 @@ fn diagnostic_showcase_spans_prose_source_and_severities() { assert!(output.stderr.is_empty()); assert!(stdout.contains("assistant-self-identification")); assert!(stdout.contains("prompt-response-scaffolding")); - assert!(stdout.contains("typography-non-ascii-hyphen")); - assert!(stdout.contains("unicode-ascii-confusable")); + assert!(stdout.contains("unicode-ascii-replacement")); assert!(stdout.contains("unicode-invisible-control")); assert!(stdout.contains("source-placeholder-todo")); assert!(stdout.contains("source-tutorial-narration")); diff --git a/tests/fixtures/evaluation/manifest.toml b/tests/fixtures/evaluation/manifest.toml index 5568115..d6dc952 100644 --- a/tests/fixtures/evaluation/manifest.toml +++ b/tests/fixtures/evaluation/manifest.toml @@ -34,13 +34,13 @@ expected = ["source-placeholder-todo", "source-tutorial-narration"] path = "typography.md" kind = "prose" expected_score = 17 -expected = ["typography-non-ascii-hyphen"] +expected = ["unicode-ascii-replacement"] [[cases]] path = "unicode.md" kind = "prose" -expected_score = 21 -expected = ["unicode-ascii-confusable", "unicode-invisible-control"] +expected_score = 28 +expected = ["unicode-ascii-replacement", "unicode-invisible-control"] [[cases]] path = "negative.md" @@ -48,8 +48,7 @@ kind = "prose" expected_score = 0 expected = [] near_miss_for = [ - "typography-non-ascii-hyphen", - "unicode-ascii-confusable", + "unicode-ascii-replacement", "unicode-invisible-control", ] diff --git a/tests/heuristics.rs b/tests/heuristics.rs index 539935e..185ee7d 100644 --- a/tests/heuristics.rs +++ b/tests/heuristics.rs @@ -93,17 +93,11 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() { kind: FixtureKind::Rust, }, RuleCase { - id: "typography-non-ascii-hyphen", + id: "unicode-ascii-replacement", positive: "Review this—then continue.", near_miss: "Review this-then continue.", kind: FixtureKind::Prose, }, - RuleCase { - id: "unicode-ascii-confusable", - positive: "The pаssword contains a Cyrillic lookalike.", - near_miss: "Résumé précis en français.", - kind: FixtureKind::Prose, - }, RuleCase { id: "unicode-invisible-control", positive: "A zero-width\u{200b}space is hidden here.", diff --git a/tests/lsp.rs b/tests/lsp.rs index 428f40e..7b9563b 100644 --- a/tests/lsp.rs +++ b/tests/lsp.rs @@ -218,7 +218,7 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() { } #[tokio::test] -async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() { +async fn unicode_review_findings_separate_safe_fixes_from_controls() { let mut harness = Harness::new(None).await; let uri = "file:///tmp/unicode-review.txt"; let source = "pаss\u{200b}word\u{202e}"; @@ -230,8 +230,8 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() { .await; let diagnostics = published["diagnostics"].as_array().unwrap(); assert_eq!(diagnostics.len(), 3); - assert_eq!(diagnostics[0]["code"], "unicode-ascii-confusable"); - assert_eq!(diagnostics[0]["data"]["fixable"], false); + assert_eq!(diagnostics[0]["code"], "unicode-ascii-replacement"); + assert_eq!(diagnostics[0]["data"]["fixable"], true); assert_eq!(diagnostics[1]["code"], "unicode-invisible-control"); assert_eq!(diagnostics[2]["code"], "unicode-invisible-control"); @@ -245,7 +245,9 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() { 40, ) .await; - assert!(formatting.result().unwrap().as_array().unwrap().is_empty()); + let formatting = formatting.result().unwrap().as_array().unwrap(); + assert_eq!(formatting.len(), 1); + assert_eq!(formatting[0]["newText"], "a"); let actions = harness .request( @@ -261,7 +263,12 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() { 41, ) .await; - assert!(actions.result().unwrap().as_array().unwrap().is_empty()); + let actions = actions.result().unwrap().as_array().unwrap(); + assert_eq!(actions.len(), 1); + assert_eq!( + actions[0]["edit"]["documentChanges"][0]["edits"][0]["newText"], + "a" + ); } fn utf16_position_to_byte(source: &str, position: &Value) -> usize {