Compare commits

...

4 Commits

42 changed files with 13489 additions and 384 deletions

View File

@ -1,11 +1,11 @@
[last_run_clean]
run_status = "Success"
lines_parsed = 389
coverage = 92.4
lines_parsed = 393
coverage = 93.05
[last_run_clean.run_duration]
secs = 29
nanos = 820282692
secs = 50
nanos = 858077900
[last_run_incremental]
run_status = "Success"

View File

@ -14,8 +14,10 @@ accusation. A result must always be explainable from the findings that produced
it.
The formatter scope is deliberately narrow: configured Unicode characters with
safe ASCII replacements. The LSP publishes every applicable detector finding
but exposes edits only for those safe replacements, reusing the same library
ASCII replacements. The generated UTS #39 profile is broad and therefore must
retain `replace`, non-mutating `review`, and `off` policies. The LSP publishes
every applicable detector finding but exposes edits only under replacement
policy, reusing the same library
implementations as the CLI and formatter. Do not add prose rewriting, generative
inference, or destructive archive editing without a separate design decision.
@ -156,11 +158,11 @@ field: replacement/rule tables merge by stable ID, while ignore globs and file
extensions replace as complete arrays unless the schema explicitly says
otherwise. Do not rely on a library's default array-merge semantics implicitly.
`crates/john-slop-core/assets/config.toml` is the canonical, documented
replacement catalog. Each
entry has an ID, Unicode source string/code point, ASCII replacement, rationale,
and enabled flag. Reject duplicate sources, empty sources, non-ASCII targets,
and conflicting edits. Longer sources win matching precedence. Provide a command
`crates/john-slop-core/assets/unicode.toml` is the canonical generated Unicode
replacement and diagnostic catalog; `config.toml` owns other defaults. Each
replacement entry has an ID, Unicode source string/code point, ASCII target,
rationale, and enabled flag. Reject duplicate sources, empty sources, non-ASCII
targets, and conflicting edits. Longer sources win matching precedence. Provide a command
that prints the resolved configuration with secrets/redacted fields if secrets
are ever introduced.

View File

@ -6,6 +6,23 @@ All notable changes to John Slop are recorded here. The format follows
## [Unreleased]
## [0.1.22] - 2026-09-30
### Added
- Added a generated, versioned Unicode policy containing 1,829 mappings and
their shared diagnostic metadata, plus an exhaustive drift test and
deterministic regeneration example.
- Added `replace`, `review`, and `off` bulk policies for printable-ASCII UTS #39
mappings.
### Changed
- Printable-ASCII Unicode 16.0 UTS #39 lookalikes now receive consistent fixes
from CLI diagnostics, the standalone formatter, LSP quick fixes, and LSP
document formatting under the default `replace` policy.
- Invisible and bidirectional controls remain diagnostic-only.
## [0.1.21] - 2026-09-30
### Added
@ -60,7 +77,8 @@ All notable changes to John Slop are recorded here. The format follows
- Stdio LSP replacement diagnostics and versioned quick fixes.
- Runnable examples, reviewed fixtures, fuzz targets, benchmarks, and CI gates.
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.21...HEAD
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.22...HEAD
[0.1.22]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.21...v0.1.22
[0.1.21]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.20...v0.1.21
[0.1.20]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.0...v0.1.20
[0.1.0]: https://git.dandaedre.com/idle-org/john_slop/releases/tag/v0.1.0

6
Cargo.lock generated
View File

@ -699,7 +699,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "john-slop-core"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"directories",
"figment",
@ -717,7 +717,7 @@ dependencies = [
[[package]]
name = "john-slop-lsp"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"john-slop-core",
"serde_json",
@ -728,7 +728,7 @@ dependencies = [
[[package]]
name = "john_slop"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"clap",
"criterion",

View File

@ -4,7 +4,7 @@ default-members = [".", "crates/john-slop-core", "crates/john-slop-lsp"]
resolver = "3"
[workspace.package]
version = "0.1.21"
version = "0.1.22"
edition = "2024"
rust-version = "1.98.1"
license = "MIT"
@ -18,8 +18,8 @@ figment = { version = "0.10.19", default-features = false, features = ["toml"] }
futures = { version = "0.3.34", default-features = false, features = ["std"] }
globset = { version = "0.4.20", default-features = false }
insta = { version = "1.48.0", default-features = false }
john-slop-core = { version = "0.1.21", path = "crates/john-slop-core" }
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.21", path = "crates/john-slop-lsp" }
john-slop-core = { version = "0.1.22", path = "crates/john-slop-core" }
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.22", path = "crates/john-slop-lsp" }
miette = { version = "7.6.0", default-features = false, features = ["derive"] }
regex = "1.13.1"
serde = { version = "1.0.229", features = ["derive"] }

View File

@ -55,8 +55,8 @@ spans:
- `john-slop` detects copied assistant scaffolding, generic style residue,
source placeholders, tutorial narration, configured replacements, ASCII-like
Unicode, and invisible or bidirectional controls.
- `john-slop-fmt` previews, checks, or explicitly applies safe configured ASCII
replacements.
- `john-slop-fmt` previews, checks, or explicitly applies configured ASCII
replacements, including the default printable-ASCII UTS #39 profile.
- `john-slop-lsp` publishes the detector's heuristic diagnostics over stdio,
offers quick fixes for safe replacements, and provides formatter-equivalent
document edits without writing editor-owned files.
@ -69,7 +69,8 @@ configured signals; it is not the probability that AI wrote the content.
Analysis is local and deterministic, with no network requests or telemetry.
False positives remain possible—especially in transcripts, tutorials,
multilingual text, and intentional typography—so findings are prompts for
review, not verdicts.
review, not verdicts. Projects with multilingual text should consider the
Unicode profile's non-mutating `review` mode before writing formatted output.
## Learn progressively

View File

@ -1 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="122" height="18" role="img" aria-label="test-coverage: 92%"><title>test-coverage: 92%</title><linearGradient id="bms-9dc65f7f72910b0a" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-9dc65f7f72910b0a"><rect width="122" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-9dc65f7f72910b0a)"><rect width="87" height="18" fill="#555"/><rect x="87" width="35" height="18" fill="#97ca00"/><rect width="122" height="18" fill="url(#bms-9dc65f7f72910b0a)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="445" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">test-coverage</text><text x="445" y="130" transform="scale(.1)" fill="#fff" textLength="770">test-coverage</text><text aria-hidden="true" x="1035" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="250">92%</text><text x="1035" y="130" transform="scale(.1)" fill="#fff" textLength="250">92%</text></g></svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="122" height="18" role="img" aria-label="test-coverage: 93%"><title>test-coverage: 93%</title><linearGradient id="bms-317a4207573154a7" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-317a4207573154a7"><rect width="122" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-317a4207573154a7)"><rect width="87" height="18" fill="#555"/><rect x="87" width="35" height="18" fill="#97ca00"/><rect width="122" height="18" fill="url(#bms-317a4207573154a7)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="445" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">test-coverage</text><text x="445" y="130" transform="scale(.1)" fill="#fff" textLength="770">test-coverage</text><text aria-hidden="true" x="1035" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="250">93%</text><text x="1035" y="130" transform="scale(.1)" fill="#fff" textLength="250">93%</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -1 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.21"><title>version: 0.1.21</title><linearGradient id="bms-3d708dc9f44ae746" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-3d708dc9f44ae746"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-3d708dc9f44ae746)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-3d708dc9f44ae746)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.21</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.21</text></g></svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.22"><title>version: 0.1.22</title><linearGradient id="bms-49fd3ccd9d32982d" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-49fd3ccd9d32982d"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-49fd3ccd9d32982d)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-49fd3ccd9d32982d)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.22</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.22</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -1,6 +1,7 @@
# Embedded replacement and input defaults. User configuration layers may
# override these values, but installed binaries never require this source file.
# Embedded input defaults. Unicode replacements live in the generated
# `unicode.toml` policy asset so detection, diagnostics, and fixes stay aligned.
schema_version = 1
replacements = []
[input]
max_document_bytes = 8388608
@ -9,83 +10,3 @@ max_expanded_bytes = 268435456
max_archive_entries = 10000
max_compression_ratio = 100
max_walk_depth = 64
# A replacement entry has a stable lowercase kebab-case ID, non-empty Unicode
# source text, an ASCII target, a rationale, and an enabled switch. When enabled
# sources share a prefix, the longest source wins. Example:
# [[replacements]]
# id = "example-symbol"
# source = "→"
# replacement = "->"
# rationale = "Prefer an ASCII arrow in plain-text output."
# enabled = true
[[replacements]]
id = "hyphen"
source = "‐"
replacement = "-"
rationale = "Replace the Unicode hyphen with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "non-breaking-hyphen"
source = "‑"
replacement = "-"
rationale = "Replace the non-breaking hyphen with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "figure-dash"
source = "‒"
replacement = "-"
rationale = "Replace the figure dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "en-dash"
source = "–"
replacement = "-"
rationale = "Replace the en dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "em-dash"
source = "—"
replacement = "-"
rationale = "Replace the em dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "horizontal-bar"
source = "―"
replacement = "-"
rationale = "Replace the horizontal bar with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "minus-sign"
source = "−"
replacement = "-"
rationale = "Replace the mathematical minus sign with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "small-em-dash"
source = "﹘"
replacement = "-"
rationale = "Replace the small em dash with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "small-hyphen-minus"
source = "﹣"
replacement = "-"
rationale = "Replace the small hyphen-minus with ASCII hyphen-minus."
enabled = true
[[replacements]]
id = "fullwidth-hyphen-minus"
source = "-"
replacement = "-"
rationale = "Replace the fullwidth hyphen-minus with ASCII hyphen-minus."
enabled = true

View File

@ -166,51 +166,3 @@ enabled = true
[rules.scope]
content = ["source-code"]
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
[[rules]]
id = "typography-non-ascii-hyphen"
category = "typography"
severity = "advice"
matcher = "replacement"
pattern = "configured-replacements"
weight = 10.0
message = "a configured non-ASCII character has an ASCII replacement"
help = "Review the punctuation or apply the configured safe replacement."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-ascii-confusable"
category = "typography"
severity = "advice"
matcher = "confusable"
pattern = "uts39-ascii-skeleton-v16"
weight = 4.0
message = "a Unicode character visually resembles ASCII under UTS #39"
help = "Review whether the lookalike character is intentional; it is not automatically rewritten."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-invisible-control"
category = "typography"
severity = "warning"
matcher = "invisible"
pattern = "gitea-invisible-9c715c0"
weight = 12.0
message = "an invisible or bidirectional Unicode control appears in the source"
help = "Review the hidden or direction-changing character; it is not removed automatically."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,156 @@
//! Regenerate `assets/unicode.toml` from the pinned Unicode Security tables.
use serde::Serialize;
const CURATED: &[(&str, &str, &str, &str)] = &[
("hyphen", "‐", "-", "Unicode hyphen"),
("non-breaking-hyphen", "‑", "-", "non-breaking hyphen"),
("figure-dash", "‒", "-", "figure dash"),
("en-dash", "–", "-", "en dash"),
("em-dash", "—", "-", "em dash"),
("horizontal-bar", "―", "-", "horizontal bar"),
("minus-sign", "−", "-", "mathematical minus sign"),
("small-em-dash", "﹘", "-", "small em dash"),
("small-hyphen-minus", "﹣", "-", "small hyphen-minus"),
(
"fullwidth-hyphen-minus",
"-",
"-",
"fullwidth hyphen-minus",
),
];
#[derive(Serialize)]
struct Catalog {
schema_version: u32,
unicode: UnicodePolicy,
replacements: Vec<Replacement>,
rules: Vec<Rule>,
}
#[derive(Serialize)]
struct UnicodePolicy {
confusables: &'static str,
}
#[derive(Serialize)]
struct Replacement {
id: String,
source: String,
replacement: String,
rationale: String,
enabled: bool,
}
#[derive(Serialize)]
struct Rule {
id: &'static str,
category: &'static str,
severity: &'static str,
matcher: &'static str,
pattern: &'static str,
weight: f64,
message: &'static str,
help: &'static str,
occurrence_cap: usize,
enabled: bool,
scope: Scope,
}
#[derive(Serialize)]
struct Scope {
content: [&'static str; 2],
extensions: [String; 0],
}
fn main() {
let mut replacements = CURATED
.iter()
.map(|(id, source, replacement, name)| Replacement {
id: (*id).to_owned(),
source: (*source).to_owned(),
replacement: (*replacement).to_owned(),
rationale: format!("Replace the {name} with reviewed ASCII `{replacement}`."),
enabled: true,
})
.collect::<Vec<_>>();
for value in 0..=char::MAX as u32 {
let Some(character) = char::from_u32(value) else {
continue;
};
if character.is_ascii()
|| CURATED
.iter()
.any(|(_, source, _, _)| source.starts_with(character))
{
continue;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if skeleton.is_empty()
|| !skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
|| skeleton == source
{
continue;
}
replacements.push(Replacement {
id: format!("uts39-u{value:04x}"),
source,
replacement: skeleton.clone(),
rationale: format!(
"Replace Unicode 16.0 UTS #39 lookalike U+{value:04X} with its printable ASCII skeleton {skeleton:?}."
),
enabled: true,
});
}
let catalog = Catalog {
schema_version: 1,
unicode: UnicodePolicy {
confusables: "replace",
},
replacements,
rules: vec![
Rule {
id: "unicode-ascii-replacement",
category: "typography",
severity: "advice",
matcher: "replacement",
pattern: "configured-replacements",
weight: 10.0,
message: "a Unicode character or sequence has a configured ASCII replacement",
help: "Review the character or apply the configured ASCII replacement.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
Rule {
id: "unicode-invisible-control",
category: "typography",
severity: "warning",
matcher: "invisible",
pattern: "gitea-invisible-9c715c0",
weight: 12.0,
message: "an invisible or bidirectional Unicode control appears in the source",
help: "Review the hidden or direction-changing character; it is not removed automatically.",
occurrence_cap: 5,
enabled: true,
scope: Scope {
content: ["prose", "source-code"],
extensions: [],
},
},
],
};
println!(
"# Generated by `cargo run -p john-slop-core --example generate_unicode_catalog`.\n\
# Unicode Security 0.1.2 / Unicode 16.0.0 UTS #39, plus reviewed punctuation.\n\
# Do not edit generated mappings by hand; edit this generator and regenerate.\n{}",
toml::to_string_pretty(&catalog).expect("the catalog model must serialize")
);
}

View File

@ -15,6 +15,7 @@ use crate::{Category, FindingSeverity};
const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml");
const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml");
const EMBEDDED_UNICODE: &str = include_str!("../assets/unicode.toml");
/// A validated named configuration without path syntax or a file extension.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@ -368,12 +369,6 @@ pub enum ConfigValidationError {
/// Heuristic rule ID.
id: String,
},
/// Confusable matchers must select the pinned UTS #39 profile.
#[error("confusable rule `{id}` must use pattern `uts39-ascii-skeleton-v16`")]
InvalidConfusableMatcher {
/// Heuristic rule ID.
id: String,
},
/// Invisible matchers must select the pinned Gitea profile.
#[error("invisible rule `{id}` must use pattern `gitea-invisible-9c715c0`")]
InvalidInvisibleMatcher {
@ -502,10 +497,17 @@ struct ConfigLayer {
schema_version: Option<u32>,
input: InputLayer,
thresholds: ThresholdLayer,
unicode: UnicodeLayer,
replacements: Vec<ReplacementPatch>,
rules: Vec<RulePatch>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
struct UnicodeLayer {
confusables: Option<ConfusablePolicy>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
struct InputLayer {
@ -561,6 +563,9 @@ fn apply_layer(
}
apply_input_layer(&mut config.input, layer.input);
apply_threshold_layer(&mut config.thresholds, layer.thresholds);
if let Some(value) = layer.unicode.confusables {
config.unicode.confusables = value;
}
merge_replacements(&mut config.replacements, layer.replacements, path)?;
merge_rules(&mut config.rules, layer.rules, path)
}
@ -779,6 +784,9 @@ pub struct Config {
pub schema_version: u32,
/// Enabled replacement rules in deterministic matching order.
pub replacements: Vec<ReplacementRule>,
/// Bulk behavior for generated Unicode mappings.
#[serde(default)]
pub unicode: UnicodePolicy,
/// Configured detection rules in deterministic declaration order.
pub rules: Vec<HeuristicRule>,
/// Named evidence-band boundaries.
@ -787,6 +795,34 @@ pub struct Config {
pub input: InputLimits,
}
/// Bulk policy for Unicode mappings whose targets come from UTS #39.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct UnicodePolicy {
/// Whether generated lookalikes are replaced, reviewed, or ignored.
pub confusables: ConfusablePolicy,
}
impl Default for UnicodePolicy {
fn default() -> Self {
Self {
confusables: ConfusablePolicy::Replace,
}
}
}
/// Action applied to generated UTS #39 printable-ASCII mappings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ConfusablePolicy {
/// Emit diagnostics and formatter/LSP edits.
Replace,
/// Emit diagnostics without edits.
Review,
/// Emit neither diagnostics nor edits for generated mappings.
Off,
}
/// One configured data-driven detection rule.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@ -827,8 +863,6 @@ pub enum MatcherType {
RepeatedRegex,
/// Findings produced by the configured replacement catalog.
Replacement,
/// Non-mutating findings for UTS #39 characters with an ASCII skeleton.
Confusable,
/// Non-mutating findings for pinned invisible and bidi control characters.
Invisible,
}
@ -916,11 +950,20 @@ impl Default for InputLimits {
}
impl Config {
/// Load the replacement defaults embedded in this build.
/// Load all defaults embedded in this build.
pub fn embedded() -> Self {
let config: Self = embedded_figment()
let mut config: Self = embedded_figment()
.extract()
.expect("embedded configuration must deserialize");
let unicode_layer = Figment::from(Toml::string(EMBEDDED_UNICODE))
.extract::<ConfigLayer>()
.expect("embedded Unicode policy must deserialize");
apply_layer(
&mut config,
unicode_layer,
std::path::Path::new("<embedded unicode.toml>"),
)
.expect("embedded Unicode policy must merge");
config
.validate()
.expect("embedded configuration must be semantically valid");
@ -1043,12 +1086,6 @@ fn validate_rules(rules: &[HeuristicRule]) -> Result<(), ConfigValidationError>
});
}
MatcherType::Replacement => {}
MatcherType::Confusable if rule.pattern != "uts39-ascii-skeleton-v16" => {
return Err(ConfigValidationError::InvalidConfusableMatcher {
id: rule.id.clone(),
});
}
MatcherType::Confusable => {}
MatcherType::Invisible if rule.pattern != "gitea-invisible-9c715c0" => {
return Err(ConfigValidationError::InvalidInvisibleMatcher {
id: rule.id.clone(),
@ -1151,7 +1188,7 @@ mod tests {
.map(|rule| rule.source.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(config.replacements.len(), 10);
assert_eq!(config.replacements.len(), 1_829);
assert_eq!(config.schema_version, 1);
assert_eq!(ids.len(), config.replacements.len());
assert_eq!(sources.len(), config.replacements.len());
@ -1161,6 +1198,38 @@ mod tests {
&& !rule.rationale.is_empty()));
}
#[test]
fn embedded_unicode_catalog_covers_every_printable_ascii_skeleton() {
assert_eq!(unicode_security::UNICODE_VERSION, (16, 0, 0));
let config = Config::embedded();
let replacements = config
.replacements
.iter()
.map(|rule| (rule.source.as_str(), rule.replacement.as_str()))
.collect::<std::collections::BTreeMap<_, _>>();
for value in 0..=char::MAX as u32 {
let Some(character) = char::from_u32(value) else {
continue;
};
if character.is_ascii() {
continue;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
if !skeleton.is_empty()
&& skeleton.bytes().all(|byte| (b' '..=b'~').contains(&byte))
&& skeleton != source
{
assert_eq!(
replacements.get(source.as_str()),
Some(&skeleton.as_str()),
"catalog drift for U+{value:04X}"
);
}
}
}
#[test]
fn default_input_limits_match_the_accepted_policy() {
let limits = InputLimits::default();
@ -1177,7 +1246,7 @@ mod tests {
fn embedded_heuristics_have_complete_metadata() {
let config = Config::embedded();
assert_eq!(config.rules.len(), 13);
assert_eq!(config.rules.len(), 12);
assert_eq!(config.thresholds.low, 20);
assert_eq!(config.thresholds.medium, 50);
assert_eq!(config.thresholds.high, 75);
@ -1294,6 +1363,10 @@ mod tests {
id = "em-dash"
enabled = false
[[replacements]]
id = "uts39-u0430"
enabled = false
[[replacements]]
id = "right-arrow"
source = "→"
@ -1311,7 +1384,7 @@ weight = 9.5
let config = ConfigLoader::from_paths(paths).load().unwrap();
assert_eq!(config.replacements.len(), 11);
assert_eq!(config.replacements.len(), 1_830);
let replacement = config
.replacements
.iter()
@ -1320,8 +1393,16 @@ weight = 9.5
assert!(!replacement.enabled);
assert_eq!(replacement.source, "—");
assert_eq!(replacement.replacement, "-");
assert!(
!config
.replacements
.iter()
.find(|rule| rule.id == "uts39-u0430")
.unwrap()
.enabled
);
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
assert_eq!(config.rules.len(), 13);
assert_eq!(config.rules.len(), 12);
let heuristic = config
.rules
.iter()
@ -1564,6 +1645,9 @@ low = 10
medium = 40
high = 90
[unicode]
confusables = "review"
[[replacements]]
id = "em-dash"
source = "⸺"
@ -1609,6 +1693,7 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
high: 90
}
);
assert_eq!(config.unicode.confusables, ConfusablePolicy::Review);
let replacement = config
.replacements
.iter()
@ -1760,18 +1845,6 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
)
},
);
assert_invalid(
|config| {
config.rules[0].matcher = MatcherType::Confusable;
config.rules[0].pattern = "wrong".into();
},
|error| {
matches!(
error,
ConfigValidationError::InvalidConfusableMatcher { .. }
)
},
);
assert_invalid(
|config| {
config.rules[0].matcher = MatcherType::Invisible;

View File

@ -44,7 +44,6 @@ impl Rule {
CompiledMatcher::Regex(regex) => regex.as_str(),
CompiledMatcher::RepeatedRegex(regex) => regex.as_str(),
CompiledMatcher::Replacement => "configured-replacements",
CompiledMatcher::Confusable => "uts39-ascii-skeleton-v16",
CompiledMatcher::Invisible => "gitea-invisible-9c715c0",
}
}
@ -117,7 +116,6 @@ enum CompiledMatcher {
Regex(Regex),
RepeatedRegex(Regex),
Replacement,
Confusable,
Invisible,
}
@ -269,7 +267,6 @@ impl Rule {
CompiledMatcher::RepeatedRegex(regex)
}
MatcherType::Replacement => CompiledMatcher::Replacement,
MatcherType::Confusable => CompiledMatcher::Confusable,
MatcherType::Invisible => CompiledMatcher::Invisible,
};
let scope = CompiledScope::compile(&definition)?;
@ -343,34 +340,18 @@ impl Rule {
.collect()
}
}
CompiledMatcher::Replacement => replacement::plan(config, source)
.into_iter()
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
.collect(),
CompiledMatcher::Confusable => source
.char_indices()
.filter_map(|(start, character)| {
let matched = character.to_string();
if config
.replacements
.iter()
.any(|rule| rule.enabled && rule.source == matched)
{
return None;
}
let skeleton = unicode_review::ascii_skeleton(character)?;
let mut finding =
self.finding(source, start..start + character.len_utf8(), None);
finding.help = Some(format!(
"Review U+{:04X}; its Unicode {}.{}.{} UTS #39 skeleton is {skeleton:?}. This review finding is not an automatic replacement.",
character as u32,
unicode_review::UTS39_UNICODE_VERSION.0,
unicode_review::UTS39_UNICODE_VERSION.1,
unicode_review::UTS39_UNICODE_VERSION.2,
));
Some(finding)
})
.collect(),
CompiledMatcher::Replacement => {
let mut findings = replacement::plan(config, source)
.into_iter()
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
.collect::<Vec<_>>();
findings.extend(
replacement::plan_confusables_for_review(config, source)
.into_iter()
.map(|edit| self.finding(source, edit.byte_span, None)),
);
findings
}
CompiledMatcher::Invisible => source
.char_indices()
.filter(|(start, character)| {
@ -566,7 +547,7 @@ mod tests {
assert_eq!(analysis.findings.len(), 1);
assert_eq!(
analysis.findings[0].rule_id.as_str(),
"typography-non-ascii-hyphen"
"unicode-ascii-replacement"
);
assert_eq!(analysis.findings[0].matched_text, "—");
assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-");
@ -575,17 +556,16 @@ mod tests {
}
#[test]
fn unicode_review_findings_have_exact_spans_and_never_become_fixes() {
fn unicode_review_findings_have_exact_spans_and_safe_fixes() {
let detector = Detector::new(&Config::embedded()).unwrap();
let source = "é pаss\u{200b}word\u{202e}\r\nnext";
let findings = detector.analyze(source).findings;
assert_eq!(findings.len(), 3);
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-confusable");
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
assert_eq!(findings[0].matched_text, "а");
assert_eq!(&source[findings[0].byte_span.clone()], "а");
assert!(findings[0].help.as_deref().unwrap().contains("\"a\""));
assert!(findings.iter().all(|finding| finding.fix.is_none()));
assert_eq!(findings[0].fix.as_ref().unwrap().replacement, "a");
assert_eq!(
findings
.iter()
@ -593,9 +573,29 @@ mod tests {
.count(),
2
);
assert!(findings[1..].iter().all(|finding| finding.fix.is_none()));
assert!(findings.iter().all(|finding| finding.matched_text != "\r"));
}
#[test]
fn confusable_policy_switches_between_fix_review_and_off() {
let mut config = Config::embedded();
config.unicode.confusables = crate::ConfusablePolicy::Review;
let review = Detector::new(&config).unwrap().analyze("pаss");
assert_eq!(review.findings.len(), 1);
assert!(review.findings[0].fix.is_none());
config.unicode.confusables = crate::ConfusablePolicy::Off;
assert!(
Detector::new(&config)
.unwrap()
.analyze("pаss")
.findings
.is_empty()
);
}
#[test]
fn safe_replacement_characters_are_not_duplicated_as_confusable_findings() {
let findings = Detector::new(&Config::embedded())
@ -604,7 +604,7 @@ mod tests {
.findings;
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].rule_id.as_str(), "typography-non-ascii-hyphen");
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-replacement");
}
#[test]
@ -675,7 +675,7 @@ mod tests {
"repeated-transition",
"source-placeholder-todo",
"source-tutorial-narration",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
] {
assert!(ids.contains(expected), "missing {expected}");
}

View File

@ -16,9 +16,9 @@ mod unicode_review;
pub use config::{
Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError,
ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, EvidenceThresholds,
HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule, RuleScope,
ThresholdOverrides,
ConfigOverrides, ConfigPaths, ConfigValidationError, ConfusablePolicy, ContentKind,
EvidenceThresholds, HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule,
RuleScope, ThresholdOverrides, UnicodePolicy,
};
pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule};
pub use diagnostic::FindingDiagnostic;

View File

@ -1,10 +1,10 @@
//! Pure planning and application of configured text replacements.
use std::{borrow::Cow, ops::Range};
use std::{borrow::Cow, collections::BTreeMap, ops::Range};
use miette::Diagnostic;
use crate::{Config, TextEdit};
use crate::{Config, ConfusablePolicy, ReplacementRule, TextEdit};
/// An edit set cannot be applied safely to its source snapshot.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error, Diagnostic)]
@ -36,12 +36,39 @@ pub enum ReplacementError {
/// Plan deterministic non-overlapping edits without modifying the source.
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
let mut rules = config
.replacements
.iter()
.filter(|rule| rule.enabled)
.collect::<Vec<_>>();
rules.sort_by(|left, right| {
plan_matching(config, source, |rule| {
rule.enabled
&& (!rule.id.starts_with("uts39-")
|| config.unicode.confusables == ConfusablePolicy::Replace)
})
}
pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec<TextEdit> {
if config.unicode.confusables != ConfusablePolicy::Review {
return Vec::new();
}
plan_matching(config, source, |rule| {
rule.enabled && rule.id.starts_with("uts39-")
})
}
fn plan_matching(
config: &Config,
source: &str,
include: impl Fn(&ReplacementRule) -> bool,
) -> Vec<TextEdit> {
let mut single_character_rules = BTreeMap::new();
let mut sequence_rules = Vec::new();
for rule in config.replacements.iter().filter(|rule| include(rule)) {
let mut characters = rule.source.chars();
let first = characters.next();
if let (Some(character), None) = (first, characters.next()) {
single_character_rules.insert(character, rule);
} else {
sequence_rules.push(rule);
}
}
sequence_rules.sort_by(|left, right| {
right
.source
.len()
@ -52,7 +79,7 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
let mut edits = Vec::new();
let mut offset = 0;
while offset < source.len() {
if let Some(rule) = rules
if let Some(rule) = sequence_rules
.iter()
.find(|rule| source[offset..].starts_with(&rule.source))
{
@ -63,7 +90,17 @@ pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
});
offset = end;
} else {
offset += source[offset..].chars().next().map_or(0, char::len_utf8);
let Some(character) = source[offset..].chars().next() else {
break;
};
let end = offset + character.len_utf8();
if let Some(rule) = single_character_rules.get(&character) {
edits.push(TextEdit {
byte_span: offset..end,
replacement: rule.replacement.clone(),
});
}
offset = end;
}
}
edits
@ -149,6 +186,28 @@ mod tests {
);
}
#[test]
fn generated_catalog_replaces_letters_digits_punctuation_and_multi_byte_skeletons() {
let source = "а𝐀!𝟘ⅿ🯹";
let edits = plan(&Config::embedded(), source);
assert_eq!(edits.len(), 6);
assert_eq!(apply(source, &edits).unwrap(), "aA!Orn9");
assert_eq!(edits[4].replacement, "rn");
}
#[test]
fn bulk_confusable_policy_preserves_curated_replacements() {
let mut config = Config::embedded();
config.unicode.confusables = ConfusablePolicy::Review;
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1);
config.unicode.confusables = ConfusablePolicy::Off;
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
assert!(plan_confusables_for_review(&config, "а—").is_empty());
}
#[test]
fn empty_or_ascii_sources_need_no_edits() {
let config = Config::embedded();

View File

@ -1,17 +1,4 @@
//! Pinned Unicode review profiles that never authorize automatic rewriting.
/// Unicode version carried by `unicode-security` 0.1.2.
pub(crate) const UTS39_UNICODE_VERSION: (u64, u64, u64) = unicode_security::UNICODE_VERSION;
/// Return an ASCII UTS #39 skeleton when one non-ASCII scalar resembles ASCII.
pub(crate) fn ascii_skeleton(character: char) -> Option<String> {
if character.is_ascii() {
return None;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
(!skeleton.is_empty() && skeleton.is_ascii() && skeleton != source).then_some(skeleton)
}
//! Pinned invisible and bidirectional-control review profile.
/// Match the invisible/control profile generated by Gitea at commit 9c715c0.
///
@ -55,15 +42,6 @@ pub(crate) const fn is_gitea_invisible(character: char) -> bool {
mod tests {
use super::*;
#[test]
fn uts39_profile_reports_ascii_lookalikes_but_not_ascii_or_accents() {
assert_eq!(ascii_skeleton('а').as_deref(), Some("a"));
assert_eq!(ascii_skeleton('𝟘').as_deref(), Some("O"));
assert_eq!(ascii_skeleton('a'), None);
assert_eq!(ascii_skeleton('é'), None);
assert_eq!(UTS39_UNICODE_VERSION, (16, 0, 0));
}
#[test]
fn gitea_profile_covers_invisible_and_bidi_ranges_without_normal_whitespace() {
assert!(is_gitea_invisible('\u{200b}'));

View File

@ -790,7 +790,7 @@ mod tests {
assert_eq!(
diagnostics[1].code,
Some(NumberOrString::String(
"typography-non-ascii-hyphen".to_owned()
"unicode-ascii-replacement".to_owned()
))
);
assert_eq!(diagnostics[1].data.as_ref().unwrap()["fixable"], true);

View File

@ -35,7 +35,8 @@ workspace Tokio requirement enables no features: the LSP library selects
The modules within those crates retain deliberately narrow responsibilities:
- `config` owns the validated embedded replacement catalog and input limits;
- `config` owns validated embedded defaults, the generated Unicode policy, and
input limits;
- `input` streams files, directory entries, and archive members as documents;
- `detector` compiles configured matchers and path scopes once, then produces
explainable findings and a bounded low-care evidence score;
@ -44,7 +45,8 @@ The modules within those crates retain deliberately narrow responsibilities:
- `replacement` produces non-overlapping edits and validates/applies arbitrary
edit sets without performing I/O;
- `unicode_review` pins the UTS #39 ASCII-skeleton version and Gitea-compatible
invisible/control ranges used only by non-mutating detector matchers;
invisible/control ranges; generated printable-ASCII mappings feed the shared
replacement planner while controls remain non-mutating;
- `lsp` indexes open snapshots, runs the compiled detector, converts canonical
finding and replacement spans to UTF-16 coordinates, and implements
versioned quick fixes plus whole-document formatting;

View File

@ -1,6 +1,6 @@
# Configuration
John Slop embeds schema-versioned replacement and heuristic catalogs in every
John Slop embeds schema-versioned base, heuristic, and Unicode catalogs in every
binary. Installed binaries therefore work without a source checkout or external
default files.
@ -29,9 +29,10 @@ files are optional; a selected explicit file must exist.
## Schema
The current `schema_version` is `1`. The complete defaults and documented record
examples live in [`config.toml`](../crates/john-slop-core/assets/config.toml) and
[`heuristics.toml`](../crates/john-slop-core/assets/heuristics.toml).
The current `schema_version` is `1`. The complete defaults live in
[`config.toml`](../crates/john-slop-core/assets/config.toml),
[`heuristics.toml`](../crates/john-slop-core/assets/heuristics.toml), and the
generated [`unicode.toml`](../crates/john-slop-core/assets/unicode.toml).
Ready-to-copy, commented partial layers live in
[`examples/config`](../examples/config/): `config.toml` demonstrates a user-wide
base and `editorial.toml` demonstrates a named overlay.
@ -60,12 +61,24 @@ replacement targets, and invalid thresholds are fatal configuration
diagnostics.
The built-in matcher names are `exact-phrase`, `regex`, `repeated-regex`,
`replacement`, `confusable`, and `invisible`. The last three use validated
sentinel patterns because their implementations are data-backed rather than
free-form strings: `configured-replacements`, `uts39-ascii-skeleton-v16`, and
`gitea-invisible-9c715c0`. They remain ordinary rule records, so later layers
`replacement`, and `invisible`. The last two use validated sentinel patterns
because their implementations are data-backed rather than free-form strings:
`configured-replacements` and `gitea-invisible-9c715c0`. They remain ordinary
rule records, so later layers
can disable them or adjust severity, weight, occurrence cap, and scope without
changing code. Confusable and invisible findings never authorize edits.
changing code. Invisible findings never authorize edits; generated confusable
mappings follow the bulk policy below.
The generated UTS #39 mappings have one bulk policy:
```toml
[unicode]
confusables = "review" # replace (default), review, or off
```
`replace` enables diagnostics and edits, `review` retains diagnostics without
formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated
punctuation replacements and invisible/control diagnostics are unaffected.
Set `enabled = false` on a known rule or replacement ID to disable it. Because
records merge by ID, do not copy the full embedded table merely to change one
@ -77,6 +90,19 @@ must be unique. When enabled sources share a prefix, the longest matching source
wins at each byte offset, with stable rule ID ordering as the deterministic
tie-break. Edits always refer to the original UTF-8 snapshot.
Generated mapping IDs use `uts39-u<lowercase-codepoint>`, so an individual
entry can still be disabled or overridden through normal merge-by-ID behavior.
Maintain the embedded list by editing
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:
```sh
cargo run -p john-slop-core --example generate_unicode_catalog \
> crates/john-slop-core/assets/unicode.toml
```
The exhaustive catalog test walks every Unicode scalar and rejects drift from
the pinned Unicode 16.0 tables.
Each heuristic also configures `severity` as `advice`, `warning`, or `error`.
Severity controls CLI presentation, LSP severity, and summary counts; the
evidence score still comes from the independently configured weight and

View File

@ -67,8 +67,8 @@ Run line coverage with Cargo Tarpaulin:
cargo tarpaulin --config tarpaulin.toml
```
The configuration enforces a 90% floor. The current suite measured 92.40%
(1,472/1,593 lines) on 2026-09-30. The remaining lines are dominated by
The configuration enforces a 90% floor. The current suite measured 93.05%
(1,487/1,598 lines) on 2026-09-30. The remaining lines are dominated by
platform/I/O failure injection, malformed archive internals, coordinate
overflow, LSP logging/reload failures, and subprocess attribution. Do not add
brittle tests or production seams solely to make Tarpaulin count those paths.

View File

@ -19,8 +19,7 @@ With the embedded schema-version 1 catalog, the corpus currently reports:
| repeated-transition | 1 | 0 | 0 | 100% | 100% |
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
| typography-non-ascii-hyphen | 1 | 0 | 0 | 100% | 100% |
| unicode-ascii-confusable | 1 | 0 | 0 | 100% | 100% |
| unicode-ascii-replacement | 2 | 0 | 0 | 100% | 100% |
| unicode-invisible-control | 1 | 0 | 0 | 100% | 100% |
Aggregate labeled-rule counts are 13 true positives, 0 false positives, and 0
@ -36,7 +35,7 @@ The same manifest pins evidence scores for regression characterization:
| style signals | 19 |
| source-code signals | 24 |
| typography signal | 17 |
| one ASCII lookalike and one invisible character | 21 |
| one ASCII lookalike and one invisible character | 28 |
| each of five negative documents | 0 |
These perfect values describe a tiny purpose-built regression set, not expected
@ -50,11 +49,11 @@ Each shipped rule has at least one explicitly labeled realistic near miss. The
initial 0.1.0 review kept all schema-version 1 rules, weights, and thresholds
unchanged. The prompt-leakage review accepted lighter `0.90ⁿ` correlation
damping for distinct prompt-leakage rules while retaining `0.75ⁿ` for other
categories. The Milestone 21 review adds the non-mutating UTS #39 ASCII-skeleton
and Gitea-compatible invisible/control profiles, with explicit multilingual
limitations. The checked-in `signoff.toml` records those decisions and contains
the SHA-256 digest of
`crates/john-slop-core/assets/heuristics.toml`.
categories. The Milestone 22 review makes printable-ASCII UTS #39 mappings
replaceable by default, retains a non-mutating `review` policy, and keeps the
Gitea-compatible invisible/control profile diagnostic-only, with explicit
multilingual limitations. The checked-in `signoff.toml` records those decisions
and contains the SHA-256 digest of both `heuristics.toml` and `unicode.toml`.
Any default rule, matcher, pattern, weight, occurrence cap, scope, or threshold
change invalidates the sign-off test. Renewal requires reviewing positive and

View File

@ -5,18 +5,19 @@ Every contribution to the score is represented by a finding with an exact
source span, stable rule ID, category, message, help, and configured weight.
The detector compiles the resolved rule catalog once. Exact phrases, regular
expressions, repeated-pattern rules, configured Unicode replacements, UTS #39
ASCII lookalikes, and invisible/control characters share the same finding
expressions, repeated-pattern rules, configured Unicode replacements, and
invisible/control characters share the same finding
model. Content kind, extension, path globs, and conventional generated/vendor/
minified paths decide whether each rule applies. Findings are deduplicated by
rule and byte span, then sorted by start offset, end offset, and rule ID so
every adapter observes the same order.
Unicode review rules are deliberately distinct from replacement rules. The
detector can flag a lookalike, invisible scalar, or direction-changing control
at its exact span, but only the explicit ten-entry safe replacement allowlist
can produce a formatter edit or LSP quick fix. A safe replacement finding is
not duplicated as a confusable finding.
The generated Unicode 16.0 profile maps every non-ASCII scalar with a printable
ASCII UTS #39 skeleton, alongside ten manually reviewed punctuation mappings.
The default `replace` policy gives each of these findings a formatter/LSP edit;
`review` keeps the same exact diagnostics without edits, and `off` suppresses
the generated lookalikes. Invisible and direction-changing controls remain
diagnostic-only because deleting them silently is not a generally safe edit.
## Evidence-score formula

View File

@ -42,8 +42,9 @@ documentation, notebooks, and uncommon programming languages are
underrepresented. File-extension inference covers a conservative common set and
defaults unknown paths to prose. Teams should validate rules against their own
reviewed material before changing enforcement thresholds. Disable or narrow
`unicode-ascii-confusable` when ordinary multilingual text makes character-level
review too noisy; locale-aware suppression is not implemented.
set `[unicode] confusables = "review"` to keep findings without edits, or use
`"off"` when ordinary multilingual text makes character-level review too noisy.
Locale-aware suppression is not implemented.
## Evidence score is not probability
@ -81,8 +82,10 @@ synchronization. It publishes the configured heuristic catalog but does not
publish a document-level evidence score; editor diagnostics remain individual
review signals. It does not read or write a document URI, mutate archives,
provide prose rewrites, or implement incremental range changes. Only configured
ASCII replacements receive quick fixes or formatting edits. Confusable,
invisible, and bidirectional-control findings are diagnostic-only, and
ASCII replacements receive quick fixes or formatting edits. Under the default
policy this includes printable-ASCII UTS #39 skeletons, which can alter valid
Greek, Cyrillic, mathematical, or other text; use `review` for non-mutating
diagnostics. Invisible and bidirectional-control findings are diagnostic-only, and
configuration reload errors retain the last valid configuration and compiled
detector.

View File

@ -52,11 +52,11 @@ modes.
},
"findings": [
{
"rule_id": "typography-non-ascii-hyphen",
"rule_id": "unicode-ascii-replacement",
"category": "typography",
"severity": "advice",
"message": "a configured non-ASCII character has an ASCII replacement",
"help": "Review the punctuation or apply the configured safe replacement.",
"message": "a Unicode character or sequence has a configured ASCII replacement",
"help": "Review the character or apply the configured ASCII replacement.",
"byte_span": { "start": 8, "end": 11 },
"matched_text": "—",
"weight": 10.0,

View File

@ -29,31 +29,41 @@ locale-aware lookalike table.
## Implemented John Slop profile
These sources are useful detection references, not safe formatter replacement
catalogs. Many listed characters are legitimate in particular languages and
scripts, and Gitea deliberately interprets them through the reviewer's locale.
Importing every pair as an automatic Unicode-to-ASCII edit would corrupt valid
text and violate the formatter's conservative replacement contract.
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
scalar whose single-character skeleton is non-empty printable ASCII, plus ten
reviewed punctuation mappings. That is 1,829 mappings in the current profile.
Each entry has a stable ID, source, target, rationale, and enabled switch, and
the same asset owns the diagnostic metadata.
John Slop keeps three policies separate:
The default `[unicode] confusables = "replace"` policy emits
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
`review` for identical findings without edits, or `off` to suppress generated
lookalikes. The ten curated punctuation mappings remain active under all three
settings and can still be disabled individually.
1. configured, explicitly safe formatter replacements;
2. non-mutating confusable findings for human review; and
3. non-mutating invisible and bidirectional-control findings.
`unicode-ascii-confusable` uses `unicode-security` 0.1.2, whose checked-in UTS
#39 tables target Unicode 16.0. It reports a non-ASCII scalar only when its
single-character skeleton is entirely ASCII. It is intentionally broader than
Gitea's locale selection and can flag legitimate scripts; locale-aware policy
remains deferred. Characters already covered by the safe replacement allowlist
produce only the fixable replacement finding, not a duplicate confusable one.
This broad mapping is intentionally more aggressive than Gitea's locale-aware
display warning. Greek, Cyrillic, mathematical, and other characters may be
legitimate text, so projects handling multilingual content should prefer
`review` unless ASCII normalization is their explicit policy.
`unicode-invisible-control` uses the invisible ranges generated by Gitea at the
pinned commit above, including bidi embedding/override and isolate controls.
Ordinary space, tab, line feed, and the carriage return in a CRLF pair are not
reported. Each other match receives an exact UTF-8 source span that adapters
convert normally. Neither rule feeds the formatter or LSP quick fixes.
convert normally. Controls never feed the formatter or LSP quick fixes because
silent deletion has no generally safe meaning.
Regenerate the checked-in policy deterministically with:
```sh
cargo run -p john-slop-core --example generate_unicode_catalog \
> crates/john-slop-core/assets/unicode.toml
```
An exhaustive test walks all Unicode scalar values and rejects mapping drift.
Future profile updates must pin their upstream Unicode/data version, retain
provenance and licensing, test ordinary multilingual near misses, renew the
hash-bound default-policy sign-off, and preserve the no-silent-rewrite boundary.
provenance and licensing, test ordinary multilingual near misses, and renew the
hash-bound default-policy sign-off.

View File

@ -73,12 +73,12 @@ embedded catalog remains limited to individually reviewed conservative rules.
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
shipped prose signal, while
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
source-placeholder and tutorial-narration rules plus all ten embedded
replacement characters, a Cyrillic ASCII lookalike, a zero-width space, and a
source-placeholder and tutorial-narration rules plus the ten curated
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
right-to-left override. The adjacent example configuration promotes assistant
self-identification from warning to error and lifts the typography occurrence
cap for this display, making one run show Miette advice, warning, error,
fixable, and deliberately non-fixable Unicode diagnostics together:
fixable, and deliberately non-fixable control diagnostics together:
```sh
cargo run --bin john-slop -- \

View File

@ -1,6 +1,6 @@
//! Showcase opt-in replacements for every printable ASCII target.
use std::{collections::BTreeSet, io::Write};
use std::io::Write;
use john_slop::{
Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements,
@ -10,11 +10,6 @@ const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
fn showcase_config() -> Result<Config, ConfigValidationError> {
let mut config = Config::embedded();
let embedded_sources = config
.replacements
.iter()
.map(|rule| rule.source.clone())
.collect::<BTreeSet<_>>();
let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| {
// U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values.
let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0)
@ -23,19 +18,29 @@ fn showcase_config() -> Result<Config, ConfigValidationError> {
}));
for (source, target) in mappings {
if embedded_sources.contains(&source.to_string()) {
continue;
let source = source.to_string();
if let Some(rule) = config
.replacements
.iter_mut()
.find(|rule| rule.source == source)
{
rule.replacement = target.to_string();
rule.rationale = format!(
"Override U+{:04X} for the complete printable-ASCII showcase.",
u32::from(source.chars().next().expect("source contains one scalar"))
);
} else {
config.replacements.push(ReplacementRule {
id: format!(
"showcase-fullwidth-{:04x}",
u32::from(source.chars().next().expect("source contains one scalar"))
),
source,
replacement: target.to_string(),
rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."),
enabled: true,
});
}
config.replacements.push(ReplacementRule {
id: format!("showcase-fullwidth-{:04x}", u32::from(source)),
source: source.to_string(),
replacement: target.to_string(),
rationale: format!(
"Demonstrate an opt-in mapping from U+{:04X} to printable ASCII.",
u32::from(source)
),
enabled: true,
});
}
config.validate()?;
@ -57,6 +62,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
const EXPECTED: &str = include_str!("ascii-replacement-showcase/ascii.txt");

View File

@ -7,5 +7,5 @@ severity = "error"
# Let the companion source fixture display every embedded replacement character
# instead of stopping at the production catalog's normal per-document cap.
[[rules]]
id = "typography-non-ascii-hyphen"
id = "unicode-ascii-replacement"
occurrence_cap = 10

View File

@ -14,7 +14,7 @@ expected_rules = [
"assistant-answer-preface",
"assistant-capability-disclaimer",
"assistant-self-identification",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
]
expected_replacements = 1
@ -31,14 +31,14 @@ expected_replacements = 0
[[cases]]
path = "typography/example.txt"
formatted = "typography/formatted.txt"
expected_rules = ["typography-non-ascii-hyphen"]
expected_rules = ["unicode-ascii-replacement"]
expected_replacements = 3
[[cases]]
path = "unicode-review/example.txt"
formatted = "unicode-review/formatted.txt"
expected_rules = ["unicode-ascii-confusable", "unicode-invisible-control"]
expected_replacements = 0
expected_rules = ["unicode-ascii-replacement", "unicode-invisible-control"]
expected_replacements = 7
[[cases]]
path = "source/example.rs"
@ -46,6 +46,6 @@ formatted = "source/formatted.rs"
expected_rules = [
"source-placeholder-todo",
"source-tutorial-narration",
"typography-non-ascii-hyphen",
"unicode-ascii-replacement",
]
expected_replacements = 1

View File

@ -1 +1,2 @@
The pаss​word contains lookalike and invisible characters.
Catalog sample: а𝐀!𝟘ⅿ🯹.

View File

@ -1 +1,2 @@
The pаss​word contains lookalike and invisible characters.
The pass​word contains lookalike and invisible characters.
Catalog sample: aA!Orn9.

6
fuzz/Cargo.lock generated
View File

@ -385,7 +385,7 @@ dependencies = [
[[package]]
name = "john-slop-core"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"directories",
"figment",
@ -411,7 +411,7 @@ dependencies = [
[[package]]
name = "john-slop-lsp"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"john-slop-core",
"serde_json",
@ -422,7 +422,7 @@ dependencies = [
[[package]]
name = "john_slop"
version = "0.1.21"
version = "0.1.22"
dependencies = [
"clap",
"globset",

View File

@ -322,7 +322,7 @@ mod tests {
)
.unwrap();
assert!(output.contains("typography-non-ascii-hyphen"));
assert!(output.contains("unicode-ascii-replacement"));
assert!(output.contains("article.md"));
assert!(output.contains('╭'));
assert!(output.contains('╰'));
@ -332,7 +332,7 @@ mod tests {
let lines = output.lines().collect::<Vec<_>>();
assert!(lines[0].starts_with("◆ "));
assert!(lines[0].contains("typography-non-ascii-hyphen"));
assert!(lines[0].contains("unicode-ascii-replacement"));
assert!(!lines[1].is_empty(), "title and message must stay adjacent");
let help_line = lines
.iter()
@ -382,7 +382,7 @@ mod tests {
let output =
render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap();
assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]"));
assert!(output.contains("article.md:8:11: advice[unicode-ascii-replacement]"));
assert!(output.contains("article.md: summary: evidence_score="));
assert!(output.contains("advice=1"));
assert_eq!(

View File

@ -547,6 +547,38 @@ matrices, dependency policy/audit checks, and workspace package assembly pass;
coverage is 92.40% (1,472/1,593 lines), and all packages, binaries, lockfiles,
changelog, and the generated badge agree on `0.1.21`.
## Milestone 22 — Maintainable ambiguous-Unicode replacement catalog
- [x] Define one versioned Unicode policy asset that owns the replacement
diagnostic metadata, reviewed legacy punctuation mappings, generated UTS #39
ASCII-skeleton mappings, and invisible-control diagnostic metadata. Add a
deterministic generator with pinned Unicode provenance and drift checks.
- [x] Make the shared replacement planner efficient for the expanded catalog,
load the catalog through every configuration path, and use its mappings for
CLI fixes, formatter output, LSP quick fixes, and LSP document formatting.
Preserve review-only behavior for invisible and bidirectional controls.
- [x] Expand fixtures and regression tests across letters, digits, punctuation,
mathematical/fullwidth forms, multilingual risk cases, user overrides, and
CLI/LSP/formatter parity. Update guides, evaluation sign-off, coverage,
changelog, package version `0.1.22`, badges, and release metadata.
Acceptance: the checked-in catalog covers every non-ASCII scalar whose pinned
Unicode 16.0 UTS #39 skeleton is non-empty printable ASCII, except explicitly
documented exclusions; each mapping produces one exact diagnostic and the same
replacement through the CLI, standalone formatter, LSP quick fix, and LSP
formatting; the catalog can be regenerated deterministically and drift is
tested; invisible/bidi controls stay non-mutating; formatting remains
idempotent; all quality, audit, package, and 90% coverage gates pass before the
immutable `v0.1.22` tag is created.
Accepted on 2026-09-30 after the generated 1,829-entry catalog reproduced
byte-for-byte, exhaustive Unicode drift validation passed, and CLI, standalone
formatter, LSP quick-fix, and LSP-formatting tests produced identical mappings.
The `replace`, `review`, and `off` policies and diagnostic-only controls are
covered across both feature matrices. Formatting, Clippy, rustdoc, evaluation,
dependency policy/audit checks, and workspace package assembly pass; coverage
is 93.05% (1,487/1,598 lines), and release metadata agrees on `0.1.22`.
## Deferred by design
- Statistical or model-based authorship classification.

View File

@ -65,7 +65,7 @@ fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() {
assert!(output.status.success());
assert!(output.stderr.is_empty());
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("unicode-ascii-replacement"));
#[cfg(feature = "fancy-diagnostics")]
{
assert!(stdout.contains('\u{1b}'));
@ -121,7 +121,7 @@ fn config_show_emits_resolved_toml_with_explicit_overrides() {
let resolved = toml::from_str::<toml::Value>(&stdout).expect("valid TOML");
assert_eq!(resolved["schema_version"].as_integer(), Some(1));
assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23));
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 10);
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_829);
}
#[test]
@ -196,16 +196,19 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
}
#[test]
fn unicode_review_diagnostics_are_non_fixing_and_exact() {
fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() {
let source = "pаss\u{200b}word";
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
assert!(output.status.success());
let record = &json_records(&output)[0];
assert_eq!(record["summary"]["total"], 2);
assert_eq!(record["findings"][0]["rule_id"], "unicode-ascii-confusable");
assert_eq!(
record["findings"][0]["rule_id"],
"unicode-ascii-replacement"
);
assert_eq!(record["findings"][0]["matched_text"], "а");
assert!(record["findings"][0]["fix"].is_null());
assert_eq!(record["findings"][0]["fix"]["replacement"], "a");
assert_eq!(
record["findings"][1]["rule_id"],
"unicode-invisible-control"

View File

@ -88,7 +88,12 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
tp,
tp + false_negatives
);
assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}");
let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1;
assert_eq!(
(tp, fp, false_negatives),
(expected_true_positives, 0, 0),
"metrics for {id}"
);
total_tp += tp;
total_fp += fp;
total_fn += false_negatives;
@ -121,20 +126,27 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
&fs::read_to_string(root.join("signoff.toml")).expect("read evaluation sign-off"),
)
.expect("parse evaluation sign-off");
let catalog_hash = Sha256::digest(include_bytes!(
let mut catalog = Sha256::new();
catalog.update(include_bytes!(
"../crates/john-slop-core/assets/heuristics.toml"
))
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
));
catalog.update([0]);
catalog.update(include_bytes!(
"../crates/john-slop-core/assets/unicode.toml"
));
let catalog_hash = catalog
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
assert_eq!(signoff.schema_version, 1);
assert_eq!(signoff.release, env!("CARGO_PKG_VERSION"));
assert_eq!(signoff.status, "accepted-after-unicode-review-profile");
assert_eq!(signoff.status, "accepted-after-unicode-replacement-profile");
assert_eq!(signoff.catalog_sha256, catalog_hash);
assert!(signoff.limitations_acknowledged);
assert_eq!(signoff.reviewed_on, "2026-09-30");
assert!(signoff.decision.contains("prompt-leakage"));
assert!(signoff.decision.contains("0.90"));
assert!(signoff.decision.contains("UTS #39"));
assert!(signoff.decision.contains("non-mutating"));
assert!(signoff.decision.contains("review"));
}

View File

@ -159,8 +159,7 @@ fn diagnostic_showcase_spans_prose_source_and_severities() {
assert!(output.stderr.is_empty());
assert!(stdout.contains("assistant-self-identification"));
assert!(stdout.contains("prompt-response-scaffolding"));
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("unicode-ascii-confusable"));
assert!(stdout.contains("unicode-ascii-replacement"));
assert!(stdout.contains("unicode-invisible-control"));
assert!(stdout.contains("source-placeholder-todo"));
assert!(stdout.contains("source-tutorial-narration"));

View File

@ -34,13 +34,13 @@ expected = ["source-placeholder-todo", "source-tutorial-narration"]
path = "typography.md"
kind = "prose"
expected_score = 17
expected = ["typography-non-ascii-hyphen"]
expected = ["unicode-ascii-replacement"]
[[cases]]
path = "unicode.md"
kind = "prose"
expected_score = 21
expected = ["unicode-ascii-confusable", "unicode-invisible-control"]
expected_score = 28
expected = ["unicode-ascii-replacement", "unicode-invisible-control"]
[[cases]]
path = "negative.md"
@ -48,8 +48,7 @@ kind = "prose"
expected_score = 0
expected = []
near_miss_for = [
"typography-non-ascii-hyphen",
"unicode-ascii-confusable",
"unicode-ascii-replacement",
"unicode-invisible-control",
]

View File

@ -1,7 +1,7 @@
schema_version = 1
release = "0.1.21"
release = "0.1.22"
reviewed_on = "2026-09-30"
status = "accepted-after-unicode-review-profile"
catalog_sha256 = "94cd36bf49f7061aad027308afb3bf8525a713af65b8ec3b730e0e75bfcef8c5"
status = "accepted-after-unicode-replacement-profile"
catalog_sha256 = "74be9257120a8830f546b3b6e87c1a73b4c55041cf1022c1fe568eb84ef30117"
limitations_acknowledged = true
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Add non-mutating Unicode 16.0 UTS #39 ASCII-skeleton and pinned Gitea invisible/control review rules with exact spans; safe replacements remain a separate allowlist. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."

View File

@ -93,17 +93,11 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
kind: FixtureKind::Rust,
},
RuleCase {
id: "typography-non-ascii-hyphen",
id: "unicode-ascii-replacement",
positive: "Review this—then continue.",
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-ascii-confusable",
positive: "The pаssword contains a Cyrillic lookalike.",
near_miss: "Résumé précis en français.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-invisible-control",
positive: "A zero-width\u{200b}space is hidden here.",

View File

@ -218,7 +218,7 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() {
}
#[tokio::test]
async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
async fn unicode_review_findings_separate_safe_fixes_from_controls() {
let mut harness = Harness::new(None).await;
let uri = "file:///tmp/unicode-review.txt";
let source = "pаss\u{200b}word\u{202e}";
@ -230,8 +230,8 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
.await;
let diagnostics = published["diagnostics"].as_array().unwrap();
assert_eq!(diagnostics.len(), 3);
assert_eq!(diagnostics[0]["code"], "unicode-ascii-confusable");
assert_eq!(diagnostics[0]["data"]["fixable"], false);
assert_eq!(diagnostics[0]["code"], "unicode-ascii-replacement");
assert_eq!(diagnostics[0]["data"]["fixable"], true);
assert_eq!(diagnostics[1]["code"], "unicode-invisible-control");
assert_eq!(diagnostics[2]["code"], "unicode-invisible-control");
@ -245,7 +245,9 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
40,
)
.await;
assert!(formatting.result().unwrap().as_array().unwrap().is_empty());
let formatting = formatting.result().unwrap().as_array().unwrap();
assert_eq!(formatting.len(), 1);
assert_eq!(formatting[0]["newText"], "a");
let actions = harness
.request(
@ -261,7 +263,12 @@ async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
41,
)
.await;
assert!(actions.result().unwrap().as_array().unwrap().is_empty());
let actions = actions.result().unwrap().as_array().unwrap();
assert_eq!(actions.len(), 1);
assert_eq!(
actions[0]["edit"]["documentChanges"][0]["edits"][0]["newText"],
"a"
);
}
fn utf16_position_to_byte(source: &str, position: &Value) -> usize {