chore: release milestone 22
This commit is contained in:
parent
f229c1445c
commit
62a2b6a608
|
|
@ -1,11 +1,11 @@
|
|||
[last_run_clean]
|
||||
run_status = "Success"
|
||||
lines_parsed = 389
|
||||
coverage = 92.4
|
||||
lines_parsed = 393
|
||||
coverage = 93.05
|
||||
|
||||
[last_run_clean.run_duration]
|
||||
secs = 29
|
||||
nanos = 820282692
|
||||
secs = 50
|
||||
nanos = 858077900
|
||||
|
||||
[last_run_incremental]
|
||||
run_status = "Success"
|
||||
|
|
|
|||
16
AGENTS.md
16
AGENTS.md
|
|
@ -14,8 +14,10 @@ accusation. A result must always be explainable from the findings that produced
|
|||
it.
|
||||
|
||||
The formatter scope is deliberately narrow: configured Unicode characters with
|
||||
safe ASCII replacements. The LSP publishes every applicable detector finding
|
||||
but exposes edits only for those safe replacements, reusing the same library
|
||||
ASCII replacements. The generated UTS #39 profile is broad and therefore must
|
||||
retain `replace`, non-mutating `review`, and `off` policies. The LSP publishes
|
||||
every applicable detector finding but exposes edits only under replacement
|
||||
policy, reusing the same library
|
||||
implementations as the CLI and formatter. Do not add prose rewriting, generative
|
||||
inference, or destructive archive editing without a separate design decision.
|
||||
|
||||
|
|
@ -156,11 +158,11 @@ field: replacement/rule tables merge by stable ID, while ignore globs and file
|
|||
extensions replace as complete arrays unless the schema explicitly says
|
||||
otherwise. Do not rely on a library's default array-merge semantics implicitly.
|
||||
|
||||
`crates/john-slop-core/assets/config.toml` is the canonical, documented
|
||||
replacement catalog. Each
|
||||
entry has an ID, Unicode source string/code point, ASCII replacement, rationale,
|
||||
and enabled flag. Reject duplicate sources, empty sources, non-ASCII targets,
|
||||
and conflicting edits. Longer sources win matching precedence. Provide a command
|
||||
`crates/john-slop-core/assets/unicode.toml` is the canonical generated Unicode
|
||||
replacement and diagnostic catalog; `config.toml` owns other defaults. Each
|
||||
replacement entry has an ID, Unicode source string/code point, ASCII target,
|
||||
rationale, and enabled flag. Reject duplicate sources, empty sources, non-ASCII
|
||||
targets, and conflicting edits. Longer sources win matching precedence. Provide a command
|
||||
that prints the resolved configuration with secrets/redacted fields if secrets
|
||||
are ever introduced.
|
||||
|
||||
|
|
|
|||
20
CHANGELOG.md
20
CHANGELOG.md
|
|
@ -6,6 +6,23 @@ All notable changes to John Slop are recorded here. The format follows
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.1.22] - 2026-09-30
|
||||
|
||||
### Added
|
||||
|
||||
- Added a generated, versioned Unicode policy containing 1,829 mappings and
|
||||
their shared diagnostic metadata, plus an exhaustive drift test and
|
||||
deterministic regeneration example.
|
||||
- Added `replace`, `review`, and `off` bulk policies for printable-ASCII UTS #39
|
||||
mappings.
|
||||
|
||||
### Changed
|
||||
|
||||
- Printable-ASCII Unicode 16.0 UTS #39 lookalikes now receive consistent fixes
|
||||
from CLI diagnostics, the standalone formatter, LSP quick fixes, and LSP
|
||||
document formatting under the default `replace` policy.
|
||||
- Invisible and bidirectional controls remain diagnostic-only.
|
||||
|
||||
## [0.1.21] - 2026-09-30
|
||||
|
||||
### Added
|
||||
|
|
@ -60,7 +77,8 @@ All notable changes to John Slop are recorded here. The format follows
|
|||
- Stdio LSP replacement diagnostics and versioned quick fixes.
|
||||
- Runnable examples, reviewed fixtures, fuzz targets, benchmarks, and CI gates.
|
||||
|
||||
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.21...HEAD
|
||||
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.22...HEAD
|
||||
[0.1.22]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.21...v0.1.22
|
||||
[0.1.21]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.20...v0.1.21
|
||||
[0.1.20]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.0...v0.1.20
|
||||
[0.1.0]: https://git.dandaedre.com/idle-org/john_slop/releases/tag/v0.1.0
|
||||
|
|
|
|||
|
|
@ -699,7 +699,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-core"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"directories",
|
||||
"figment",
|
||||
|
|
@ -717,7 +717,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-lsp"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"john-slop-core",
|
||||
"serde_json",
|
||||
|
|
@ -728,7 +728,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john_slop"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"criterion",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ default-members = [".", "crates/john-slop-core", "crates/john-slop-lsp"]
|
|||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
edition = "2024"
|
||||
rust-version = "1.98.1"
|
||||
license = "MIT"
|
||||
|
|
@ -18,8 +18,8 @@ figment = { version = "0.10.19", default-features = false, features = ["toml"] }
|
|||
futures = { version = "0.3.34", default-features = false, features = ["std"] }
|
||||
globset = { version = "0.4.20", default-features = false }
|
||||
insta = { version = "1.48.0", default-features = false }
|
||||
john-slop-core = { version = "0.1.21", path = "crates/john-slop-core" }
|
||||
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.21", path = "crates/john-slop-lsp" }
|
||||
john-slop-core = { version = "0.1.22", path = "crates/john-slop-core" }
|
||||
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.22", path = "crates/john-slop-lsp" }
|
||||
miette = { version = "7.6.0", default-features = false, features = ["derive"] }
|
||||
regex = "1.13.1"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ spans:
|
|||
- `john-slop` detects copied assistant scaffolding, generic style residue,
|
||||
source placeholders, tutorial narration, configured replacements, ASCII-like
|
||||
Unicode, and invisible or bidirectional controls.
|
||||
- `john-slop-fmt` previews, checks, or explicitly applies safe configured ASCII
|
||||
replacements.
|
||||
- `john-slop-fmt` previews, checks, or explicitly applies configured ASCII
|
||||
replacements, including the default printable-ASCII UTS #39 profile.
|
||||
- `john-slop-lsp` publishes the detector's heuristic diagnostics over stdio,
|
||||
offers quick fixes for safe replacements, and provides formatter-equivalent
|
||||
document edits without writing editor-owned files.
|
||||
|
|
@ -69,7 +69,8 @@ configured signals; it is not the probability that AI wrote the content.
|
|||
Analysis is local and deterministic, with no network requests or telemetry.
|
||||
False positives remain possible—especially in transcripts, tutorials,
|
||||
multilingual text, and intentional typography—so findings are prompts for
|
||||
review, not verdicts.
|
||||
review, not verdicts. Projects with multilingual text should consider the
|
||||
Unicode profile's non-mutating `review` mode before writing formatted output.
|
||||
|
||||
## Learn progressively
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="122" height="18" role="img" aria-label="test-coverage: 92%"><title>test-coverage: 92%</title><linearGradient id="bms-9dc65f7f72910b0a" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-9dc65f7f72910b0a"><rect width="122" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-9dc65f7f72910b0a)"><rect width="87" height="18" fill="#555"/><rect x="87" width="35" height="18" fill="#97ca00"/><rect width="122" height="18" fill="url(#bms-9dc65f7f72910b0a)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="445" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">test-coverage</text><text x="445" y="130" transform="scale(.1)" fill="#fff" textLength="770">test-coverage</text><text aria-hidden="true" x="1035" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="250">92%</text><text x="1035" y="130" transform="scale(.1)" fill="#fff" textLength="250">92%</text></g></svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="122" height="18" role="img" aria-label="test-coverage: 93%"><title>test-coverage: 93%</title><linearGradient id="bms-317a4207573154a7" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-317a4207573154a7"><rect width="122" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-317a4207573154a7)"><rect width="87" height="18" fill="#555"/><rect x="87" width="35" height="18" fill="#97ca00"/><rect width="122" height="18" fill="url(#bms-317a4207573154a7)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="445" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">test-coverage</text><text x="445" y="130" transform="scale(.1)" fill="#fff" textLength="770">test-coverage</text><text aria-hidden="true" x="1035" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="250">93%</text><text x="1035" y="130" transform="scale(.1)" fill="#fff" textLength="250">93%</text></g></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |
|
|
@ -1 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.21"><title>version: 0.1.21</title><linearGradient id="bms-3d708dc9f44ae746" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-3d708dc9f44ae746"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-3d708dc9f44ae746)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-3d708dc9f44ae746)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.21</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.21</text></g></svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.22"><title>version: 0.1.22</title><linearGradient id="bms-49fd3ccd9d32982d" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-49fd3ccd9d32982d"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-49fd3ccd9d32982d)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-49fd3ccd9d32982d)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.22</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.22</text></g></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |
|
|
@ -3,6 +3,9 @@
|
|||
# Do not edit generated mappings by hand; edit this generator and regenerate.
|
||||
schema_version = 1
|
||||
|
||||
[unicode]
|
||||
confusables = "replace"
|
||||
|
||||
[[replacements]]
|
||||
id = "hyphen"
|
||||
source = "‐"
|
||||
|
|
|
|||
|
|
@ -23,10 +23,16 @@ const CURATED: &[(&str, &str, &str, &str)] = &[
|
|||
#[derive(Serialize)]
|
||||
struct Catalog {
|
||||
schema_version: u32,
|
||||
unicode: UnicodePolicy,
|
||||
replacements: Vec<Replacement>,
|
||||
rules: Vec<Rule>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct UnicodePolicy {
|
||||
confusables: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Replacement {
|
||||
id: String,
|
||||
|
|
@ -101,6 +107,9 @@ fn main() {
|
|||
|
||||
let catalog = Catalog {
|
||||
schema_version: 1,
|
||||
unicode: UnicodePolicy {
|
||||
confusables: "replace",
|
||||
},
|
||||
replacements,
|
||||
rules: vec![
|
||||
Rule {
|
||||
|
|
|
|||
|
|
@ -369,12 +369,6 @@ pub enum ConfigValidationError {
|
|||
/// Heuristic rule ID.
|
||||
id: String,
|
||||
},
|
||||
/// Confusable matchers must select the pinned UTS #39 profile.
|
||||
#[error("confusable rule `{id}` must use pattern `uts39-ascii-skeleton-v16`")]
|
||||
InvalidConfusableMatcher {
|
||||
/// Heuristic rule ID.
|
||||
id: String,
|
||||
},
|
||||
/// Invisible matchers must select the pinned Gitea profile.
|
||||
#[error("invisible rule `{id}` must use pattern `gitea-invisible-9c715c0`")]
|
||||
InvalidInvisibleMatcher {
|
||||
|
|
@ -503,10 +497,17 @@ struct ConfigLayer {
|
|||
schema_version: Option<u32>,
|
||||
input: InputLayer,
|
||||
thresholds: ThresholdLayer,
|
||||
unicode: UnicodeLayer,
|
||||
replacements: Vec<ReplacementPatch>,
|
||||
rules: Vec<RulePatch>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
struct UnicodeLayer {
|
||||
confusables: Option<ConfusablePolicy>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
struct InputLayer {
|
||||
|
|
@ -562,6 +563,9 @@ fn apply_layer(
|
|||
}
|
||||
apply_input_layer(&mut config.input, layer.input);
|
||||
apply_threshold_layer(&mut config.thresholds, layer.thresholds);
|
||||
if let Some(value) = layer.unicode.confusables {
|
||||
config.unicode.confusables = value;
|
||||
}
|
||||
merge_replacements(&mut config.replacements, layer.replacements, path)?;
|
||||
merge_rules(&mut config.rules, layer.rules, path)
|
||||
}
|
||||
|
|
@ -780,6 +784,9 @@ pub struct Config {
|
|||
pub schema_version: u32,
|
||||
/// Enabled replacement rules in deterministic matching order.
|
||||
pub replacements: Vec<ReplacementRule>,
|
||||
/// Bulk behavior for generated Unicode mappings.
|
||||
#[serde(default)]
|
||||
pub unicode: UnicodePolicy,
|
||||
/// Configured detection rules in deterministic declaration order.
|
||||
pub rules: Vec<HeuristicRule>,
|
||||
/// Named evidence-band boundaries.
|
||||
|
|
@ -788,6 +795,34 @@ pub struct Config {
|
|||
pub input: InputLimits,
|
||||
}
|
||||
|
||||
/// Bulk policy for Unicode mappings whose targets come from UTS #39.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UnicodePolicy {
|
||||
/// Whether generated lookalikes are replaced, reviewed, or ignored.
|
||||
pub confusables: ConfusablePolicy,
|
||||
}
|
||||
|
||||
impl Default for UnicodePolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
confusables: ConfusablePolicy::Replace,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Action applied to generated UTS #39 printable-ASCII mappings.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum ConfusablePolicy {
|
||||
/// Emit diagnostics and formatter/LSP edits.
|
||||
Replace,
|
||||
/// Emit diagnostics without edits.
|
||||
Review,
|
||||
/// Emit neither diagnostics nor edits for generated mappings.
|
||||
Off,
|
||||
}
|
||||
|
||||
/// One configured data-driven detection rule.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
|
@ -828,8 +863,6 @@ pub enum MatcherType {
|
|||
RepeatedRegex,
|
||||
/// Findings produced by the configured replacement catalog.
|
||||
Replacement,
|
||||
/// Non-mutating findings for UTS #39 characters with an ASCII skeleton.
|
||||
Confusable,
|
||||
/// Non-mutating findings for pinned invisible and bidi control characters.
|
||||
Invisible,
|
||||
}
|
||||
|
|
@ -1053,12 +1086,6 @@ fn validate_rules(rules: &[HeuristicRule]) -> Result<(), ConfigValidationError>
|
|||
});
|
||||
}
|
||||
MatcherType::Replacement => {}
|
||||
MatcherType::Confusable if rule.pattern != "uts39-ascii-skeleton-v16" => {
|
||||
return Err(ConfigValidationError::InvalidConfusableMatcher {
|
||||
id: rule.id.clone(),
|
||||
});
|
||||
}
|
||||
MatcherType::Confusable => {}
|
||||
MatcherType::Invisible if rule.pattern != "gitea-invisible-9c715c0" => {
|
||||
return Err(ConfigValidationError::InvalidInvisibleMatcher {
|
||||
id: rule.id.clone(),
|
||||
|
|
@ -1173,6 +1200,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn embedded_unicode_catalog_covers_every_printable_ascii_skeleton() {
|
||||
assert_eq!(unicode_security::UNICODE_VERSION, (16, 0, 0));
|
||||
let config = Config::embedded();
|
||||
let replacements = config
|
||||
.replacements
|
||||
|
|
@ -1617,6 +1645,9 @@ low = 10
|
|||
medium = 40
|
||||
high = 90
|
||||
|
||||
[unicode]
|
||||
confusables = "review"
|
||||
|
||||
[[replacements]]
|
||||
id = "em-dash"
|
||||
source = "⸺"
|
||||
|
|
@ -1662,6 +1693,7 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
|
|||
high: 90
|
||||
}
|
||||
);
|
||||
assert_eq!(config.unicode.confusables, ConfusablePolicy::Review);
|
||||
let replacement = config
|
||||
.replacements
|
||||
.iter()
|
||||
|
|
@ -1813,18 +1845,6 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
|
|||
)
|
||||
},
|
||||
);
|
||||
assert_invalid(
|
||||
|config| {
|
||||
config.rules[0].matcher = MatcherType::Confusable;
|
||||
config.rules[0].pattern = "wrong".into();
|
||||
},
|
||||
|error| {
|
||||
matches!(
|
||||
error,
|
||||
ConfigValidationError::InvalidConfusableMatcher { .. }
|
||||
)
|
||||
},
|
||||
);
|
||||
assert_invalid(
|
||||
|config| {
|
||||
config.rules[0].matcher = MatcherType::Invisible;
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ impl Rule {
|
|||
CompiledMatcher::Regex(regex) => regex.as_str(),
|
||||
CompiledMatcher::RepeatedRegex(regex) => regex.as_str(),
|
||||
CompiledMatcher::Replacement => "configured-replacements",
|
||||
CompiledMatcher::Confusable => "uts39-ascii-skeleton-v16",
|
||||
CompiledMatcher::Invisible => "gitea-invisible-9c715c0",
|
||||
}
|
||||
}
|
||||
|
|
@ -117,7 +116,6 @@ enum CompiledMatcher {
|
|||
Regex(Regex),
|
||||
RepeatedRegex(Regex),
|
||||
Replacement,
|
||||
Confusable,
|
||||
Invisible,
|
||||
}
|
||||
|
||||
|
|
@ -269,7 +267,6 @@ impl Rule {
|
|||
CompiledMatcher::RepeatedRegex(regex)
|
||||
}
|
||||
MatcherType::Replacement => CompiledMatcher::Replacement,
|
||||
MatcherType::Confusable => CompiledMatcher::Confusable,
|
||||
MatcherType::Invisible => CompiledMatcher::Invisible,
|
||||
};
|
||||
let scope = CompiledScope::compile(&definition)?;
|
||||
|
|
@ -343,34 +340,18 @@ impl Rule {
|
|||
.collect()
|
||||
}
|
||||
}
|
||||
CompiledMatcher::Replacement => replacement::plan(config, source)
|
||||
.into_iter()
|
||||
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
|
||||
.collect(),
|
||||
CompiledMatcher::Confusable => source
|
||||
.char_indices()
|
||||
.filter_map(|(start, character)| {
|
||||
let matched = character.to_string();
|
||||
if config
|
||||
.replacements
|
||||
.iter()
|
||||
.any(|rule| rule.enabled && rule.source == matched)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let skeleton = unicode_review::ascii_skeleton(character)?;
|
||||
let mut finding =
|
||||
self.finding(source, start..start + character.len_utf8(), None);
|
||||
finding.help = Some(format!(
|
||||
"Review U+{:04X}; its Unicode {}.{}.{} UTS #39 skeleton is {skeleton:?}. This review finding is not an automatic replacement.",
|
||||
character as u32,
|
||||
unicode_review::UTS39_UNICODE_VERSION.0,
|
||||
unicode_review::UTS39_UNICODE_VERSION.1,
|
||||
unicode_review::UTS39_UNICODE_VERSION.2,
|
||||
));
|
||||
Some(finding)
|
||||
})
|
||||
.collect(),
|
||||
CompiledMatcher::Replacement => {
|
||||
let mut findings = replacement::plan(config, source)
|
||||
.into_iter()
|
||||
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
|
||||
.collect::<Vec<_>>();
|
||||
findings.extend(
|
||||
replacement::plan_confusables_for_review(config, source)
|
||||
.into_iter()
|
||||
.map(|edit| self.finding(source, edit.byte_span, None)),
|
||||
);
|
||||
findings
|
||||
}
|
||||
CompiledMatcher::Invisible => source
|
||||
.char_indices()
|
||||
.filter(|(start, character)| {
|
||||
|
|
@ -596,6 +577,25 @@ mod tests {
|
|||
assert!(findings.iter().all(|finding| finding.matched_text != "\r"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confusable_policy_switches_between_fix_review_and_off() {
|
||||
let mut config = Config::embedded();
|
||||
|
||||
config.unicode.confusables = crate::ConfusablePolicy::Review;
|
||||
let review = Detector::new(&config).unwrap().analyze("pаss");
|
||||
assert_eq!(review.findings.len(), 1);
|
||||
assert!(review.findings[0].fix.is_none());
|
||||
|
||||
config.unicode.confusables = crate::ConfusablePolicy::Off;
|
||||
assert!(
|
||||
Detector::new(&config)
|
||||
.unwrap()
|
||||
.analyze("pаss")
|
||||
.findings
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_replacement_characters_are_not_duplicated_as_confusable_findings() {
|
||||
let findings = Detector::new(&Config::embedded())
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ mod unicode_review;
|
|||
|
||||
pub use config::{
|
||||
Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError,
|
||||
ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, EvidenceThresholds,
|
||||
HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule, RuleScope,
|
||||
ThresholdOverrides,
|
||||
ConfigOverrides, ConfigPaths, ConfigValidationError, ConfusablePolicy, ContentKind,
|
||||
EvidenceThresholds, HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule,
|
||||
RuleScope, ThresholdOverrides, UnicodePolicy,
|
||||
};
|
||||
pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule};
|
||||
pub use diagnostic::FindingDiagnostic;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use std::{borrow::Cow, collections::BTreeMap, ops::Range};
|
|||
|
||||
use miette::Diagnostic;
|
||||
|
||||
use crate::{Config, TextEdit};
|
||||
use crate::{Config, ConfusablePolicy, ReplacementRule, TextEdit};
|
||||
|
||||
/// An edit set cannot be applied safely to its source snapshot.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error, Diagnostic)]
|
||||
|
|
@ -36,9 +36,30 @@ pub enum ReplacementError {
|
|||
|
||||
/// Plan deterministic non-overlapping edits without modifying the source.
|
||||
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||
plan_matching(config, source, |rule| {
|
||||
rule.enabled
|
||||
&& (!rule.id.starts_with("uts39-")
|
||||
|| config.unicode.confusables == ConfusablePolicy::Replace)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn plan_confusables_for_review(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||
if config.unicode.confusables != ConfusablePolicy::Review {
|
||||
return Vec::new();
|
||||
}
|
||||
plan_matching(config, source, |rule| {
|
||||
rule.enabled && rule.id.starts_with("uts39-")
|
||||
})
|
||||
}
|
||||
|
||||
fn plan_matching(
|
||||
config: &Config,
|
||||
source: &str,
|
||||
include: impl Fn(&ReplacementRule) -> bool,
|
||||
) -> Vec<TextEdit> {
|
||||
let mut single_character_rules = BTreeMap::new();
|
||||
let mut sequence_rules = Vec::new();
|
||||
for rule in config.replacements.iter().filter(|rule| rule.enabled) {
|
||||
for rule in config.replacements.iter().filter(|rule| include(rule)) {
|
||||
let mut characters = rule.source.chars();
|
||||
let first = characters.next();
|
||||
if let (Some(character), None) = (first, characters.next()) {
|
||||
|
|
@ -175,6 +196,18 @@ mod tests {
|
|||
assert_eq!(edits[4].replacement, "rn");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_confusable_policy_preserves_curated_replacements() {
|
||||
let mut config = Config::embedded();
|
||||
config.unicode.confusables = ConfusablePolicy::Review;
|
||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
||||
assert_eq!(plan_confusables_for_review(&config, "а—").len(), 1);
|
||||
|
||||
config.unicode.confusables = ConfusablePolicy::Off;
|
||||
assert_eq!(apply("а—", &plan(&config, "а—")).unwrap(), "а-");
|
||||
assert!(plan_confusables_for_review(&config, "а—").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_or_ascii_sources_need_no_edits() {
|
||||
let config = Config::embedded();
|
||||
|
|
|
|||
|
|
@ -1,17 +1,4 @@
|
|||
//! Pinned Unicode review profiles that never authorize automatic rewriting.
|
||||
|
||||
/// Unicode version carried by `unicode-security` 0.1.2.
|
||||
pub(crate) const UTS39_UNICODE_VERSION: (u64, u64, u64) = unicode_security::UNICODE_VERSION;
|
||||
|
||||
/// Return an ASCII UTS #39 skeleton when one non-ASCII scalar resembles ASCII.
|
||||
pub(crate) fn ascii_skeleton(character: char) -> Option<String> {
|
||||
if character.is_ascii() {
|
||||
return None;
|
||||
}
|
||||
let source = character.to_string();
|
||||
let skeleton = unicode_security::skeleton(&source).collect::<String>();
|
||||
(!skeleton.is_empty() && skeleton.is_ascii() && skeleton != source).then_some(skeleton)
|
||||
}
|
||||
//! Pinned invisible and bidirectional-control review profile.
|
||||
|
||||
/// Match the invisible/control profile generated by Gitea at commit 9c715c0.
|
||||
///
|
||||
|
|
@ -55,15 +42,6 @@ pub(crate) const fn is_gitea_invisible(character: char) -> bool {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn uts39_profile_reports_ascii_lookalikes_but_not_ascii_or_accents() {
|
||||
assert_eq!(ascii_skeleton('а').as_deref(), Some("a"));
|
||||
assert_eq!(ascii_skeleton('𝟘').as_deref(), Some("O"));
|
||||
assert_eq!(ascii_skeleton('a'), None);
|
||||
assert_eq!(ascii_skeleton('é'), None);
|
||||
assert_eq!(UTS39_UNICODE_VERSION, (16, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gitea_profile_covers_invisible_and_bidi_ranges_without_normal_whitespace() {
|
||||
assert!(is_gitea_invisible('\u{200b}'));
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ workspace Tokio requirement enables no features: the LSP library selects
|
|||
|
||||
The modules within those crates retain deliberately narrow responsibilities:
|
||||
|
||||
- `config` owns the validated embedded replacement catalog and input limits;
|
||||
- `config` owns validated embedded defaults, the generated Unicode policy, and
|
||||
input limits;
|
||||
- `input` streams files, directory entries, and archive members as documents;
|
||||
- `detector` compiles configured matchers and path scopes once, then produces
|
||||
explainable findings and a bounded low-care evidence score;
|
||||
|
|
@ -44,7 +45,8 @@ The modules within those crates retain deliberately narrow responsibilities:
|
|||
- `replacement` produces non-overlapping edits and validates/applies arbitrary
|
||||
edit sets without performing I/O;
|
||||
- `unicode_review` pins the UTS #39 ASCII-skeleton version and Gitea-compatible
|
||||
invisible/control ranges used only by non-mutating detector matchers;
|
||||
invisible/control ranges; generated printable-ASCII mappings feed the shared
|
||||
replacement planner while controls remain non-mutating;
|
||||
- `lsp` indexes open snapshots, runs the compiled detector, converts canonical
|
||||
finding and replacement spans to UTF-16 coordinates, and implements
|
||||
versioned quick fixes plus whole-document formatting;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Configuration
|
||||
|
||||
John Slop embeds schema-versioned replacement and heuristic catalogs in every
|
||||
John Slop embeds schema-versioned base, heuristic, and Unicode catalogs in every
|
||||
binary. Installed binaries therefore work without a source checkout or external
|
||||
default files.
|
||||
|
||||
|
|
@ -29,9 +29,10 @@ files are optional; a selected explicit file must exist.
|
|||
|
||||
## Schema
|
||||
|
||||
The current `schema_version` is `1`. The complete defaults and documented record
|
||||
examples live in [`config.toml`](../crates/john-slop-core/assets/config.toml) and
|
||||
[`heuristics.toml`](../crates/john-slop-core/assets/heuristics.toml).
|
||||
The current `schema_version` is `1`. The complete defaults live in
|
||||
[`config.toml`](../crates/john-slop-core/assets/config.toml),
|
||||
[`heuristics.toml`](../crates/john-slop-core/assets/heuristics.toml), and the
|
||||
generated [`unicode.toml`](../crates/john-slop-core/assets/unicode.toml).
|
||||
Ready-to-copy, commented partial layers live in
|
||||
[`examples/config`](../examples/config/): `config.toml` demonstrates a user-wide
|
||||
base and `editorial.toml` demonstrates a named overlay.
|
||||
|
|
@ -60,12 +61,24 @@ replacement targets, and invalid thresholds are fatal configuration
|
|||
diagnostics.
|
||||
|
||||
The built-in matcher names are `exact-phrase`, `regex`, `repeated-regex`,
|
||||
`replacement`, `confusable`, and `invisible`. The last three use validated
|
||||
sentinel patterns because their implementations are data-backed rather than
|
||||
free-form strings: `configured-replacements`, `uts39-ascii-skeleton-v16`, and
|
||||
`gitea-invisible-9c715c0`. They remain ordinary rule records, so later layers
|
||||
`replacement`, and `invisible`. The last two use validated sentinel patterns
|
||||
because their implementations are data-backed rather than free-form strings:
|
||||
`configured-replacements` and `gitea-invisible-9c715c0`. They remain ordinary
|
||||
rule records, so later layers
|
||||
can disable them or adjust severity, weight, occurrence cap, and scope without
|
||||
changing code. Confusable and invisible findings never authorize edits.
|
||||
changing code. Invisible findings never authorize edits; generated confusable
|
||||
mappings follow the bulk policy below.
|
||||
|
||||
The generated UTS #39 mappings have one bulk policy:
|
||||
|
||||
```toml
|
||||
[unicode]
|
||||
confusables = "review" # replace (default), review, or off
|
||||
```
|
||||
|
||||
`replace` enables diagnostics and edits, `review` retains diagnostics without
|
||||
formatter/LSP edits, and `off` suppresses generated lookalikes. The ten curated
|
||||
punctuation replacements and invisible/control diagnostics are unaffected.
|
||||
|
||||
Set `enabled = false` on a known rule or replacement ID to disable it. Because
|
||||
records merge by ID, do not copy the full embedded table merely to change one
|
||||
|
|
@ -77,6 +90,19 @@ must be unique. When enabled sources share a prefix, the longest matching source
|
|||
wins at each byte offset, with stable rule ID ordering as the deterministic
|
||||
tie-break. Edits always refer to the original UTF-8 snapshot.
|
||||
|
||||
Generated mapping IDs use `uts39-u<lowercase-codepoint>`, so an individual
|
||||
entry can still be disabled or overridden through normal merge-by-ID behavior.
|
||||
Maintain the embedded list by editing
|
||||
`crates/john-slop-core/examples/generate_unicode_catalog.rs` and running:
|
||||
|
||||
```sh
|
||||
cargo run -p john-slop-core --example generate_unicode_catalog \
|
||||
> crates/john-slop-core/assets/unicode.toml
|
||||
```
|
||||
|
||||
The exhaustive catalog test walks every Unicode scalar and rejects drift from
|
||||
the pinned Unicode 16.0 tables.
|
||||
|
||||
Each heuristic also configures `severity` as `advice`, `warning`, or `error`.
|
||||
Severity controls CLI presentation, LSP severity, and summary counts; the
|
||||
evidence score still comes from the independently configured weight and
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@ Run line coverage with Cargo Tarpaulin:
|
|||
cargo tarpaulin --config tarpaulin.toml
|
||||
```
|
||||
|
||||
The configuration enforces a 90% floor. The current suite measured 92.40%
|
||||
(1,472/1,593 lines) on 2026-09-30. The remaining lines are dominated by
|
||||
The configuration enforces a 90% floor. The current suite measured 93.05%
|
||||
(1,487/1,598 lines) on 2026-09-30. The remaining lines are dominated by
|
||||
platform/I/O failure injection, malformed archive internals, coordinate
|
||||
overflow, LSP logging/reload failures, and subprocess attribution. Do not add
|
||||
brittle tests or production seams solely to make Tarpaulin count those paths.
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ With the embedded schema-version 1 catalog, the corpus currently reports:
|
|||
| repeated-transition | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
|
||||
| typography-non-ascii-hyphen | 1 | 0 | 0 | 100% | 100% |
|
||||
| unicode-ascii-confusable | 1 | 0 | 0 | 100% | 100% |
|
||||
| unicode-ascii-replacement | 2 | 0 | 0 | 100% | 100% |
|
||||
| unicode-invisible-control | 1 | 0 | 0 | 100% | 100% |
|
||||
|
||||
Aggregate labeled-rule counts are 13 true positives, 0 false positives, and 0
|
||||
|
|
@ -36,7 +35,7 @@ The same manifest pins evidence scores for regression characterization:
|
|||
| style signals | 19 |
|
||||
| source-code signals | 24 |
|
||||
| typography signal | 17 |
|
||||
| one ASCII lookalike and one invisible character | 21 |
|
||||
| one ASCII lookalike and one invisible character | 28 |
|
||||
| each of five negative documents | 0 |
|
||||
|
||||
These perfect values describe a tiny purpose-built regression set, not expected
|
||||
|
|
@ -50,11 +49,11 @@ Each shipped rule has at least one explicitly labeled realistic near miss. The
|
|||
initial 0.1.0 review kept all schema-version 1 rules, weights, and thresholds
|
||||
unchanged. The prompt-leakage review accepted lighter `0.90ⁿ` correlation
|
||||
damping for distinct prompt-leakage rules while retaining `0.75ⁿ` for other
|
||||
categories. The Milestone 21 review adds the non-mutating UTS #39 ASCII-skeleton
|
||||
and Gitea-compatible invisible/control profiles, with explicit multilingual
|
||||
limitations. The checked-in `signoff.toml` records those decisions and contains
|
||||
the SHA-256 digest of
|
||||
`crates/john-slop-core/assets/heuristics.toml`.
|
||||
categories. The Milestone 22 review makes printable-ASCII UTS #39 mappings
|
||||
replaceable by default, retains a non-mutating `review` policy, and keeps the
|
||||
Gitea-compatible invisible/control profile diagnostic-only, with explicit
|
||||
multilingual limitations. The checked-in `signoff.toml` records those decisions
|
||||
and contains the SHA-256 digest of both `heuristics.toml` and `unicode.toml`.
|
||||
|
||||
Any default rule, matcher, pattern, weight, occurrence cap, scope, or threshold
|
||||
change invalidates the sign-off test. Renewal requires reviewing positive and
|
||||
|
|
|
|||
|
|
@ -5,18 +5,19 @@ Every contribution to the score is represented by a finding with an exact
|
|||
source span, stable rule ID, category, message, help, and configured weight.
|
||||
|
||||
The detector compiles the resolved rule catalog once. Exact phrases, regular
|
||||
expressions, repeated-pattern rules, configured Unicode replacements, UTS #39
|
||||
ASCII lookalikes, and invisible/control characters share the same finding
|
||||
expressions, repeated-pattern rules, configured Unicode replacements, and
|
||||
invisible/control characters share the same finding
|
||||
model. Content kind, extension, path globs, and conventional generated/vendor/
|
||||
minified paths decide whether each rule applies. Findings are deduplicated by
|
||||
rule and byte span, then sorted by start offset, end offset, and rule ID so
|
||||
every adapter observes the same order.
|
||||
|
||||
Unicode review rules are deliberately distinct from replacement rules. The
|
||||
detector can flag a lookalike, invisible scalar, or direction-changing control
|
||||
at its exact span, but only the explicit ten-entry safe replacement allowlist
|
||||
can produce a formatter edit or LSP quick fix. A safe replacement finding is
|
||||
not duplicated as a confusable finding.
|
||||
The generated Unicode 16.0 profile maps every non-ASCII scalar with a printable
|
||||
ASCII UTS #39 skeleton, alongside ten manually reviewed punctuation mappings.
|
||||
The default `replace` policy gives each of these findings a formatter/LSP edit;
|
||||
`review` keeps the same exact diagnostics without edits, and `off` suppresses
|
||||
the generated lookalikes. Invisible and direction-changing controls remain
|
||||
diagnostic-only because deleting them silently is not a generally safe edit.
|
||||
|
||||
## Evidence-score formula
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,9 @@ documentation, notebooks, and uncommon programming languages are
|
|||
underrepresented. File-extension inference covers a conservative common set and
|
||||
defaults unknown paths to prose. Teams should validate rules against their own
|
||||
reviewed material before changing enforcement thresholds. Disable or narrow
|
||||
`unicode-ascii-confusable` when ordinary multilingual text makes character-level
|
||||
review too noisy; locale-aware suppression is not implemented.
|
||||
set `[unicode] confusables = "review"` to keep findings without edits, or use
|
||||
`"off"` when ordinary multilingual text makes character-level review too noisy.
|
||||
Locale-aware suppression is not implemented.
|
||||
|
||||
## Evidence score is not probability
|
||||
|
||||
|
|
@ -81,8 +82,10 @@ synchronization. It publishes the configured heuristic catalog but does not
|
|||
publish a document-level evidence score; editor diagnostics remain individual
|
||||
review signals. It does not read or write a document URI, mutate archives,
|
||||
provide prose rewrites, or implement incremental range changes. Only configured
|
||||
ASCII replacements receive quick fixes or formatting edits. Confusable,
|
||||
invisible, and bidirectional-control findings are diagnostic-only, and
|
||||
ASCII replacements receive quick fixes or formatting edits. Under the default
|
||||
policy this includes printable-ASCII UTS #39 skeletons, which can alter valid
|
||||
Greek, Cyrillic, mathematical, or other text; use `review` for non-mutating
|
||||
diagnostics. Invisible and bidirectional-control findings are diagnostic-only, and
|
||||
configuration reload errors retain the last valid configuration and compiled
|
||||
detector.
|
||||
|
||||
|
|
|
|||
|
|
@ -52,11 +52,11 @@ modes.
|
|||
},
|
||||
"findings": [
|
||||
{
|
||||
"rule_id": "typography-non-ascii-hyphen",
|
||||
"rule_id": "unicode-ascii-replacement",
|
||||
"category": "typography",
|
||||
"severity": "advice",
|
||||
"message": "a configured non-ASCII character has an ASCII replacement",
|
||||
"help": "Review the punctuation or apply the configured safe replacement.",
|
||||
"message": "a Unicode character or sequence has a configured ASCII replacement",
|
||||
"help": "Review the character or apply the configured ASCII replacement.",
|
||||
"byte_span": { "start": 8, "end": 11 },
|
||||
"matched_text": "—",
|
||||
"weight": 10.0,
|
||||
|
|
|
|||
|
|
@ -29,31 +29,41 @@ locale-aware lookalike table.
|
|||
|
||||
## Implemented John Slop profile
|
||||
|
||||
These sources are useful detection references, not safe formatter replacement
|
||||
catalogs. Many listed characters are legitimate in particular languages and
|
||||
scripts, and Gitea deliberately interprets them through the reviewer's locale.
|
||||
Importing every pair as an automatic Unicode-to-ASCII edit would corrupt valid
|
||||
text and violate the formatter's conservative replacement contract.
|
||||
John Slop uses `unicode-security` 0.1.2, whose checked-in UTS #39 tables target
|
||||
Unicode 16.0. The generated `assets/unicode.toml` contains every non-ASCII
|
||||
scalar whose single-character skeleton is non-empty printable ASCII, plus ten
|
||||
reviewed punctuation mappings. That is 1,829 mappings in the current profile.
|
||||
Each entry has a stable ID, source, target, rationale, and enabled switch, and
|
||||
the same asset owns the diagnostic metadata.
|
||||
|
||||
John Slop keeps three policies separate:
|
||||
The default `[unicode] confusables = "replace"` policy emits
|
||||
`unicode-ascii-replacement` diagnostics and supplies the same edits to CLI
|
||||
output, `john-slop-fmt`, LSP quick fixes, and LSP formatting. Set the value to
|
||||
`review` for identical findings without edits, or `off` to suppress generated
|
||||
lookalikes. The ten curated punctuation mappings remain active under all three
|
||||
settings and can still be disabled individually.
|
||||
|
||||
1. configured, explicitly safe formatter replacements;
|
||||
2. non-mutating confusable findings for human review; and
|
||||
3. non-mutating invisible and bidirectional-control findings.
|
||||
|
||||
`unicode-ascii-confusable` uses `unicode-security` 0.1.2, whose checked-in UTS
|
||||
#39 tables target Unicode 16.0. It reports a non-ASCII scalar only when its
|
||||
single-character skeleton is entirely ASCII. It is intentionally broader than
|
||||
Gitea's locale selection and can flag legitimate scripts; locale-aware policy
|
||||
remains deferred. Characters already covered by the safe replacement allowlist
|
||||
produce only the fixable replacement finding, not a duplicate confusable one.
|
||||
This broad mapping is intentionally more aggressive than Gitea's locale-aware
|
||||
display warning. Greek, Cyrillic, mathematical, and other characters may be
|
||||
legitimate text, so projects handling multilingual content should prefer
|
||||
`review` unless ASCII normalization is their explicit policy.
|
||||
|
||||
`unicode-invisible-control` uses the invisible ranges generated by Gitea at the
|
||||
pinned commit above, including bidi embedding/override and isolate controls.
|
||||
Ordinary space, tab, line feed, and the carriage return in a CRLF pair are not
|
||||
reported. Each other match receives an exact UTF-8 source span that adapters
|
||||
convert normally. Neither rule feeds the formatter or LSP quick fixes.
|
||||
convert normally. Controls never feed the formatter or LSP quick fixes because
|
||||
silent deletion has no generally safe meaning.
|
||||
|
||||
Regenerate the checked-in policy deterministically with:
|
||||
|
||||
```sh
|
||||
cargo run -p john-slop-core --example generate_unicode_catalog \
|
||||
> crates/john-slop-core/assets/unicode.toml
|
||||
```
|
||||
|
||||
An exhaustive test walks all Unicode scalar values and rejects mapping drift.
|
||||
|
||||
Future profile updates must pin their upstream Unicode/data version, retain
|
||||
provenance and licensing, test ordinary multilingual near misses, renew the
|
||||
hash-bound default-policy sign-off, and preserve the no-silent-rewrite boundary.
|
||||
provenance and licensing, test ordinary multilingual near misses, and renew the
|
||||
hash-bound default-policy sign-off.
|
||||
|
|
|
|||
|
|
@ -73,12 +73,12 @@ embedded catalog remains limited to individually reviewed conservative rules.
|
|||
[`diagnostic-showcase.txt`](diagnostic-showcase.txt) contains every currently
|
||||
shipped prose signal, while
|
||||
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
|
||||
source-placeholder and tutorial-narration rules plus all ten embedded
|
||||
replacement characters, a Cyrillic ASCII lookalike, a zero-width space, and a
|
||||
source-placeholder and tutorial-narration rules plus the ten curated
|
||||
punctuation replacements, a fixable Cyrillic ASCII lookalike, a zero-width space, and a
|
||||
right-to-left override. The adjacent example configuration promotes assistant
|
||||
self-identification from warning to error and lifts the typography occurrence
|
||||
cap for this display, making one run show Miette advice, warning, error,
|
||||
fixable, and deliberately non-fixable Unicode diagnostics together:
|
||||
fixable, and deliberately non-fixable control diagnostics together:
|
||||
|
||||
```sh
|
||||
cargo run --bin john-slop -- \
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
//! Showcase opt-in replacements for every printable ASCII target.
|
||||
|
||||
use std::{collections::BTreeSet, io::Write};
|
||||
use std::io::Write;
|
||||
|
||||
use john_slop::{
|
||||
Config, ConfigValidationError, ReplacementRule, apply_replacements, plan_replacements,
|
||||
|
|
@ -10,11 +10,6 @@ const SOURCE: &str = include_str!("ascii-replacement-showcase/fullwidth.txt");
|
|||
|
||||
fn showcase_config() -> Result<Config, ConfigValidationError> {
|
||||
let mut config = Config::embedded();
|
||||
let embedded_sources = config
|
||||
.replacements
|
||||
.iter()
|
||||
.map(|rule| rule.source.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mappings = std::iter::once(('\u{3000}', ' ')).chain((b'!'..=b'~').map(|ascii| {
|
||||
// U+FF01..=U+FF5E is a contiguous block of valid Unicode scalar values.
|
||||
let fullwidth = char::from_u32(u32::from(ascii) + 0xfee0)
|
||||
|
|
@ -23,19 +18,29 @@ fn showcase_config() -> Result<Config, ConfigValidationError> {
|
|||
}));
|
||||
|
||||
for (source, target) in mappings {
|
||||
if embedded_sources.contains(&source.to_string()) {
|
||||
continue;
|
||||
let source = source.to_string();
|
||||
if let Some(rule) = config
|
||||
.replacements
|
||||
.iter_mut()
|
||||
.find(|rule| rule.source == source)
|
||||
{
|
||||
rule.replacement = target.to_string();
|
||||
rule.rationale = format!(
|
||||
"Override U+{:04X} for the complete printable-ASCII showcase.",
|
||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
||||
);
|
||||
} else {
|
||||
config.replacements.push(ReplacementRule {
|
||||
id: format!(
|
||||
"showcase-fullwidth-{:04x}",
|
||||
u32::from(source.chars().next().expect("source contains one scalar"))
|
||||
),
|
||||
source,
|
||||
replacement: target.to_string(),
|
||||
rationale: format!("Demonstrate an opt-in mapping to printable ASCII {target:?}."),
|
||||
enabled: true,
|
||||
});
|
||||
}
|
||||
config.replacements.push(ReplacementRule {
|
||||
id: format!("showcase-fullwidth-{:04x}", u32::from(source)),
|
||||
source: source.to_string(),
|
||||
replacement: target.to_string(),
|
||||
rationale: format!(
|
||||
"Demonstrate an opt-in mapping from U+{:04X} to printable ASCII.",
|
||||
u32::from(source)
|
||||
),
|
||||
enabled: true,
|
||||
});
|
||||
}
|
||||
|
||||
config.validate()?;
|
||||
|
|
@ -57,6 +62,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
|
||||
const EXPECTED: &str = include_str!("ascii-replacement-showcase/ascii.txt");
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ expected_replacements = 3
|
|||
path = "unicode-review/example.txt"
|
||||
formatted = "unicode-review/formatted.txt"
|
||||
expected_rules = ["unicode-ascii-replacement", "unicode-invisible-control"]
|
||||
expected_replacements = 1
|
||||
expected_replacements = 7
|
||||
|
||||
[[cases]]
|
||||
path = "source/example.rs"
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
The pаssword contains lookalike and invisible characters.
|
||||
Catalog sample: а𝐀!𝟘ⅿ🯹.
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
The password contains lookalike and invisible characters.
|
||||
Catalog sample: aA!Orn9.
|
||||
|
|
|
|||
|
|
@ -385,7 +385,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-core"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"directories",
|
||||
"figment",
|
||||
|
|
@ -411,7 +411,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-lsp"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"john-slop-core",
|
||||
"serde_json",
|
||||
|
|
@ -422,7 +422,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john_slop"
|
||||
version = "0.1.21"
|
||||
version = "0.1.22"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"globset",
|
||||
|
|
|
|||
10
tasks.md
10
tasks.md
|
|
@ -557,7 +557,7 @@ changelog, and the generated badge agree on `0.1.21`.
|
|||
load the catalog through every configuration path, and use its mappings for
|
||||
CLI fixes, formatter output, LSP quick fixes, and LSP document formatting.
|
||||
Preserve review-only behavior for invisible and bidirectional controls.
|
||||
- [ ] Expand fixtures and regression tests across letters, digits, punctuation,
|
||||
- [x] Expand fixtures and regression tests across letters, digits, punctuation,
|
||||
mathematical/fullwidth forms, multilingual risk cases, user overrides, and
|
||||
CLI/LSP/formatter parity. Update guides, evaluation sign-off, coverage,
|
||||
changelog, package version `0.1.22`, badges, and release metadata.
|
||||
|
|
@ -571,6 +571,14 @@ tested; invisible/bidi controls stay non-mutating; formatting remains
|
|||
idempotent; all quality, audit, package, and 90% coverage gates pass before the
|
||||
immutable `v0.1.22` tag is created.
|
||||
|
||||
Accepted on 2026-09-30 after the generated 1,829-entry catalog reproduced
|
||||
byte-for-byte, exhaustive Unicode drift validation passed, and CLI, standalone
|
||||
formatter, LSP quick-fix, and LSP-formatting tests produced identical mappings.
|
||||
The `replace`, `review`, and `off` policies and diagnostic-only controls are
|
||||
covered across both feature matrices. Formatting, Clippy, rustdoc, evaluation,
|
||||
dependency policy/audit checks, and workspace package assembly pass; coverage
|
||||
is 93.05% (1,487/1,598 lines), and release metadata agrees on `0.1.22`.
|
||||
|
||||
## Deferred by design
|
||||
|
||||
- Statistical or model-based authorship classification.
|
||||
|
|
|
|||
|
|
@ -126,20 +126,27 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
|||
&fs::read_to_string(root.join("signoff.toml")).expect("read evaluation sign-off"),
|
||||
)
|
||||
.expect("parse evaluation sign-off");
|
||||
let catalog_hash = Sha256::digest(include_bytes!(
|
||||
let mut catalog = Sha256::new();
|
||||
catalog.update(include_bytes!(
|
||||
"../crates/john-slop-core/assets/heuristics.toml"
|
||||
))
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>();
|
||||
));
|
||||
catalog.update([0]);
|
||||
catalog.update(include_bytes!(
|
||||
"../crates/john-slop-core/assets/unicode.toml"
|
||||
));
|
||||
let catalog_hash = catalog
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>();
|
||||
assert_eq!(signoff.schema_version, 1);
|
||||
assert_eq!(signoff.release, env!("CARGO_PKG_VERSION"));
|
||||
assert_eq!(signoff.status, "accepted-after-unicode-review-profile");
|
||||
assert_eq!(signoff.status, "accepted-after-unicode-replacement-profile");
|
||||
assert_eq!(signoff.catalog_sha256, catalog_hash);
|
||||
assert!(signoff.limitations_acknowledged);
|
||||
assert_eq!(signoff.reviewed_on, "2026-09-30");
|
||||
assert!(signoff.decision.contains("prompt-leakage"));
|
||||
assert!(signoff.decision.contains("0.90"));
|
||||
assert!(signoff.decision.contains("UTS #39"));
|
||||
assert!(signoff.decision.contains("non-mutating"));
|
||||
assert!(signoff.decision.contains("review"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
schema_version = 1
|
||||
release = "0.1.21"
|
||||
release = "0.1.22"
|
||||
reviewed_on = "2026-09-30"
|
||||
status = "accepted-after-unicode-review-profile"
|
||||
catalog_sha256 = "94cd36bf49f7061aad027308afb3bf8525a713af65b8ec3b730e0e75bfcef8c5"
|
||||
status = "accepted-after-unicode-replacement-profile"
|
||||
catalog_sha256 = "74be9257120a8830f546b3b6e87c1a73b4c55041cf1022c1fe568eb84ef30117"
|
||||
limitations_acknowledged = true
|
||||
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Add non-mutating Unicode 16.0 UTS #39 ASCII-skeleton and pinned Gitea invisible/control review rules with exact spans; safe replacements remain a separate allowlist. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
|
||||
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
|
||||
|
|
|
|||
Loading…
Reference in New Issue