Compare commits

...

4 Commits

39 changed files with 698 additions and 79 deletions

View File

@ -1,11 +1,11 @@
[last_run_clean]
run_status = "Success"
lines_parsed = 379
coverage = 92.34
lines_parsed = 389
coverage = 92.4
[last_run_clean.run_duration]
secs = 29
nanos = 803755166
nanos = 820282692
[last_run_incremental]
run_status = "Success"

View File

@ -6,6 +6,20 @@ All notable changes to John Slop are recorded here. The format follows
## [Unreleased]
## [0.1.21] - 2026-09-30
### Added
- Added real LSP document formatting backed by the same resolved replacement
plan as `john-slop-fmt`.
- Added non-mutating exact-span diagnostics for Unicode 16.0 UTS #39
ASCII-lookalikes and pinned Gitea invisible/bidirectional controls.
### Changed
- Expanded the shared fixtures and expected-failure diagnostic showcase with
deliberately non-fixable Unicode review cases.
## [0.1.20] - 2026-09-29
### Changed
@ -46,6 +60,7 @@ All notable changes to John Slop are recorded here. The format follows
- Stdio LSP replacement diagnostics and versioned quick fixes.
- Runnable examples, reviewed fixtures, fuzz targets, benchmarks, and CI gates.
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.20...HEAD
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.21...HEAD
[0.1.21]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.20...v0.1.21
[0.1.20]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.0...v0.1.20
[0.1.0]: https://git.dandaedre.com/idle-org/john_slop/releases/tag/v0.1.0

38
Cargo.lock generated
View File

@ -699,7 +699,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "john-slop-core"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"directories",
"figment",
@ -710,13 +710,14 @@ dependencies = [
"tempfile",
"thiserror",
"toml 1.1.6+spec-1.1.0",
"unicode-security",
"walkdir",
"zip",
]
[[package]]
name = "john-slop-lsp"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"john-slop-core",
"serde_json",
@ -727,7 +728,7 @@ dependencies = [
[[package]]
name = "john_slop"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"clap",
"criterion",
@ -1300,6 +1301,12 @@ dependencies = [
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
[[package]]
name = "tokio"
version = "1.53.1"
@ -1519,6 +1526,31 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-script"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"
[[package]]
name = "unicode-security"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e4ddba1535dd35ed8b61c52166b7155d7f4e4b8847cec6f48e71dc66d8b5e50"
dependencies = [
"unicode-normalization",
"unicode-script",
]
[[package]]
name = "unicode-width"
version = "0.1.14"

View File

@ -4,7 +4,7 @@ default-members = [".", "crates/john-slop-core", "crates/john-slop-lsp"]
resolver = "3"
[workspace.package]
version = "0.1.20"
version = "0.1.21"
edition = "2024"
rust-version = "1.98.1"
license = "MIT"
@ -18,8 +18,8 @@ figment = { version = "0.10.19", default-features = false, features = ["toml"] }
futures = { version = "0.3.34", default-features = false, features = ["std"] }
globset = { version = "0.4.20", default-features = false }
insta = { version = "1.48.0", default-features = false }
john-slop-core = { version = "0.1.20", path = "crates/john-slop-core" }
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.20", path = "crates/john-slop-lsp" }
john-slop-core = { version = "0.1.21", path = "crates/john-slop-core" }
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.21", path = "crates/john-slop-lsp" }
miette = { version = "7.6.0", default-features = false, features = ["derive"] }
regex = "1.13.1"
serde = { version = "1.0.229", features = ["derive"] }
@ -32,6 +32,7 @@ tokio = { version = "1.53.1", default-features = false }
toml = "1.1.6"
tower = { version = "0.5.3", default-features = false, features = ["util"] }
tower-lsp-server = { version = "0.23.0", default-features = false, features = ["runtime-tokio"] }
unicode-security = "0.1.2"
walkdir = "2.5.0"
zip = { version = "8.6.0", default-features = false, features = ["deflate-flate2-zlib-rs"] }

View File

@ -53,12 +53,13 @@ John Slop provides three binaries backed by the same configuration and source
spans:
- `john-slop` detects copied assistant scaffolding, generic style residue,
source placeholders, tutorial narration, and configured non-ASCII characters.
source placeholders, tutorial narration, configured replacements, ASCII-like
Unicode, and invisible or bidirectional controls.
- `john-slop-fmt` previews, checks, or explicitly applies safe configured ASCII
replacements.
- `john-slop-lsp` publishes the detector's heuristic diagnostics over stdio and
offers quick fixes for findings backed by safe replacements, without writing
editor-owned files.
- `john-slop-lsp` publishes the detector's heuristic diagnostics over stdio,
offers quick fixes for safe replacements, and provides formatter-equivalent
document edits without writing editor-owned files.
The detector accepts UTF-8 files, directories, ZIP archives, and common
ZIP-based office documents. Each finding includes its source span, stable rule

View File

@ -1 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.20"><title>version: 0.1.20</title><linearGradient id="bms-4fed1554fc5b0211" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-4fed1554fc5b0211"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-4fed1554fc5b0211)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-4fed1554fc5b0211)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.20</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.20</text></g></svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.21"><title>version: 0.1.21</title><linearGradient id="bms-3d708dc9f44ae746" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-3d708dc9f44ae746"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-3d708dc9f44ae746)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-3d708dc9f44ae746)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.21</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.21</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -15,6 +15,7 @@ miette.workspace = true
regex.workspace = true
serde.workspace = true
thiserror.workspace = true
unicode-security.workspace = true
walkdir.workspace = true
zip.workspace = true

View File

@ -182,3 +182,35 @@ enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-ascii-confusable"
category = "typography"
severity = "advice"
matcher = "confusable"
pattern = "uts39-ascii-skeleton-v16"
weight = 4.0
message = "a Unicode character visually resembles ASCII under UTS #39"
help = "Review whether the lookalike character is intentional; it is not automatically rewritten."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "unicode-invisible-control"
category = "typography"
severity = "warning"
matcher = "invisible"
pattern = "gitea-invisible-9c715c0"
weight = 12.0
message = "an invisible or bidirectional Unicode control appears in the source"
help = "Review the hidden or direction-changing character; it is not removed automatically."
occurrence_cap = 5
enabled = true
[rules.scope]
content = ["prose", "source-code"]
extensions = []

View File

@ -368,6 +368,18 @@ pub enum ConfigValidationError {
/// Heuristic rule ID.
id: String,
},
/// Confusable matchers must select the pinned UTS #39 profile.
#[error("confusable rule `{id}` must use pattern `uts39-ascii-skeleton-v16`")]
InvalidConfusableMatcher {
/// Heuristic rule ID.
id: String,
},
/// Invisible matchers must select the pinned Gitea profile.
#[error("invisible rule `{id}` must use pattern `gitea-invisible-9c715c0`")]
InvalidInvisibleMatcher {
/// Heuristic rule ID.
id: String,
},
/// A rule has no applicable content kinds.
#[error("rule `{id}` must apply to at least one content kind")]
EmptyScope {
@ -815,6 +827,10 @@ pub enum MatcherType {
RepeatedRegex,
/// Findings produced by the configured replacement catalog.
Replacement,
/// Non-mutating findings for UTS #39 characters with an ASCII skeleton.
Confusable,
/// Non-mutating findings for pinned invisible and bidi control characters.
Invisible,
}
/// Applicability filters attached to a heuristic rule.
@ -1027,6 +1043,18 @@ fn validate_rules(rules: &[HeuristicRule]) -> Result<(), ConfigValidationError>
});
}
MatcherType::Replacement => {}
MatcherType::Confusable if rule.pattern != "uts39-ascii-skeleton-v16" => {
return Err(ConfigValidationError::InvalidConfusableMatcher {
id: rule.id.clone(),
});
}
MatcherType::Confusable => {}
MatcherType::Invisible if rule.pattern != "gitea-invisible-9c715c0" => {
return Err(ConfigValidationError::InvalidInvisibleMatcher {
id: rule.id.clone(),
});
}
MatcherType::Invisible => {}
}
if rule.scope.content.is_empty() {
return Err(ConfigValidationError::EmptyScope {
@ -1149,7 +1177,7 @@ mod tests {
fn embedded_heuristics_have_complete_metadata() {
let config = Config::embedded();
assert_eq!(config.rules.len(), 11);
assert_eq!(config.rules.len(), 13);
assert_eq!(config.thresholds.low, 20);
assert_eq!(config.thresholds.medium, 50);
assert_eq!(config.thresholds.high, 75);
@ -1293,7 +1321,7 @@ weight = 9.5
assert_eq!(replacement.source, "—");
assert_eq!(replacement.replacement, "-");
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
assert_eq!(config.rules.len(), 11);
assert_eq!(config.rules.len(), 13);
let heuristic = config
.rules
.iter()
@ -1732,6 +1760,25 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
)
},
);
assert_invalid(
|config| {
config.rules[0].matcher = MatcherType::Confusable;
config.rules[0].pattern = "wrong".into();
},
|error| {
matches!(
error,
ConfigValidationError::InvalidConfusableMatcher { .. }
)
},
);
assert_invalid(
|config| {
config.rules[0].matcher = MatcherType::Invisible;
config.rules[0].pattern = "wrong".into();
},
|error| matches!(error, ConfigValidationError::InvalidInvisibleMatcher { .. }),
);
assert_invalid(
|config| config.rules[0].scope.content.clear(),
|error| matches!(error, ConfigValidationError::EmptyScope { .. }),

View File

@ -10,7 +10,7 @@ use regex::Regex;
use crate::{
Analysis, Category, Config, ConfigValidationError, ContentKind, EvidenceBand, EvidenceScore,
Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement,
Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement, unicode_review,
};
/// A validated heuristic definition paired with its reusable matcher.
@ -44,6 +44,8 @@ impl Rule {
CompiledMatcher::Regex(regex) => regex.as_str(),
CompiledMatcher::RepeatedRegex(regex) => regex.as_str(),
CompiledMatcher::Replacement => "configured-replacements",
CompiledMatcher::Confusable => "uts39-ascii-skeleton-v16",
CompiledMatcher::Invisible => "gitea-invisible-9c715c0",
}
}
}
@ -115,6 +117,8 @@ enum CompiledMatcher {
Regex(Regex),
RepeatedRegex(Regex),
Replacement,
Confusable,
Invisible,
}
#[derive(Debug, Clone)]
@ -265,6 +269,8 @@ impl Rule {
CompiledMatcher::RepeatedRegex(regex)
}
MatcherType::Replacement => CompiledMatcher::Replacement,
MatcherType::Confusable => CompiledMatcher::Confusable,
MatcherType::Invisible => CompiledMatcher::Invisible,
};
let scope = CompiledScope::compile(&definition)?;
Ok(Self {
@ -341,6 +347,47 @@ impl Rule {
.into_iter()
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
.collect(),
CompiledMatcher::Confusable => source
.char_indices()
.filter_map(|(start, character)| {
let matched = character.to_string();
if config
.replacements
.iter()
.any(|rule| rule.enabled && rule.source == matched)
{
return None;
}
let skeleton = unicode_review::ascii_skeleton(character)?;
let mut finding =
self.finding(source, start..start + character.len_utf8(), None);
finding.help = Some(format!(
"Review U+{:04X}; its Unicode {}.{}.{} UTS #39 skeleton is {skeleton:?}. This review finding is not an automatic replacement.",
character as u32,
unicode_review::UTS39_UNICODE_VERSION.0,
unicode_review::UTS39_UNICODE_VERSION.1,
unicode_review::UTS39_UNICODE_VERSION.2,
));
Some(finding)
})
.collect(),
CompiledMatcher::Invisible => source
.char_indices()
.filter(|(start, character)| {
unicode_review::is_gitea_invisible(*character)
&& !(*character == '\r'
&& source.as_bytes().get(start + 1) == Some(&b'\n'))
})
.map(|(start, character)| {
let mut finding =
self.finding(source, start..start + character.len_utf8(), None);
finding.help = Some(format!(
"Review the hidden or direction-changing character U+{:04X}. It is never removed automatically.",
character as u32
));
finding
})
.collect(),
}
}
@ -527,6 +574,39 @@ mod tests {
assert_eq!(analysis.evidence_band, Some(EvidenceBand::BelowThreshold));
}
#[test]
fn unicode_review_findings_have_exact_spans_and_never_become_fixes() {
let detector = Detector::new(&Config::embedded()).unwrap();
let source = "é pаss\u{200b}word\u{202e}\r\nnext";
let findings = detector.analyze(source).findings;
assert_eq!(findings.len(), 3);
assert_eq!(findings[0].rule_id.as_str(), "unicode-ascii-confusable");
assert_eq!(findings[0].matched_text, "а");
assert_eq!(&source[findings[0].byte_span.clone()], "а");
assert!(findings[0].help.as_deref().unwrap().contains("\"a\""));
assert!(findings.iter().all(|finding| finding.fix.is_none()));
assert_eq!(
findings
.iter()
.filter(|finding| finding.rule_id.as_str() == "unicode-invisible-control")
.count(),
2
);
assert!(findings.iter().all(|finding| finding.matched_text != "\r"));
}
#[test]
fn safe_replacement_characters_are_not_duplicated_as_confusable_findings() {
let findings = Detector::new(&Config::embedded())
.unwrap()
.analyze("one—two")
.findings;
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].rule_id.as_str(), "typography-non-ascii-hyphen");
}
#[test]
fn compiled_regex_uses_utf8_byte_offsets() {
let regex = Regex::new(r"\u{1D306}").unwrap();

View File

@ -12,6 +12,7 @@ mod error;
mod input;
mod model;
mod replacement;
mod unicode_review;
pub use config::{
Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError,

View File

@ -0,0 +1,77 @@
//! Pinned Unicode review profiles that never authorize automatic rewriting.
/// Unicode version carried by `unicode-security` 0.1.2.
pub(crate) const UTS39_UNICODE_VERSION: (u64, u64, u64) = unicode_security::UNICODE_VERSION;
/// Return an ASCII UTS #39 skeleton when one non-ASCII scalar resembles ASCII.
pub(crate) fn ascii_skeleton(character: char) -> Option<String> {
if character.is_ascii() {
return None;
}
let source = character.to_string();
let skeleton = unicode_security::skeleton(&source).collect::<String>();
(!skeleton.is_empty() && skeleton.is_ascii() && skeleton != source).then_some(skeleton)
}
/// Match the invisible/control profile generated by Gitea at commit 9c715c0.
///
/// Gitea removes ordinary space, tab, and line feed before generating its
/// table. Callers remain responsible for treating CRLF as one ordinary line
/// ending rather than a suspicious standalone carriage return.
pub(crate) const fn is_gitea_invisible(character: char) -> bool {
matches!(
character as u32,
11..=13
| 127
| 160
| 173
| 847
| 1564
| 4447
| 4448
| 6068
| 6069
| 6155..=6158
| 7355..=7356
| 8192..=8207
| 8234..=8239
| 8287..=8303
| 10240
| 12288
| 12644
| 65024..=65039
| 65279
| 65440
| 65520..=65528
| 65532
| 78844
| 119155..=119162
| 917504..=917631
| 917760..=917999
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uts39_profile_reports_ascii_lookalikes_but_not_ascii_or_accents() {
assert_eq!(ascii_skeleton('а').as_deref(), Some("a"));
assert_eq!(ascii_skeleton('𝟘').as_deref(), Some("O"));
assert_eq!(ascii_skeleton('a'), None);
assert_eq!(ascii_skeleton('é'), None);
assert_eq!(UTS39_UNICODE_VERSION, (16, 0, 0));
}
#[test]
fn gitea_profile_covers_invisible_and_bidi_ranges_without_normal_whitespace() {
assert!(is_gitea_invisible('\u{200b}'));
assert!(is_gitea_invisible('\u{202e}'));
assert!(is_gitea_invisible('\u{2066}'));
assert!(is_gitea_invisible('\u{e0001}'));
assert!(!is_gitea_invisible(' '));
assert!(!is_gitea_invisible('\t'));
assert!(!is_gitea_invisible('\n'));
}
}

View File

@ -10,8 +10,8 @@ use tower_lsp_server::{
CodeAction, CodeActionKind, CodeActionOptions, CodeActionOrCommand, CodeActionParams,
CodeActionProviderCapability, CodeActionResponse, Diagnostic, DiagnosticSeverity,
DidChangeConfigurationParams, DidChangeTextDocumentParams, DidCloseTextDocumentParams,
DidOpenTextDocumentParams, DocumentChanges, InitializeParams, InitializeResult,
InitializedParams, MessageType, NumberOrString, OneOf,
DidOpenTextDocumentParams, DocumentChanges, DocumentFormattingParams, InitializeParams,
InitializeResult, InitializedParams, MessageType, NumberOrString, OneOf,
OptionalVersionedTextDocumentIdentifier, Position, Range, ServerCapabilities, ServerInfo,
TextDocumentEdit, TextDocumentSyncCapability, TextDocumentSyncKind,
TextEdit as LspTextEdit, Uri, WorkspaceEdit,
@ -250,6 +250,19 @@ fn replacement_code_actions(
.collect()
}
fn formatting_edits(config: &Config, source: &str) -> Result<Vec<LspTextEdit>, PositionError> {
let index = LineIndex::new(source);
plan_replacements(config, source)
.into_iter()
.map(|edit| {
Ok(LspTextEdit::new(
index.range(edit.byte_span)?,
edit.replacement,
))
})
.collect()
}
fn ranges_overlap(left: &Range, right: &Range) -> bool {
left.start <= right.end && right.start <= left.end
}
@ -472,6 +485,7 @@ impl LanguageServer for LspBackend {
..CodeActionOptions::default()
},
)),
document_formatting_provider: Some(OneOf::Left(true)),
..ServerCapabilities::default()
},
offset_encoding: None,
@ -570,6 +584,20 @@ impl LanguageServer for LspBackend {
.map_err(|error| JsonError::invalid_params(error.to_string()))?;
Ok(Some(actions))
}
async fn formatting(
&self,
params: DocumentFormattingParams,
) -> JsonResult<Option<Vec<LspTextEdit>>> {
let uri = params.text_document.uri;
let Some(snapshot) = self.documents.read().await.get(&uri) else {
return Ok(None);
};
let config = self.runtime.read().await.config.clone();
formatting_edits(&config, &snapshot.text)
.map(Some)
.map_err(|error| JsonError::invalid_params(error.to_string()))
}
}
#[cfg(test)]

View File

@ -43,9 +43,11 @@ The modules within those crates retain deliberately narrow responsibilities:
diagnostics;
- `replacement` produces non-overlapping edits and validates/applies arbitrary
edit sets without performing I/O;
- `unicode_review` pins the UTS #39 ASCII-skeleton version and Gitea-compatible
invisible/control ranges used only by non-mutating detector matchers;
- `lsp` indexes open snapshots, runs the compiled detector, converts canonical
finding spans to UTF-16 coordinates, and implements versioned replacement
quick fixes;
finding and replacement spans to UTF-16 coordinates, and implements
versioned quick fixes plus whole-document formatting;
- `model` contains presentation-neutral domain values;
- `error` exposes typed input failures for library consumers;
- `src/bin` contains the detector presentation/exit policy and formatter/LSP
@ -92,7 +94,8 @@ documents in deterministic URI order. The detector is compiled once per
resolved configuration rather than once per edit. File URI paths drive normal
scope applicability; language IDs supply source/prose context for untitled or
extensionless buffers. All findings become diagnostics, while only findings
with core `TextEdit`s become code actions. See
with core `TextEdit`s become code actions. Document formatting calls the same
replacement planner directly and never derives mutations from diagnostics. See
[ADR 0003](decisions/0003-lsp-framework-and-protocol.md) for framework and
protocol choices.

View File

@ -59,6 +59,14 @@ layer, invalid regular expressions or globs, unsafe matchers, non-ASCII
replacement targets, and invalid thresholds are fatal configuration
diagnostics.
The built-in matcher names are `exact-phrase`, `regex`, `repeated-regex`,
`replacement`, `confusable`, and `invisible`. The last three use validated
sentinel patterns because their implementations are data-backed rather than
free-form strings: `configured-replacements`, `uts39-ascii-skeleton-v16`, and
`gitea-invisible-9c715c0`. They remain ordinary rule records, so later layers
can disable them or adjust severity, weight, occurrence cap, and scope without
changing code. Confusable and invisible findings never authorize edits.
Set `enabled = false` on a known rule or replacement ID to disable it. Because
records merge by ID, do not copy the full embedded table merely to change one
field; a later partial record preserves every unspecified value.

View File

@ -76,3 +76,15 @@ This amendment does not broaden mutation behavior:
The original full-snapshot synchronization, UTF-16 conversion, version checks,
stdio purity, no-write boundary, and stale-action guarantees remain unchanged.
## Amendment: document formatting parity
Milestone 21 adds the standard `textDocument/formatting` capability. The server
plans edits from its current open-document snapshot with the same resolved
configuration and core replacement planner used by `john-slop-fmt`, then
converts only the edit coordinates from UTF-8 bytes to LSP UTF-16 positions.
Formatting remains limited to configured safe ASCII replacements; the server
does not write files, infer formatting from diagnostics, or apply broader
Unicode review findings. Protocol tests compare the result of an actual
formatting request with the standalone formatter contract, including Unicode,
CRLF, no-op, and configuration-reload cases.

View File

@ -9,7 +9,8 @@ The normal dependency graph contains 135 distinct package names. Direct crates
serve narrow boundaries: Clap parses adapters, Figment/Serde/TOML load typed
configuration, Regex/Globset compile rules and scopes, Miette renders
diagnostics, Walkdir/Zip acquire inputs, Tokio/tower-lsp-server implement the
stdio protocol, and Tempfile/Similar support atomic writes and diffs.
stdio protocol, Unicode Security supplies the pinned UTS #39 skeleton data, and
Tempfile/Similar support atomic writes and diffs.
Default features are disabled where they would add unrelated behavior. In
particular, Zip previously enabled AES, Bzip2, Deflate64, LZMA, PPMd, Zstandard,
@ -30,7 +31,7 @@ boundary and the synchronized package metadata.
## Security, licenses, and unsafe code
`cargo audit --deny warnings` scanned 187 locked packages against 1,277 RustSec
`cargo audit --deny warnings` scanned 193 locked packages against 1,277 RustSec
advisories without a finding. `cargo deny --locked check` passed advisories,
licenses, and sources; it reports seven transitive duplicate-version families
as warnings for ongoing review. [`deny.toml`](../deny.toml) contains the reviewed

View File

@ -67,8 +67,8 @@ Run line coverage with Cargo Tarpaulin:
cargo tarpaulin --config tarpaulin.toml
```
The configuration enforces a 90% floor. The current suite measured 92.34%
(1,411/1,528 lines) on 2026-09-29. The remaining lines are dominated by
The configuration enforces a 90% floor. The current suite measured 92.40%
(1,472/1,593 lines) on 2026-09-30. The remaining lines are dominated by
platform/I/O failure injection, malformed archive internals, coordinate
overflow, LSP logging/reload failures, and subprocess attribution. Do not add
brittle tests or production seams solely to make Tarpaulin count those paths.

View File

@ -3,7 +3,9 @@
Both `john-slop-fmt` and `john-slop-lsp` are operational. The LSP publishes all
applicable configured heuristic findings, including prompt leakage, repeated
transitions, source placeholders, tutorial narration, and typography. Safe
ASCII-replacement findings additionally provide versioned quick fixes.
ASCII-replacement findings additionally provide versioned quick fixes, and the
server implements whole-document formatting with the exact same replacement
planner as `john-slop-fmt`.
Install all three binaries somewhere on `PATH`, then confirm the editor-facing
help is available:
@ -34,9 +36,12 @@ auto-format = false
```
This complete custom language entry makes the repository's `example.txt`
fixtures useful without depending on a built-in plain-text grammar. Use
`:format` to run the formatter. Set `auto-format = true` only if save-time
replacement is wanted.
fixtures useful without depending on a built-in plain-text grammar. It selects
the external `john-slop-fmt` adapter. To exercise formatting through the LSP
instead, omit the `formatter = ...` line; `:format` then requests
`textDocument/formatting` from `john-slop-lsp`. Both routes resolve the same
configuration and produce the same text. Set `auto-format = true` only if
save-time replacement is wanted.
For an existing language, merge John Slop into that language's entry rather
than replacing its useful defaults. For example, Rust should retain
@ -49,6 +54,11 @@ formatter = { command = "john-slop-fmt" }
language-servers = ["rust-analyzer", "john-slop"]
```
When several language servers advertise formatting, retain an explicit Helix
`formatter` or select the intended server interactively; John Slop must not
silently replace language-specific formatting from `rust-analyzer` or another
primary server.
Run `hx --health rust` (or the selected language name) to confirm both binaries
are visible. Helix documents that a formatter reads the buffer from stdin and
writes it to stdout, while an LSP command must be on `PATH`; those are the
@ -91,6 +101,11 @@ commands to inspect server state, move through heuristic diagnostics, and apply
available ASCII quick fixes. The registration shape and `allowlist` follow the
plugin's current upstream documentation.
Clients that expose LSP document formatting can request it directly from John
Slop instead of configuring `formatprg`; both paths use the same replacement
catalog. Keep `formatprg=john-slop-fmt` when the Vim client cannot select one
formatter among multiple attached language servers.
The server uses the same configuration layers as the other binaries. Add
`args = ["--config-name", "editorial"]` to the Helix language-server table, or
append those arguments to the Vim `cmd` list, when an editor-specific named

View File

@ -20,8 +20,10 @@ With the embedded schema-version 1 catalog, the corpus currently reports:
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
| typography-non-ascii-hyphen | 1 | 0 | 0 | 100% | 100% |
| unicode-ascii-confusable | 1 | 0 | 0 | 100% | 100% |
| unicode-invisible-control | 1 | 0 | 0 | 100% | 100% |
Aggregate labeled-rule counts are 11 true positives, 0 false positives, and 0
Aggregate labeled-rule counts are 13 true positives, 0 false positives, and 0
false negatives. The clean-document false-positive rate is 0/5 (0%). The test
suite recomputes these values from the manifest and fails on drift.
@ -34,6 +36,7 @@ The same manifest pins evidence scores for regression characterization:
| style signals | 19 |
| source-code signals | 24 |
| typography signal | 17 |
| one ASCII lookalike and one invisible character | 21 |
| each of five negative documents | 0 |
These perfect values describe a tiny purpose-built regression set, not expected
@ -45,11 +48,12 @@ are required before tuning default weights or thresholds.
Each shipped rule has at least one explicitly labeled realistic near miss. The
initial 0.1.0 review kept all schema-version 1 rules, weights, and thresholds
unchanged. The renewed review, carried forward by the unchanged catalog hash to
0.1.20, accepts lighter `0.90ⁿ` correlation damping for distinct prompt-leakage
rules while retaining `0.75ⁿ` for other categories. The checked-in `signoff.toml`
records that decision, acknowledges the corpus limitations, and contains the
unchanged SHA-256 digest of
unchanged. The prompt-leakage review accepted lighter `0.90ⁿ` correlation
damping for distinct prompt-leakage rules while retaining `0.75ⁿ` for other
categories. The Milestone 21 review adds the non-mutating UTS #39 ASCII-skeleton
and Gitea-compatible invisible/control profiles, with explicit multilingual
limitations. The checked-in `signoff.toml` records those decisions and contains
the SHA-256 digest of
`crates/john-slop-core/assets/heuristics.toml`.
Any default rule, matcher, pattern, weight, occurrence cap, scope, or threshold

View File

@ -5,11 +5,18 @@ Every contribution to the score is represented by a finding with an exact
source span, stable rule ID, category, message, help, and configured weight.
The detector compiles the resolved rule catalog once. Exact phrases, regular
expressions, repeated-pattern rules, and configured Unicode replacements share
the same finding model. Content kind, extension, path globs, and conventional
generated/vendor/minified paths decide whether each rule applies. Findings are
deduplicated by rule and byte span, then sorted by start offset, end offset, and
rule ID so every adapter observes the same order.
expressions, repeated-pattern rules, configured Unicode replacements, UTS #39
ASCII lookalikes, and invisible/control characters share the same finding
model. Content kind, extension, path globs, and conventional generated/vendor/
minified paths decide whether each rule applies. Findings are deduplicated by
rule and byte span, then sorted by start offset, end offset, and rule ID so
every adapter observes the same order.
Unicode review rules are deliberately distinct from replacement rules. The
detector can flag a lookalike, invisible scalar, or direction-changing control
at its exact span, but only the explicit ten-entry safe replacement allowlist
can produce a formatter edit or LSP quick fix. A safe replacement finding is
not duplicated as a confusable finding.
## Evidence-score formula

View File

@ -11,8 +11,10 @@ Legitimate material can contain every shipped signal. Documentation may quote a
chat transcript, an article may discuss knowledge cutoffs, a tutorial may use
step-by-step comments deliberately, and a tracked source-code TODO may be valid.
Unicode punctuation can be correct typography even when a downstream format
requires ASCII. Generic transitions are weak style evidence, not defects by
themselves.
requires ASCII. The UTS #39 ASCII-skeleton profile intentionally flags many
legitimate Greek, Cyrillic, mathematical, and other non-Latin characters for
review because it has no locale or identifier context. Generic transitions are
weak style evidence, not defects by themselves.
Use path scopes, rule disabling, and project-specific weights to reflect local
policy. Do not automatically reject, shame, or attribute a document based on a
@ -39,7 +41,9 @@ specialized editorial styles, historical corpora, transcripts, generated API
documentation, notebooks, and uncommon programming languages are
underrepresented. File-extension inference covers a conservative common set and
defaults unknown paths to prose. Teams should validate rules against their own
reviewed material before changing enforcement thresholds.
reviewed material before changing enforcement thresholds. Disable or narrow
`unicode-ascii-confusable` when ordinary multilingual text makes character-level
review too noisy; locale-aware suppression is not implemented.
## Evidence score is not probability
@ -77,8 +81,10 @@ synchronization. It publishes the configured heuristic catalog but does not
publish a document-level evidence score; editor diagnostics remain individual
review signals. It does not read or write a document URI, mutate archives,
provide prose rewrites, or implement incremental range changes. Only configured
ASCII replacements receive quick fixes, and configuration reload errors retain
the last valid configuration and compiled detector.
ASCII replacements receive quick fixes or formatting edits. Confusable,
invisible, and bidirectional-control findings are diagnostic-only, and
configuration reload errors retain the last valid configuration and compiled
detector.
Analysis is local and performs no network lookup. Consequently, it cannot check
citations, current facts, provenance, licensing, or similarity to external

View File

@ -27,7 +27,7 @@ data file. GitHub separately documents a warning for hidden bidirectional text;
that protection is about bidi controls rather than a published Gitea-style
locale-aware lookalike table.
## Implication for John Slop
## Implemented John Slop profile
These sources are useful detection references, not safe formatter replacement
catalogs. Many listed characters are legitimate in particular languages and
@ -35,12 +35,25 @@ scripts, and Gitea deliberately interprets them through the reviewer's locale.
Importing every pair as an automatic Unicode-to-ASCII edit would corrupt valid
text and violate the formatter's conservative replacement contract.
A future implementation should therefore keep three policies separate:
John Slop keeps three policies separate:
1. configured, explicitly safe formatter replacements;
2. locale-aware confusable findings for human review; and
3. invisible and bidirectional-control security findings.
2. non-mutating confusable findings for human review; and
3. non-mutating invisible and bidirectional-control findings.
Any vendored or generated catalog should pin its upstream Unicode/data version,
retain provenance and licensing, test locale behavior and ordinary multilingual
near-misses, and report exact source spans without silently rewriting text.
`unicode-ascii-confusable` uses `unicode-security` 0.1.2, whose checked-in UTS
#39 tables target Unicode 16.0. It reports a non-ASCII scalar only when its
single-character skeleton is entirely ASCII. It is intentionally broader than
Gitea's locale selection and can flag legitimate scripts; locale-aware policy
remains deferred. Characters already covered by the safe replacement allowlist
produce only the fixable replacement finding, not a duplicate confusable one.
`unicode-invisible-control` uses the invisible ranges generated by Gitea at the
pinned commit above, including bidi embedding/override and isolate controls.
Ordinary space, tab, line feed, and the carriage return in a CRLF pair are not
reported. Each other match receives an exact UTF-8 source span that adapters
convert normally. Neither rule feeds the formatter or LSP quick fixes.
Future profile updates must pin their upstream Unicode/data version, retain
provenance and licensing, test ordinary multilingual near misses, renew the
hash-bound default-policy sign-off, and preserve the no-silent-rewrite boundary.

View File

@ -28,8 +28,11 @@ layering should call `john_slop::load_config()` instead.
[`fixtures/manifest.toml`](fixtures/manifest.toml) describes small inputs shared
by detector CLI, formatter, and LSP testing. Several directories deliberately
contain an `example.txt`, covering clean prose, copied assistant residue, stock
style, and Unicode punctuation. A Rust source example covers source-only rules.
Each input has a checked-in expected formatted result.
style, safe Unicode punctuation replacements, and diagnostic-only Unicode
lookalikes/invisibles. A Rust source example covers source-only rules. Each
input has a checked-in expected formatted result; the Unicode review fixture's
expected result is intentionally byte-identical because those findings do not
authorize rewriting.
Exercise the currently implemented boundaries with:
@ -71,10 +74,11 @@ embedded catalog remains limited to individually reviewed conservative rules.
shipped prose signal, while
[`diagnostic-showcase/source.rs`](diagnostic-showcase/source.rs) adds the
source-placeholder and tutorial-narration rules plus all ten embedded
replacement characters. The adjacent example configuration promotes assistant
replacement characters, a Cyrillic ASCII lookalike, a zero-width space, and a
right-to-left override. The adjacent example configuration promotes assistant
self-identification from warning to error and lifts the typography occurrence
cap for this display, making one run show Miette advice, warning, error, and
fixable diagnostics together:
cap for this display, making one run show Miette advice, warning, error,
fixable, and deliberately non-fixable Unicode diagnostics together:
```sh
cargo run --bin john-slop -- \

View File

@ -3,4 +3,5 @@ fn main() {
// Next, we leave the implementation placeholder visible for review.
// TODO: implement
let _replacement_characters = "‐ ‑ ‒ – — ― − ﹘ ﹣ -";
let _unicode_review_only = "pаss​word‮";
}

View File

@ -34,6 +34,12 @@ formatted = "typography/formatted.txt"
expected_rules = ["typography-non-ascii-hyphen"]
expected_replacements = 3
[[cases]]
path = "unicode-review/example.txt"
formatted = "unicode-review/formatted.txt"
expected_rules = ["unicode-ascii-confusable", "unicode-invisible-control"]
expected_replacements = 0
[[cases]]
path = "source/example.rs"
formatted = "source/formatted.rs"

View File

@ -0,0 +1 @@
The pаss​word contains lookalike and invisible characters.

View File

@ -0,0 +1 @@
The pаss​word contains lookalike and invisible characters.

38
fuzz/Cargo.lock generated
View File

@ -385,7 +385,7 @@ dependencies = [
[[package]]
name = "john-slop-core"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"directories",
"figment",
@ -394,6 +394,7 @@ dependencies = [
"regex",
"serde",
"thiserror",
"unicode-security",
"walkdir",
"zip",
]
@ -410,7 +411,7 @@ dependencies = [
[[package]]
name = "john-slop-lsp"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"john-slop-core",
"serde_json",
@ -421,7 +422,7 @@ dependencies = [
[[package]]
name = "john_slop"
version = "0.1.20"
version = "0.1.21"
dependencies = [
"clap",
"globset",
@ -828,6 +829,12 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "tinyvec"
version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
[[package]]
name = "tokio"
version = "1.53.1"
@ -1041,6 +1048,31 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-script"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"
[[package]]
name = "unicode-security"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e4ddba1535dd35ed8b61c52166b7155d7f4e4b8847cec6f48e71dc66d8b5e50"
dependencies = [
"unicode-normalization",
"unicode-script",
]
[[package]]
name = "unicode-width"
version = "0.1.14"

View File

@ -13,6 +13,9 @@ pub const EDITOR_SETUP: &str = r#"EDITOR SETUP (formatter and heuristic LSP avai
formatter = { command = "john-slop-fmt" }
language-servers = ["john-slop"]
Omit the formatter line to send :format through john-slop-lsp instead;
both routes use the same configured replacement plan.
Vim ~/.vimrc formatter (use gggqG for the whole buffer):
autocmd BufRead,BufNewFile *.txt setfiletype text
autocmd FileType text,markdown,rust setlocal formatprg=john-slop-fmt

View File

@ -515,14 +515,45 @@ badge drift. Formatting, Clippy, rustdoc, both feature matrices, pre-publication
package assembly, and Tarpaulin passed; coverage remains 92.34%
(1,411/1,528 lines).
## Milestone 21 — LSP formatter parity and Unicode review diagnostics
- [x] Implement and advertise `textDocument/formatting` in the LSP using the
same resolved configuration and core replacement planner as `john-slop-fmt`.
Return deterministic UTF-16 edits against the current editor snapshot, cover
Unicode/CRLF/no-op/custom-config/reload cases, and replace the misleading
quick-fix-only equivalence assertion with an actual protocol request.
- [x] Add non-fixing review diagnostics for Unicode characters that resemble
ASCII under a pinned Unicode UTS #39 profile, plus Gitea-compatible invisible
and bidirectional controls. Keep these findings distinct from the smaller
automatic replacement allowlist, configurable as normal heuristic rules, and
document provenance, multilingual false positives, and profile boundaries.
- [x] Refresh examples, editor/architecture/configuration/limitations guides,
evaluation fixtures and sign-off, coverage metadata, package version
`0.1.21`, changelog, and generated badges.
Acceptance: a real LSP formatting request produces exactly the same text as
`john-slop-fmt` for the same source and layered configuration; formatter output
contains no broader confusable rewrite; representative ASCII-lookalikes,
invisible characters, and bidi controls receive exact-span CLI and LSP
diagnostics without quick fixes; ordinary ASCII and reviewed multilingual near
misses remain covered; formatting, Clippy, rustdoc, both feature matrices,
package assembly, and the coverage floor pass before the immutable `v0.1.21`
tag is created.
Accepted on 2026-09-30 after real protocol-formatting parity tests, exact-span
CLI/LSP Unicode diagnostics, a non-mutating shared fixture, and the renewed
13-rule evaluation corpus passed. Formatting, Clippy, rustdoc, both feature
matrices, dependency policy/audit checks, and workspace package assembly pass;
coverage is 92.40% (1,472/1,593 lines), and all packages, binaries, lockfiles,
changelog, and the generated badge agree on `0.1.21`.
## Deferred by design
- Statistical or model-based authorship classification.
- Automatic prose or source-code rewriting beyond configured ASCII replacements.
- Locale-aware confusable, invisible, and bidirectional Unicode detection. The
researched catalogs and adoption constraints are recorded in
`docs/unicode-confusables.md`; detection must remain separate from automatic
replacement.
- Locale-aware confusable policy beyond the pinned review profile. Broader
locale selection still requires product validation; detection remains
separate from automatic replacement.
- In-place modification of archive members or office documents.
- Network-backed analysis or telemetry.
- Editor-specific extensions; the LSP is the integration boundary.

View File

@ -195,6 +195,31 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
assert!(!String::from_utf8_lossy(&output.stdout).contains('\u{1b}'));
}
#[test]
fn unicode_review_diagnostics_are_non_fixing_and_exact() {
let source = "pаss\u{200b}word";
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
assert!(output.status.success());
let record = &json_records(&output)[0];
assert_eq!(record["summary"]["total"], 2);
assert_eq!(record["findings"][0]["rule_id"], "unicode-ascii-confusable");
assert_eq!(record["findings"][0]["matched_text"], "а");
assert!(record["findings"][0]["fix"].is_null());
assert_eq!(
record["findings"][1]["rule_id"],
"unicode-invisible-control"
);
assert_eq!(
&source[record["findings"][1]["byte_span"]["start"]
.as_u64()
.unwrap() as usize
..record["findings"][1]["byte_span"]["end"].as_u64().unwrap() as usize],
"\u{200b}"
);
assert!(record["findings"][1]["fix"].is_null());
}
#[test]
fn stdin_filename_controls_source_code_scope() {
let text = b"// TODO: implement the parser";

View File

@ -106,7 +106,7 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
);
assert_eq!((total_tp, total_fp, total_fn), (11, 0, 0));
assert_eq!((total_tp, total_fp, total_fn), (13, 0, 0));
assert_eq!((false_positive_cases, negative_cases), (0, 5));
assert_eq!(
reviewed_near_misses,
@ -129,10 +129,12 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
.collect::<String>();
assert_eq!(signoff.schema_version, 1);
assert_eq!(signoff.release, env!("CARGO_PKG_VERSION"));
assert_eq!(signoff.status, "accepted-after-prompt-damping-review");
assert_eq!(signoff.status, "accepted-after-unicode-review-profile");
assert_eq!(signoff.catalog_sha256, catalog_hash);
assert!(signoff.limitations_acknowledged);
assert_eq!(signoff.reviewed_on, "2026-09-29");
assert_eq!(signoff.reviewed_on, "2026-09-30");
assert!(signoff.decision.contains("prompt-leakage"));
assert!(signoff.decision.contains("0.90"));
assert!(signoff.decision.contains("UTS #39"));
assert!(signoff.decision.contains("non-mutating"));
}

View File

@ -160,6 +160,8 @@ fn diagnostic_showcase_spans_prose_source_and_severities() {
assert!(stdout.contains("assistant-self-identification"));
assert!(stdout.contains("prompt-response-scaffolding"));
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("unicode-ascii-confusable"));
assert!(stdout.contains("unicode-invisible-control"));
assert!(stdout.contains("source-placeholder-todo"));
assert!(stdout.contains("source-tutorial-narration"));
assert!(stdout.contains('‐'));

View File

@ -36,12 +36,22 @@ kind = "prose"
expected_score = 17
expected = ["typography-non-ascii-hyphen"]
[[cases]]
path = "unicode.md"
kind = "prose"
expected_score = 21
expected = ["unicode-ascii-confusable", "unicode-invisible-control"]
[[cases]]
path = "negative.md"
kind = "prose"
expected_score = 0
expected = []
near_miss_for = ["typography-non-ascii-hyphen"]
near_miss_for = [
"typography-non-ascii-hyphen",
"unicode-ascii-confusable",
"unicode-invisible-control",
]
[[cases]]
path = "negative-assistant.md"

View File

@ -1,7 +1,7 @@
schema_version = 1
release = "0.1.20"
reviewed_on = "2026-09-29"
status = "accepted-after-prompt-damping-review"
catalog_sha256 = "8617e56c1f874307fd967ef688c37d42efa750bfb833eb364b797dc588ce1fb0"
release = "0.1.21"
reviewed_on = "2026-09-30"
status = "accepted-after-unicode-review-profile"
catalog_sha256 = "94cd36bf49f7061aad027308afb3bf8525a713af65b8ec3b730e0e75bfcef8c5"
limitations_acknowledged = true
decision = "Accept category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Rules, weights, thresholds, and the catalog hash remain unchanged. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world accuracy."
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Add non-mutating Unicode 16.0 UTS #39 ASCII-skeleton and pinned Gitea invisible/control review rules with exact spans; safe replacements remain a separate allowlist. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."

1
tests/fixtures/evaluation/unicode.md vendored Normal file
View File

@ -0,0 +1 @@
The pаss​word contains a hidden lookalike.

View File

@ -98,6 +98,18 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-ascii-confusable",
positive: "The pаssword contains a Cyrillic lookalike.",
near_miss: "Résumé précis en français.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-invisible-control",
positive: "A zero-width\u{200b}space is hidden here.",
near_miss: "Ordinary spaces and\t tabs are visible structure.",
kind: FixtureKind::Prose,
},
];
assert_eq!(cases.len(), detector.rules().len());

View File

@ -54,6 +54,10 @@ impl Harness {
response.result().unwrap()["capabilities"]["textDocumentSync"],
1
);
assert_eq!(
response.result().unwrap()["capabilities"]["documentFormattingProvider"],
true
);
harness.notify("initialized", json!({})).await;
harness
}
@ -213,6 +217,53 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() {
);
}
#[tokio::test]
async fn unicode_review_findings_are_diagnostic_only_and_not_formatted() {
let mut harness = Harness::new(None).await;
let uri = "file:///tmp/unicode-review.txt";
let source = "pаss\u{200b}word\u{202e}";
harness
.notify("textDocument/didOpen", open_params(uri, 1, source))
.await;
let published = harness
.notification("textDocument/publishDiagnostics")
.await;
let diagnostics = published["diagnostics"].as_array().unwrap();
assert_eq!(diagnostics.len(), 3);
assert_eq!(diagnostics[0]["code"], "unicode-ascii-confusable");
assert_eq!(diagnostics[0]["data"]["fixable"], false);
assert_eq!(diagnostics[1]["code"], "unicode-invisible-control");
assert_eq!(diagnostics[2]["code"], "unicode-invisible-control");
let formatting = harness
.request(
"textDocument/formatting",
json!({
"textDocument": { "uri": uri },
"options": { "tabSize": 4, "insertSpaces": true }
}),
40,
)
.await;
assert!(formatting.result().unwrap().as_array().unwrap().is_empty());
let actions = harness
.request(
"textDocument/codeAction",
json!({
"textDocument": { "uri": uri },
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 12 }
},
"context": { "diagnostics": diagnostics }
}),
41,
)
.await;
assert!(actions.result().unwrap().as_array().unwrap().is_empty());
}
fn utf16_position_to_byte(source: &str, position: &Value) -> usize {
let target_line = position["line"].as_u64().unwrap() as usize;
let target_units = position["character"].as_u64().unwrap() as usize;
@ -255,6 +306,28 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
json!({"line": 0, "character": 2})
);
let formatting = harness
.request(
"textDocument/formatting",
json!({
"textDocument": { "uri": uri },
"options": { "tabSize": 4, "insertSpaces": true }
}),
2,
)
.await;
let formatting_edits = formatting.result().unwrap().as_array().unwrap();
assert_eq!(formatting_edits.len(), 2);
let mut formatted = source.to_owned();
for edit in formatting_edits.iter().rev() {
let start = utf16_position_to_byte(source, &edit["range"]["start"]);
let end = utf16_position_to_byte(source, &edit["range"]["end"]);
formatted.replace_range(start..end, edit["newText"].as_str().unwrap());
}
let formatter_output =
apply_replacements(source, &plan_replacements(&Config::embedded(), source)).unwrap();
assert_eq!(formatted, formatter_output);
let response = harness
.request(
"textDocument/codeAction",
@ -266,7 +339,7 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
},
"context": { "diagnostics": published["diagnostics"] }
}),
2,
3,
)
.await;
let actions = response.result().unwrap().as_array().unwrap();
@ -285,8 +358,6 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
.collect::<Vec<_>>();
edits.sort_by_key(|edit| edit.byte_span.start);
let lsp_output = apply_replacements(source, &edits).unwrap();
let formatter_output =
apply_replacements(source, &plan_replacements(&Config::embedded(), source)).unwrap();
assert_eq!(lsp_output, formatter_output);
harness
@ -312,7 +383,7 @@ async fn lifecycle_versions_unicode_quick_fixes_and_formatter_equivalence() {
"range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 1 } },
"context": { "diagnostics": published["diagnostics"] }
}),
3,
4,
)
.await;
assert_eq!(stale.error().unwrap().code, ErrorCode::InvalidParams);
@ -358,6 +429,24 @@ enabled = false
.collect::<std::collections::HashMap<_, _>>();
assert!(by_uri[first]["diagnostics"].as_array().unwrap().is_empty());
assert_eq!(by_uri[second]["diagnostics"].as_array().unwrap().len(), 1);
let initially_disabled_format = harness
.request(
"textDocument/formatting",
json!({
"textDocument": { "uri": first },
"options": { "tabSize": 4, "insertSpaces": true }
}),
30,
)
.await;
assert!(
initially_disabled_format
.result()
.unwrap()
.as_array()
.unwrap()
.is_empty()
);
fs::write(
harness.explicit_path(),
@ -391,6 +480,21 @@ enabled = true
refreshed[second]["diagnostics"].as_array().unwrap().len(),
1
);
let reloaded_format = harness
.request(
"textDocument/formatting",
json!({
"textDocument": { "uri": first },
"options": { "tabSize": 4, "insertSpaces": true }
}),
31,
)
.await;
assert_eq!(
reloaded_format.result().unwrap()[0]["newText"],
"-",
"formatting must use the reloaded replacement configuration"
);
fs::write(harness.explicit_path(), "schema_version = 999\n").unwrap();
let failed_reload = Request::build("workspace/didChangeConfiguration")