Compare commits
3 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
4a0c7c9920 | |
|
|
590202da05 | |
|
|
aedc130082 |
|
|
@ -1,11 +1,11 @@
|
|||
[last_run_clean]
|
||||
run_status = "Success"
|
||||
lines_parsed = 421
|
||||
coverage = 92.84
|
||||
lines_parsed = 427
|
||||
coverage = 92.72
|
||||
|
||||
[last_run_clean.run_duration]
|
||||
secs = 55
|
||||
nanos = 635116370
|
||||
secs = 60
|
||||
nanos = 633592735
|
||||
|
||||
[last_run_incremental]
|
||||
run_status = "Success"
|
||||
|
|
|
|||
22
CHANGELOG.md
22
CHANGELOG.md
|
|
@ -6,6 +6,25 @@ All notable changes to John Slop are recorded here. The format follows
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.1.26] - 2026-09-30
|
||||
|
||||
### Added
|
||||
|
||||
- Added conservative response-residue, editorial-placeholder, source-residue,
|
||||
stock-closing, and clustered excess-vocabulary rules with positive and
|
||||
realistic near-miss characterization fixtures.
|
||||
- Added a research-backed review-signal methodology that separates directly
|
||||
reviewable residue, low-weight population clues, verification targets, and
|
||||
deliberately excluded language-proficiency proxies.
|
||||
- Added LSP `--all-diagnostics` and `--minimum-diagnostic-weight WEIGHT`
|
||||
switches for explicit full-catalog and weighted editor review.
|
||||
|
||||
### Changed
|
||||
|
||||
- The LSP now publishes only configured Unicode diagnostics by default. The
|
||||
visibility policy does not alter shared detector results, evidence scores,
|
||||
replacement planning, or formatting.
|
||||
|
||||
## [0.1.25] - 2026-09-30
|
||||
|
||||
### Fixed
|
||||
|
|
@ -121,7 +140,8 @@ All notable changes to John Slop are recorded here. The format follows
|
|||
- Stdio LSP replacement diagnostics and versioned quick fixes.
|
||||
- Runnable examples, reviewed fixtures, fuzz targets, benchmarks, and CI gates.
|
||||
|
||||
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.25...HEAD
|
||||
[Unreleased]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.26...HEAD
|
||||
[0.1.26]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.25...v0.1.26
|
||||
[0.1.25]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.24...v0.1.25
|
||||
[0.1.24]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.23...v0.1.24
|
||||
[0.1.23]: https://git.dandaedre.com/idle-org/john_slop/compare/v0.1.22...v0.1.23
|
||||
|
|
|
|||
|
|
@ -699,7 +699,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-core"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"directories",
|
||||
"figment",
|
||||
|
|
@ -717,7 +717,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-lsp"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"john-slop-core",
|
||||
"serde_json",
|
||||
|
|
@ -728,7 +728,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john_slop"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"criterion",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ default-members = [".", "crates/john-slop-core", "crates/john-slop-lsp"]
|
|||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
edition = "2024"
|
||||
rust-version = "1.98.1"
|
||||
license = "MIT"
|
||||
|
|
@ -18,8 +18,8 @@ figment = { version = "0.10.19", default-features = false, features = ["toml"] }
|
|||
futures = { version = "0.3.34", default-features = false, features = ["std"] }
|
||||
globset = { version = "0.4.20", default-features = false }
|
||||
insta = { version = "1.48.0", default-features = false }
|
||||
john-slop-core = { version = "0.1.25", path = "crates/john-slop-core" }
|
||||
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.25", path = "crates/john-slop-lsp" }
|
||||
john-slop-core = { version = "0.1.26", path = "crates/john-slop-core" }
|
||||
john-slop-lsp-engine = { package = "john-slop-lsp", version = "0.1.26", path = "crates/john-slop-lsp" }
|
||||
miette = { version = "7.6.0", default-features = false, features = ["derive"] }
|
||||
regex = "1.13.1"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
|
|
|
|||
11
README.md
11
README.md
|
|
@ -68,9 +68,9 @@ spans:
|
|||
Unicode, and invisible or bidirectional controls.
|
||||
- `john-slop-fmt` previews, checks, or explicitly applies configured ASCII
|
||||
replacements, including the default printable-ASCII UTS #39 profile.
|
||||
- `john-slop-lsp` publishes the detector's heuristic diagnostics over stdio,
|
||||
offers quick fixes for safe replacements, and provides formatter-equivalent
|
||||
document edits without writing editor-owned files.
|
||||
- `john-slop-lsp` publishes Unicode diagnostics over stdio by default, offers
|
||||
explicit weighted or all-diagnostic opt-ins, provides quick fixes for safe
|
||||
replacements, and formats without writing editor-owned files.
|
||||
|
||||
The detector accepts UTF-8 files, directories, ZIP archives, and common
|
||||
ZIP-based office documents. Each finding includes its source span, stable rule
|
||||
|
|
@ -93,8 +93,9 @@ Start with the guide closest to what you want to do:
|
|||
with [commented configuration examples](examples/config/).
|
||||
4. [Run the Rust and expected-failure examples](examples/README.md).
|
||||
5. [Connect the formatter and LSP to Helix or Vim](docs/editors.md).
|
||||
6. [Understand the heuristics and evidence score](docs/heuristics.md), then
|
||||
review the [interpretation limits](docs/limitations.md).
|
||||
6. [Understand the review-signal methodology](docs/review-signals.md), the
|
||||
[heuristics and evidence score](docs/heuristics.md), and then the
|
||||
[interpretation limits](docs/limitations.md).
|
||||
|
||||
The [documentation index](docs/README.md) offers the same guided path with a
|
||||
little more context.
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.25"><title>version: 0.1.25</title><linearGradient id="bms-5a94bc58efdeae27" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-5a94bc58efdeae27"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-5a94bc58efdeae27)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-5a94bc58efdeae27)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.25</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.25</text></g></svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="96" height="18" role="img" aria-label="version: 0.1.26"><title>version: 0.1.26</title><linearGradient id="bms-51cb0e88c02c82bd" x2="0" y2="100%"><stop offset="0" stop-color="#fff" stop-opacity=".7"/><stop offset=".1" stop-color="#aaa" stop-opacity=".1"/><stop offset=".9" stop-color="#000" stop-opacity=".3"/><stop offset="1" stop-color="#000" stop-opacity=".5"/></linearGradient><clipPath id="bmr-51cb0e88c02c82bd"><rect width="96" height="18" rx="4" fill="#fff"/></clipPath><g clip-path="url(#bmr-51cb0e88c02c82bd)"><rect width="51" height="18" fill="#555"/><rect x="51" width="45" height="18" fill="#fe7d37"/><rect width="96" height="18" fill="url(#bms-51cb0e88c02c82bd)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="130" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="725" y="140" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">0.1.26</text><text x="725" y="130" transform="scale(.1)" fill="#fff" textLength="350">0.1.26</text></g></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |
|
|
@ -87,6 +87,38 @@ enabled = true
|
|||
content = ["prose", "source-code"]
|
||||
extensions = []
|
||||
|
||||
[[rules]]
|
||||
id = "assistant-follow-up-offer"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\b(?:let me know|tell me) if you(?:'d| would) like me to\\b"
|
||||
weight = 12.0
|
||||
message = "a conversational assistant follow-up offer appears in the source"
|
||||
help = "Review whether a response-closing offer was copied into the document."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "assistant-revision-preface"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\bhere(?:'s| is) (?:a|the) (?:revised|polished|rewritten) (?:version|draft)\\b"
|
||||
weight = 12.0
|
||||
message = "an assistant-style revision preface appears in the source"
|
||||
help = "Remove conversational revision scaffolding if it is not part of the document."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "templated-conclusion"
|
||||
category = "style"
|
||||
|
|
@ -135,6 +167,54 @@ enabled = true
|
|||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "editorial-insertion-placeholder"
|
||||
category = "style"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\[(?:insert|add) (?:citation|reference|source|example|details?)(?: here)?\\]"
|
||||
weight = 10.0
|
||||
message = "an unresolved editorial insertion placeholder appears in the source"
|
||||
help = "Supply the requested material or remove the placeholder before publishing."
|
||||
occurrence_cap = 2
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "stock-helpful-closing"
|
||||
category = "style"
|
||||
severity = "advice"
|
||||
matcher = "regex"
|
||||
pattern = "(?im)^[ \\t]*I hope this helps[.!][ \\t]*$"
|
||||
weight = 2.0
|
||||
message = "a stock conversational closing appears in the source"
|
||||
help = "Keep the closing only when the document is intentionally conversational."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "excess-vocabulary-cluster"
|
||||
category = "style"
|
||||
severity = "advice"
|
||||
matcher = "repeated-regex"
|
||||
pattern = "(?i)\\b(?:delv(?:e|es|ed|ing)|intricate|multifaceted|pivotal|underscore[sd]?|realm|landscape)\\b"
|
||||
weight = 1.0
|
||||
message = "several terms from a population-level excess-vocabulary cluster appear"
|
||||
help = "This weak corpus-level clue is not authorship evidence; review only for fit and specificity."
|
||||
occurrence_cap = 3
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "source-placeholder-todo"
|
||||
category = "source-code"
|
||||
|
|
@ -166,3 +246,35 @@ enabled = true
|
|||
[rules.scope]
|
||||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
|
||||
|
||||
[[rules]]
|
||||
id = "source-omitted-code-placeholder"
|
||||
category = "source-code"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?im)^[ \\t]*(?://|#)[ \\t]*(?:\\.\\.\\.[ \\t]*)?(?:existing|remaining|rest of (?:the )?)[ \\t]+code(?:[ \\t]+(?:here|remains?|unchanged))?[ \\t]*$"
|
||||
weight = 10.0
|
||||
message = "an omitted-code placeholder remains in source code"
|
||||
help = "Restore the omitted implementation or remove the placeholder."
|
||||
occurrence_cap = 2
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
|
||||
|
||||
[[rules]]
|
||||
id = "source-markdown-fence"
|
||||
category = "source-code"
|
||||
severity = "advice"
|
||||
matcher = "regex"
|
||||
pattern = "(?m)^[ \\t]*```(?:[A-Za-z0-9_+-]+)?[ \\t]*$"
|
||||
weight = 6.0
|
||||
message = "a Markdown code fence appears in a source-code file"
|
||||
help = "Remove response-formatting fences unless they are intentionally embedded data."
|
||||
occurrence_cap = 2
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
|
||||
|
|
|
|||
|
|
@ -1466,7 +1466,7 @@ mod tests {
|
|||
fn embedded_heuristics_have_complete_metadata() {
|
||||
let config = Config::embedded();
|
||||
|
||||
assert_eq!(config.rules.len(), 12);
|
||||
assert_eq!(config.rules.len(), 19);
|
||||
assert_eq!(config.thresholds.low, 20);
|
||||
assert_eq!(config.thresholds.medium, 50);
|
||||
assert_eq!(config.thresholds.high, 75);
|
||||
|
|
@ -1572,7 +1572,7 @@ scope = { content = ["prose", "source-code"], extensions = [] }
|
|||
|
||||
assert!(!rule.enabled);
|
||||
assert_eq!(rule.weight, 11.0);
|
||||
assert_eq!(config.rules.len(), 13);
|
||||
assert_eq!(config.rules.len(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1788,7 +1788,7 @@ weight = 9.5
|
|||
.enabled
|
||||
);
|
||||
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
|
||||
assert_eq!(config.rules.len(), 12);
|
||||
assert_eq!(config.rules.len(), 19);
|
||||
let heuristic = config
|
||||
.rules
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -19,9 +19,34 @@ use tower_lsp_server::{
|
|||
};
|
||||
|
||||
use john_slop_core::{
|
||||
AnalysisContext, Config, ConfigLoader, Detector, FindingSeverity, plan_replacements,
|
||||
AnalysisContext, Config, ConfigLoader, Detector, Finding, FindingSeverity, plan_replacements,
|
||||
};
|
||||
|
||||
/// Select which shared-detector findings are published to an editor.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||
pub enum DiagnosticFilter {
|
||||
/// Publish only exact Unicode replacement and control findings.
|
||||
#[default]
|
||||
UnicodeOnly,
|
||||
/// Publish Unicode findings plus non-Unicode findings at or above this weight.
|
||||
MinimumWeight(f64),
|
||||
/// Publish every finding produced by the resolved detector configuration.
|
||||
All,
|
||||
}
|
||||
|
||||
impl DiagnosticFilter {
|
||||
fn includes(self, finding: &Finding) -> bool {
|
||||
let unicode = finding.rule_id.as_str().starts_with("unicode-");
|
||||
match self {
|
||||
Self::UnicodeOnly => unicode,
|
||||
Self::MinimumWeight(minimum) => {
|
||||
unicode || (minimum.is_finite() && minimum >= 0.0 && finding.weight >= minimum)
|
||||
}
|
||||
Self::All => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Failure to convert a canonical UTF-8 byte offset into an LSP position.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum PositionError {
|
||||
|
|
@ -148,12 +173,14 @@ fn finding_diagnostics(
|
|||
source: &str,
|
||||
context: &AnalysisContext,
|
||||
version: i32,
|
||||
filter: DiagnosticFilter,
|
||||
) -> Result<Vec<Diagnostic>, PositionError> {
|
||||
let index = LineIndex::new(source);
|
||||
detector
|
||||
.analyze_with(source, context)
|
||||
.findings
|
||||
.into_iter()
|
||||
.filter(|finding| filter.includes(finding))
|
||||
.map(|finding| {
|
||||
let message = match (&finding.fix, &finding.help) {
|
||||
(Some(edit), Some(help)) => format!(
|
||||
|
|
@ -179,6 +206,7 @@ fn finding_diagnostics(
|
|||
diagnostic.data = Some(serde_json::json!({
|
||||
"category": finding.category.as_str(),
|
||||
"fixable": finding.fix.is_some(),
|
||||
"weight": finding.weight,
|
||||
"version": version,
|
||||
}));
|
||||
Ok(diagnostic)
|
||||
|
|
@ -392,6 +420,7 @@ pub struct LspBackend {
|
|||
config_loader: ConfigLoader,
|
||||
runtime: RwLock<AnalysisRuntime>,
|
||||
documents: RwLock<DocumentStore>,
|
||||
diagnostic_filter: DiagnosticFilter,
|
||||
log_json: bool,
|
||||
}
|
||||
|
||||
|
|
@ -408,10 +437,18 @@ impl LspBackend {
|
|||
config_loader,
|
||||
runtime: RwLock::new(runtime),
|
||||
documents: RwLock::new(DocumentStore::default()),
|
||||
diagnostic_filter: DiagnosticFilter::default(),
|
||||
log_json: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Set the editor-facing diagnostic policy without changing detector scoring.
|
||||
#[must_use]
|
||||
pub const fn with_diagnostic_filter(mut self, filter: DiagnosticFilter) -> Self {
|
||||
self.diagnostic_filter = filter;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable newline-delimited JSON lifecycle logging on standard error.
|
||||
#[must_use]
|
||||
pub const fn with_json_logging(mut self, enabled: bool) -> Self {
|
||||
|
|
@ -449,6 +486,7 @@ impl LspBackend {
|
|||
&snapshot.text,
|
||||
&context,
|
||||
snapshot.version,
|
||||
self.diagnostic_filter,
|
||||
)
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
|
@ -612,7 +650,7 @@ mod tests {
|
|||
};
|
||||
|
||||
use super::{
|
||||
DocumentStore, LineIndex, OpenDocument, PositionError, analysis_context,
|
||||
DiagnosticFilter, DocumentStore, LineIndex, OpenDocument, PositionError, analysis_context,
|
||||
finding_diagnostics, ranges_overlap, replacement_code_actions, replacement_diagnostics,
|
||||
requested_version_is_current,
|
||||
};
|
||||
|
|
@ -770,6 +808,7 @@ mod tests {
|
|||
source,
|
||||
&crate::AnalysisContext::for_path("example.txt"),
|
||||
12,
|
||||
DiagnosticFilter::All,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -797,6 +836,49 @@ mod tests {
|
|||
assert_eq!(diagnostics[2].data.as_ref().unwrap()["fixable"], false);
|
||||
assert_eq!(diagnostics[2].data.as_ref().unwrap()["version"], 12);
|
||||
assert_eq!(diagnostics[2].data.as_ref().unwrap()["category"], "style");
|
||||
assert_eq!(diagnostics[2].data.as_ref().unwrap()["weight"], 4.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_filters_keep_unicode_and_gate_non_unicode_by_weight() {
|
||||
let detector = crate::Detector::new(&crate::Config::embedded()).unwrap();
|
||||
let source = "As an AI language model— I hope this helps.";
|
||||
let context = crate::AnalysisContext::for_path("example.txt");
|
||||
|
||||
let unicode = finding_diagnostics(
|
||||
&detector,
|
||||
source,
|
||||
&context,
|
||||
1,
|
||||
DiagnosticFilter::UnicodeOnly,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(unicode.len(), 1);
|
||||
assert_eq!(
|
||||
unicode[0].code,
|
||||
Some(NumberOrString::String(
|
||||
"unicode-ascii-replacement".to_owned()
|
||||
))
|
||||
);
|
||||
|
||||
let weighted = finding_diagnostics(
|
||||
&detector,
|
||||
source,
|
||||
&context,
|
||||
1,
|
||||
DiagnosticFilter::MinimumWeight(10.0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(weighted.len(), 2);
|
||||
assert!(weighted.iter().any(|diagnostic| {
|
||||
diagnostic.code
|
||||
== Some(NumberOrString::String(
|
||||
"assistant-self-identification".to_owned(),
|
||||
))
|
||||
}));
|
||||
assert!(!weighted.iter().any(|diagnostic| {
|
||||
diagnostic.code == Some(NumberOrString::String("stock-helpful-closing".to_owned()))
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ internals to use the tools.
|
|||
diagnostic showcase.
|
||||
5. [Editor setup](editors.md) provides working Helix (`hx`) and Vim formatter
|
||||
and LSP configurations with validation steps.
|
||||
6. [Heuristics and evidence scoring](heuristics.md) explains what each rule
|
||||
6. [Review signals and evidence methodology](review-signals.md) explains the
|
||||
research boundary, evidence tiers, and deliberately excluded proxies.
|
||||
7. [Heuristics and evidence scoring](heuristics.md) explains what each rule
|
||||
contributes and how the bounded score is calculated.
|
||||
7. [Limitations](limitations.md) covers false positives, evasion, domain shift,
|
||||
8. [Limitations](limitations.md) covers false positives, evasion, domain shift,
|
||||
score semantics, and safe interpretation.
|
||||
|
||||
## Go deeper
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ points for day-to-day use.
|
|||
- [Heuristics and evidence scoring](heuristics.md) describes applicability,
|
||||
occurrence damping, normalization, and why the score is not authorship
|
||||
confidence.
|
||||
- [Review signals and evidence methodology](review-signals.md) records the
|
||||
research basis, evidence tiers, and signals deliberately kept out.
|
||||
- [Evaluation](evaluation.md) records corpus provenance, per-rule measurements,
|
||||
aggregate false positives, and the limits of the current metrics.
|
||||
- [Limitations](limitations.md) covers evasion, domain shift, interpretation,
|
||||
|
|
|
|||
|
|
@ -97,8 +97,9 @@ Configuration reload swaps only a fully validated model and republishes open
|
|||
documents in deterministic URI order. The detector is compiled once per
|
||||
resolved configuration rather than once per edit. File URI paths drive normal
|
||||
scope applicability; language IDs supply source/prose context for untitled or
|
||||
extensionless buffers. All findings become diagnostics, while only findings
|
||||
with core `TextEdit`s become code actions. Document formatting calls the same
|
||||
extensionless buffers. The default publication filter retains only Unicode
|
||||
findings; explicit all-diagnostic and minimum-weight modes expose broader
|
||||
findings, while only findings with core `TextEdit`s become code actions. Document formatting calls the same
|
||||
replacement planner directly and never derives mutations from diagnostics. See
|
||||
[ADR 0003](decisions/0003-lsp-framework-and-protocol.md) for framework and
|
||||
protocol choices.
|
||||
|
|
|
|||
|
|
@ -77,6 +77,17 @@ This amendment does not broaden mutation behavior:
|
|||
The original full-snapshot synchronization, UTF-16 conversion, version checks,
|
||||
stdio purity, no-write boundary, and stale-action guarantees remain unchanged.
|
||||
|
||||
## Amendment: quiet diagnostic publication
|
||||
|
||||
On 2026-09-30, Milestone 26 changed the presentation default without changing
|
||||
the shared detector. The LSP now publishes only Unicode findings unless it is
|
||||
started with `--all-diagnostics` or `--minimum-diagnostic-weight WEIGHT`.
|
||||
Minimum-weight mode always retains Unicode findings and filters only the other
|
||||
categories. This makes exact, fixable problems the quiet editor default while
|
||||
keeping the full catalog easy to exercise in tests. Filtering occurs after
|
||||
analysis, so CLI results, evidence scores, formatting, configuration reload,
|
||||
and the core finding contract remain unchanged.
|
||||
|
||||
## Amendment: document formatting parity
|
||||
|
||||
Milestone 21 adds the standard `textDocument/formatting` capability. The server
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@ Run line coverage with Cargo Tarpaulin:
|
|||
cargo tarpaulin --config tarpaulin.toml
|
||||
```
|
||||
|
||||
The configuration enforces a 90% floor. The current suite measured 92.83%
|
||||
(1,579/1,701 lines) on 2026-09-30. The remaining lines are dominated by
|
||||
The configuration enforces a 90% floor. The current suite measured 92.72%
|
||||
(1,604/1,730 lines) on 2026-09-30. The remaining lines are dominated by
|
||||
platform/I/O failure injection, malformed archive internals, coordinate
|
||||
overflow, LSP logging/reload failures, and subprocess attribution. Do not add
|
||||
brittle tests or production seams solely to make Tarpaulin count those paths.
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
# Editor setup
|
||||
|
||||
Both `john-slop-fmt` and `john-slop-lsp` are operational. The LSP publishes all
|
||||
applicable configured heuristic findings, including prompt leakage, repeated
|
||||
transitions, source placeholders, tutorial narration, and typography. Safe
|
||||
ASCII-replacement findings additionally provide versioned quick fixes, and the
|
||||
server implements whole-document formatting with the exact same replacement
|
||||
planner as `john-slop-fmt`.
|
||||
Both `john-slop-fmt` and `john-slop-lsp` are operational. The LSP publishes only
|
||||
configured Unicode replacement and control findings by default, keeping noisy
|
||||
or error-prone heuristics out of the normal editor loop. Safe ASCII-replacement
|
||||
findings additionally provide versioned quick fixes, and the server implements
|
||||
whole-document formatting with the exact same replacement planner as
|
||||
`john-slop-fmt`.
|
||||
|
||||
Install all three binaries somewhere on `PATH`, then confirm the editor-facing
|
||||
help is available:
|
||||
|
|
@ -111,6 +111,10 @@ The server uses the same configuration layers as the other binaries. Add
|
|||
append those arguments to the Vim `cmd` list, when an editor-specific named
|
||||
configuration is wanted. Use `args = ["--diagnostics", "diagnostics.toml"]` (and
|
||||
the equivalent Vim command argument) for a project runtime diagnostic catalog.
|
||||
Add `--all-diagnostics` to exercise every configured rule during testing, or
|
||||
`--minimum-diagnostic-weight 10` to retain Unicode findings and opt into only
|
||||
non-Unicode findings with weight 10 or greater. This visibility filter does not
|
||||
change detector scoring or formatting.
|
||||
`--log-json` is available for troubleshooting and writes lifecycle metadata to
|
||||
stderr without document contents.
|
||||
|
||||
|
|
@ -128,7 +132,7 @@ cargo run --bin john-slop-fmt -- \
|
|||
examples/fixtures/typography/example.txt
|
||||
```
|
||||
|
||||
For non-fixable heuristic diagnostics, open
|
||||
After starting the server with `--all-diagnostics`, open
|
||||
`examples/fixtures/style/example.txt` to see `repeated-transition` and other
|
||||
style findings, or `examples/fixtures/source/example.rs` to see source-only
|
||||
placeholder and narration findings alongside typography. Advice maps to LSP
|
||||
|
|
|
|||
|
|
@ -14,15 +14,22 @@ With the embedded schema-version 1 catalog, the corpus currently reports:
|
|||
| assistant-answer-preface | 1 | 0 | 0 | 100% | 100% |
|
||||
| knowledge-cutoff-disclaimer | 1 | 0 | 0 | 100% | 100% |
|
||||
| prompt-response-scaffolding | 1 | 0 | 0 | 100% | 100% |
|
||||
| assistant-follow-up-offer | 1 | 0 | 0 | 100% | 100% |
|
||||
| assistant-revision-preface | 1 | 0 | 0 | 100% | 100% |
|
||||
| templated-conclusion | 1 | 0 | 0 | 100% | 100% |
|
||||
| templated-filler | 1 | 0 | 0 | 100% | 100% |
|
||||
| repeated-transition | 1 | 0 | 0 | 100% | 100% |
|
||||
| editorial-insertion-placeholder | 1 | 0 | 0 | 100% | 100% |
|
||||
| stock-helpful-closing | 1 | 0 | 0 | 100% | 100% |
|
||||
| excess-vocabulary-cluster | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-omitted-code-placeholder | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-markdown-fence | 1 | 0 | 0 | 100% | 100% |
|
||||
| unicode-ascii-replacement | 2 | 0 | 0 | 100% | 100% |
|
||||
| unicode-invisible-control | 1 | 0 | 0 | 100% | 100% |
|
||||
|
||||
Aggregate labeled-rule counts are 13 true positives, 0 false positives, and 0
|
||||
Aggregate labeled-rule counts are 20 true positives, 0 false positives, and 0
|
||||
false negatives. The clean-document false-positive rate is 0/5 (0%). The test
|
||||
suite recomputes these values from the manifest and fails on drift.
|
||||
|
||||
|
|
@ -30,10 +37,10 @@ The same manifest pins evidence scores for regression characterization:
|
|||
|
||||
| Corpus case | Score |
|
||||
| --- | ---: |
|
||||
| four assistant disclaimer/preface rules | 60 |
|
||||
| six assistant disclaimer/preface rules | 64 |
|
||||
| one prompt/response marker | 33 |
|
||||
| style signals | 19 |
|
||||
| source-code signals | 24 |
|
||||
| style signals | 25 |
|
||||
| source-code signals | 33 |
|
||||
| typography signal | 17 |
|
||||
| one ASCII lookalike and one invisible character | 28 |
|
||||
| each of five negative documents | 0 |
|
||||
|
|
@ -54,6 +61,9 @@ replaceable by default, retains a non-mutating `review` policy, and keeps the
|
|||
Gitea-compatible invisible/control profile diagnostic-only, with explicit
|
||||
multilingual limitations. The Milestone 25 review gives one-to-one fullwidth
|
||||
ASCII compatibility folds precedence over visual skeletons under every policy.
|
||||
The Milestone 26 review adds narrow response and unfinished-editing residue,
|
||||
keeps population-level vocabulary and stock closings deliberately low-weight,
|
||||
and excludes language-proficiency proxies.
|
||||
The checked-in `signoff.toml` records those decisions
|
||||
and contains the SHA-256 digest of `config.toml`, `heuristics.toml`, and
|
||||
`unicode.toml`. This binds the reviewed scoring behavior to both rule catalogs
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@ and LSP publication as embedded rules. New matcher algorithms and categories
|
|||
remain code changes rather than executable configuration.
|
||||
|
||||
The generated Unicode 16.0 profile maps every non-ASCII scalar with a printable
|
||||
ASCII UTS #39 skeleton, alongside ten manually reviewed punctuation mappings.
|
||||
ASCII UTS #39 skeleton after explicitly reviewed compatibility-width and
|
||||
punctuation mappings.
|
||||
The default `replace` policy gives each of these findings a formatter/LSP edit;
|
||||
`review` keeps the same exact diagnostics without edits, and `off` suppresses
|
||||
the generated lookalikes. Invisible and direction-changing controls remain
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
# Review signals and evidence methodology
|
||||
|
||||
John Slop finds concrete text that may deserve review. It does **not** determine
|
||||
whether AI wrote a document, identify a model, or assign an authorship
|
||||
probability. A finding names the exact matched text and the configured reason
|
||||
for reviewing it; the evidence score is only a bounded aggregation of those
|
||||
findings.
|
||||
|
||||
This boundary matters because general-purpose AI-text detection is not reliable
|
||||
enough to support an authorship claim. Sadasivan et al. show both practical
|
||||
paraphrasing attacks and a theoretical trade-off between detection and false
|
||||
positives as human and model distributions become closer [1]. A comparative
|
||||
evaluation of 14 detection tools likewise found them neither accurate nor
|
||||
reliable, with performance worsening after obfuscation [2]. Liang et al. found
|
||||
that perplexity-based detectors disproportionately classified non-native English
|
||||
writing as AI-generated [3]. John Slop therefore does not use perplexity,
|
||||
burstiness, grammar, spelling, vocabulary richness, dialect, or English fluency
|
||||
as evidence.
|
||||
|
||||
## Signal tiers
|
||||
|
||||
### Direct response residue
|
||||
|
||||
These are relatively strong, locally explainable signs that conversational
|
||||
material may have been pasted into a document:
|
||||
|
||||
- assistant self-identification, capability, or knowledge-cutoff disclaimers;
|
||||
- `User:` / `Assistant:` transcript scaffolding;
|
||||
- answer or revision prefaces; and
|
||||
- follow-up offers such as asking whether the reader wants an expansion.
|
||||
|
||||
They still do not prove authorship. A quotation, article about assistants, or
|
||||
intentional transcript can contain the same text, so applicability scopes and
|
||||
near-miss fixtures are part of every rule's review.
|
||||
|
||||
### Unfinished editing and source residue
|
||||
|
||||
Unresolved insertion notes, implementation placeholders, omitted-code comments,
|
||||
tutorial narration, and stray Markdown fences are useful low-care signals
|
||||
regardless of how they were produced. These rules ask whether work is complete;
|
||||
they do not infer who or what produced it.
|
||||
|
||||
### Subtle population-level clues
|
||||
|
||||
Some patterns are meaningful across a large corpus but weak in one document.
|
||||
Kobak et al. measured abrupt, population-level increases in selected style words
|
||||
across more than 15 million biomedical abstracts [4]. John Slop represents this
|
||||
only as `excess-vocabulary-cluster`: it requires multiple matches, has weight
|
||||
`1.0`, is occurrence-capped, and says explicitly that it is not authorship
|
||||
evidence. A stock closing has similarly low weight. These clues should prompt a
|
||||
specificity edit at most, never an accusation.
|
||||
|
||||
Repeated transitions and templated phrases sit between the tiers. They activate
|
||||
only on an exact conservative pattern; repetition from one rule is damped and
|
||||
cannot alone create a high evidence band.
|
||||
|
||||
### Verification targets, not offline shape rules
|
||||
|
||||
Generated references and package names can be wrong. Walters and Wilder found
|
||||
fabricated and substantially erroneous citations in a controlled set of
|
||||
generated bibliographies [5]. Spracklen et al. measured package hallucinations
|
||||
in a large code-generation study [6]. But a citation's typography or a package
|
||||
name's shape cannot establish that it is fictitious. Reliable checks require an
|
||||
authoritative catalog and contextual validation, while John Slop analysis is
|
||||
local and makes no network calls. These remain human review targets rather than
|
||||
shipped heuristic rules.
|
||||
|
||||
## How a rule is admitted
|
||||
|
||||
A shipped rule must have a stable ID, a narrow pattern, a rationale, review
|
||||
help, an applicability scope, a finite non-negative weight, and an occurrence
|
||||
cap. It also needs:
|
||||
|
||||
1. a positive fixture showing the intended match;
|
||||
2. a realistic near miss showing what must not match;
|
||||
3. deterministic byte spans and ordering; and
|
||||
4. review of likely harm, especially language-proficiency and demographic
|
||||
proxies.
|
||||
|
||||
Exact residue and fixable Unicode receive higher confidence than broad stylistic
|
||||
clues. New research is used to propose testable review signals, not to import a
|
||||
paper's population-level conclusion into an individual-document verdict.
|
||||
|
||||
## Scoring and editor visibility
|
||||
|
||||
Rule weight contributes bounded evidence to the shared score. Per-rule caps and
|
||||
correlation damping prevent repeated copies of one pattern from dominating it.
|
||||
The number is neither accuracy nor probability; see
|
||||
[Heuristics and evidence scoring](heuristics.md) for the formula.
|
||||
|
||||
LSP visibility is a separate presentation choice. By default the editor shows
|
||||
only configured Unicode findings, which are exact and may have safe replacement
|
||||
edits. `--all-diagnostics` enables the complete detector catalog for testing or
|
||||
deliberate review. `--minimum-diagnostic-weight <WEIGHT>` is the quieter opt-in:
|
||||
it keeps Unicode findings and adds only non-Unicode findings whose configured
|
||||
weight meets the threshold. Filtering affects editor diagnostics, not CLI
|
||||
analysis or scoring.
|
||||
|
||||
## References
|
||||
|
||||
1. Sadasivan et al., [Can AI-Generated Text be Reliably Detected?](https://arxiv.org/abs/2303.11156)
|
||||
2. Weber-Wulff et al., [Testing of detection tools for AI-generated text](https://link.springer.com/article/10.1007/s40979-023-00146-z)
|
||||
3. Liang et al., [GPT detectors are biased against non-native English writers](https://pmc.ncbi.nlm.nih.gov/articles/PMC10382961/)
|
||||
4. Kobak et al., [Delving into LLM-assisted writing in biomedical publications through excess vocabulary](https://arxiv.org/abs/2406.07016)
|
||||
5. Walters and Wilder, [Fabrication and errors in the bibliographic citations generated by ChatGPT](https://pmc.ncbi.nlm.nih.gov/articles/PMC10484980/)
|
||||
6. Spracklen et al., [We Have a Package for You! A Comprehensive Analysis of Package Hallucinations by Code Generating LLMs](https://www.usenix.org/conference/usenixsecurity25/presentation/spracklen)
|
||||
|
|
@ -68,6 +68,12 @@ Use `john-slop config show` to print the resolved TOML and `john-slop config
|
|||
paths` to inspect source precedence. See [Configuration](configuration.md) for
|
||||
the schema and merge rules.
|
||||
|
||||
The LSP publishes Unicode findings by default. Start `john-slop-lsp` with
|
||||
`--all-diagnostics` to publish the complete configured detector catalog, or use
|
||||
`--minimum-diagnostic-weight WEIGHT` to retain Unicode findings and add only
|
||||
non-Unicode findings at or above that weight. This editor-only filter does not
|
||||
change CLI analysis, evidence scores, replacement planning, or formatting.
|
||||
|
||||
## Rust examples
|
||||
|
||||
The [`examples` directory](../examples/README.md) contains runnable Cargo
|
||||
|
|
@ -141,15 +147,19 @@ john-slop-lsp
|
|||
john-slop-lsp --config-name editorial
|
||||
john-slop-lsp --config ./john-slop.toml
|
||||
john-slop-lsp --diagnostics ./diagnostics.toml
|
||||
john-slop-lsp --all-diagnostics
|
||||
john-slop-lsp --minimum-diagnostic-weight 10
|
||||
```
|
||||
|
||||
The server advertises full-document synchronization and quick-fix code actions.
|
||||
It stores only currently open UTF-8 snapshots, their URI, language ID, and
|
||||
monotonically increasing version. On open or an accepted newer full-document
|
||||
change it runs the same configured detector as the CLI and publishes every
|
||||
applicable heuristic finding. File URI extensions select prose/source and path
|
||||
scopes; editor language IDs provide the prose/source fallback for untitled or
|
||||
extensionless buffers.
|
||||
applicable Unicode finding. `--all-diagnostics` publishes the complete result;
|
||||
`--minimum-diagnostic-weight WEIGHT` adds only non-Unicode findings meeting the
|
||||
threshold. File URI extensions select prose/source and path scopes; editor
|
||||
language IDs provide the prose/source fallback for untitled or extensionless
|
||||
buffers.
|
||||
|
||||
Each diagnostic has source `john-slop`, the stable heuristic rule ID as its
|
||||
code, the configured severity and remediation text, document-version metadata,
|
||||
|
|
@ -165,7 +175,7 @@ source-review findings are diagnostic-only. A request carrying stale John Slop
|
|||
diagnostic metadata is rejected. Closing a document removes its snapshot and
|
||||
publishes an empty diagnostic set. `workspace/didChangeConfiguration` reloads
|
||||
the normal embedded, user, named, and explicit layers, recompiles the detector,
|
||||
and republishes every open snapshot; a failed reload keeps the last valid
|
||||
and republishes every open snapshot under the unchanged visibility filter; a failed reload keeps the last valid
|
||||
configuration and detector.
|
||||
|
||||
Stdout is reserved for JSON-RPC framing. The server is quiet by default; pass
|
||||
|
|
|
|||
|
|
@ -385,7 +385,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-core"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"directories",
|
||||
"figment",
|
||||
|
|
@ -411,7 +411,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john-slop-lsp"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"john-slop-core",
|
||||
"serde_json",
|
||||
|
|
@ -422,7 +422,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "john_slop"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"globset",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ mod editor_help;
|
|||
use std::path::PathBuf;
|
||||
|
||||
use clap::Parser;
|
||||
use john_slop::{ConfigLoader, ConfigName, lsp::LspBackend};
|
||||
use john_slop::{
|
||||
ConfigLoader, ConfigName,
|
||||
lsp::{DiagnosticFilter, LspBackend},
|
||||
};
|
||||
use miette::Result;
|
||||
use tower_lsp_server::{LspService, Server};
|
||||
|
||||
|
|
@ -31,6 +34,19 @@ struct Cli {
|
|||
#[arg(long, value_name = "FILE", action = clap::ArgAction::Append)]
|
||||
diagnostics: Vec<PathBuf>,
|
||||
|
||||
/// Publish all configured diagnostics instead of the Unicode-only default.
|
||||
#[arg(long, conflicts_with = "minimum_diagnostic_weight")]
|
||||
all_diagnostics: bool,
|
||||
|
||||
/// Also publish non-Unicode diagnostics at or above this configured weight.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "WEIGHT",
|
||||
value_parser = parse_diagnostic_weight,
|
||||
conflicts_with = "all_diagnostics"
|
||||
)]
|
||||
minimum_diagnostic_weight: Option<f64>,
|
||||
|
||||
/// Emit newline-delimited JSON lifecycle logs to standard error.
|
||||
#[arg(long)]
|
||||
log_json: bool,
|
||||
|
|
@ -39,6 +55,13 @@ struct Cli {
|
|||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let diagnostic_filter = if cli.all_diagnostics {
|
||||
DiagnosticFilter::All
|
||||
} else if let Some(minimum) = cli.minimum_diagnostic_weight {
|
||||
DiagnosticFilter::MinimumWeight(minimum)
|
||||
} else {
|
||||
DiagnosticFilter::UnicodeOnly
|
||||
};
|
||||
let loader = ConfigLoader::discover(cli.config_name.as_ref(), cli.config)?
|
||||
.with_diagnostic_files(cli.diagnostics);
|
||||
let config = loader.load()?;
|
||||
|
|
@ -46,6 +69,7 @@ async fn main() -> Result<()> {
|
|||
// `ConfigLoader::load` validated this exact snapshot immediately above.
|
||||
LspBackend::new(client, loader, config)
|
||||
.expect("validated configuration must build the LSP detector")
|
||||
.with_diagnostic_filter(diagnostic_filter)
|
||||
.with_json_logging(cli.log_json)
|
||||
});
|
||||
Server::new(tokio::io::stdin(), tokio::io::stdout(), socket)
|
||||
|
|
@ -53,3 +77,13 @@ async fn main() -> Result<()> {
|
|||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_diagnostic_weight(value: &str) -> Result<f64, String> {
|
||||
let weight = value
|
||||
.parse::<f64>()
|
||||
.map_err(|_| "weight must be a finite non-negative number".to_owned())?;
|
||||
if !weight.is_finite() || weight < 0.0 {
|
||||
return Err("weight must be a finite non-negative number".to_owned());
|
||||
}
|
||||
Ok(weight)
|
||||
}
|
||||
|
|
|
|||
35
tasks.md
35
tasks.md
|
|
@ -666,6 +666,41 @@ evaluation, dependency policy/audit checks, Unicode-catalog reproducibility,
|
|||
and workspace package assembly pass; coverage is 92.84% (1,583/1,705 lines),
|
||||
and all release metadata and binary versions agree on `0.1.25`.
|
||||
|
||||
## Milestone 26 — Evidence tiers and quiet editor defaults
|
||||
|
||||
- [x] Document the research boundary between directly reviewable residue,
|
||||
low-confidence population-level signals, and unreliable authorship inference.
|
||||
Add conservative data-driven rules with positive and near-miss evaluation
|
||||
fixtures, while explicitly excluding fluency and perplexity proxies.
|
||||
- [x] Make LSP diagnostics Unicode-only by default. Add an
|
||||
`--all-diagnostics` testing switch and a minimum-weight opt-in that always
|
||||
retains Unicode findings while suppressing lighter non-Unicode signals.
|
||||
Cover the policy through unit, protocol, and CLI-help tests.
|
||||
- [x] Update editor/configuration guidance, changelog, evaluation sign-off,
|
||||
release metadata, coverage, badges, and recap; run all quality, audit,
|
||||
regeneration, and package gates.
|
||||
|
||||
Decision: rule weight remains bounded evidence for scoring, not a probability
|
||||
of AI authorship. LSP visibility is a separate presentation policy: Unicode
|
||||
findings are shown by default because they are exact and fixable; non-Unicode
|
||||
rules require an explicit all-diagnostics switch or minimum-weight threshold.
|
||||
This keeps subtle research-derived vocabulary and style clues out of the
|
||||
default editor experience without weakening CLI analysis or configuration.
|
||||
|
||||
Acceptance: the methodology cites primary evidence and states what the tool
|
||||
will not infer; each new rule has reviewed positive and near-miss coverage; the
|
||||
default LSP publishes only Unicode findings, both opt-in modes are deterministic,
|
||||
and all release artifacts agree on `0.1.26` before the immutable `v0.1.26` tag
|
||||
is created.
|
||||
|
||||
Accepted on 2026-09-30 after 19 shipped rules each passed positive and realistic
|
||||
near-miss characterization, the default protocol path published only Unicode,
|
||||
and weighted/all-diagnostic opt-ins passed unit, CLI, and real LSP tests.
|
||||
Formatting, Clippy, rustdoc, both feature matrices, evaluation, dependency
|
||||
policy/audit checks, Unicode-catalog reproducibility, and workspace package
|
||||
assembly pass; coverage is 92.72% (1,604/1,730 lines), and all release metadata
|
||||
and binary versions agree on `0.1.26`.
|
||||
|
||||
## Deferred by design
|
||||
|
||||
- Statistical or model-based authorship classification.
|
||||
|
|
|
|||
|
|
@ -34,4 +34,26 @@ fn lsp_help_contains_helix_and_vim_setup() {
|
|||
assert!(stdout.contains("lsp#register_server"));
|
||||
assert!(stdout.contains("['john-slop-lsp']"));
|
||||
assert!(stdout.contains("--diagnostics <FILE>"));
|
||||
assert!(stdout.contains("--all-diagnostics"));
|
||||
assert!(stdout.contains("--minimum-diagnostic-weight <WEIGHT>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lsp_rejects_invalid_or_conflicting_diagnostic_filters() {
|
||||
let invalid = Command::new(env!("CARGO_BIN_EXE_john-slop-lsp"))
|
||||
.args(["--minimum-diagnostic-weight", "NaN"])
|
||||
.output()
|
||||
.expect("run LSP with invalid weight");
|
||||
assert!(!invalid.status.success());
|
||||
assert!(
|
||||
String::from_utf8(invalid.stderr)
|
||||
.unwrap()
|
||||
.contains("finite non-negative")
|
||||
);
|
||||
|
||||
let conflicting = Command::new(env!("CARGO_BIN_EXE_john-slop-lsp"))
|
||||
.args(["--all-diagnostics", "--minimum-diagnostic-weight", "10"])
|
||||
.output()
|
||||
.expect("run LSP with conflicting filters");
|
||||
assert!(!conflicting.status.success());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ fn diagnostics_init_generates_a_loadable_runtime_catalog() {
|
|||
String::from_utf8_lossy(&loaded.stderr)
|
||||
);
|
||||
let resolved = toml::from_slice::<toml::Value>(&loaded.stdout).unwrap();
|
||||
assert_eq!(resolved["rules"].as_array().unwrap().len(), 13);
|
||||
assert_eq!(resolved["rules"].as_array().unwrap().len(), 20);
|
||||
|
||||
let refused = detector()
|
||||
.current_dir(directory.path())
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
|||
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
|
||||
);
|
||||
|
||||
assert_eq!((total_tp, total_fp, total_fn), (13, 0, 0));
|
||||
assert_eq!((total_tp, total_fp, total_fn), (20, 0, 0));
|
||||
assert_eq!((false_positive_cases, negative_cases), (0, 5));
|
||||
assert_eq!(
|
||||
reviewed_near_misses,
|
||||
|
|
|
|||
|
|
@ -2,3 +2,5 @@ Certainly! Here's a compact answer.
|
|||
|
||||
As an AI language model, I cannot browse the internet. My knowledge cutoff may
|
||||
also make current details unavailable.
|
||||
|
||||
Here's the revised version. Let me know if you'd like me to expand it.
|
||||
|
|
|
|||
|
|
@ -2,3 +2,6 @@
|
|||
fn parse() {
|
||||
// TODO: implement the parser
|
||||
}
|
||||
|
||||
// ... remaining code here
|
||||
```rust
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ license = "MIT"
|
|||
[[cases]]
|
||||
path = "assistant.md"
|
||||
kind = "prose"
|
||||
expected_score = 60
|
||||
expected_score = 64
|
||||
expected = [
|
||||
"assistant-answer-preface",
|
||||
"assistant-capability-disclaimer",
|
||||
"assistant-follow-up-offer",
|
||||
"assistant-revision-preface",
|
||||
"assistant-self-identification",
|
||||
"knowledge-cutoff-disclaimer",
|
||||
]
|
||||
|
|
@ -21,14 +23,26 @@ expected = ["prompt-response-scaffolding"]
|
|||
[[cases]]
|
||||
path = "style.md"
|
||||
kind = "prose"
|
||||
expected_score = 19
|
||||
expected = ["repeated-transition", "templated-conclusion", "templated-filler"]
|
||||
expected_score = 25
|
||||
expected = [
|
||||
"editorial-insertion-placeholder",
|
||||
"excess-vocabulary-cluster",
|
||||
"repeated-transition",
|
||||
"stock-helpful-closing",
|
||||
"templated-conclusion",
|
||||
"templated-filler",
|
||||
]
|
||||
|
||||
[[cases]]
|
||||
path = "example.rs"
|
||||
kind = "source-code"
|
||||
expected_score = 24
|
||||
expected = ["source-placeholder-todo", "source-tutorial-narration"]
|
||||
expected_score = 33
|
||||
expected = [
|
||||
"source-markdown-fence",
|
||||
"source-omitted-code-placeholder",
|
||||
"source-placeholder-todo",
|
||||
"source-tutorial-narration",
|
||||
]
|
||||
|
||||
[[cases]]
|
||||
path = "typography.md"
|
||||
|
|
@ -60,6 +74,8 @@ expected = []
|
|||
near_miss_for = [
|
||||
"assistant-answer-preface",
|
||||
"assistant-capability-disclaimer",
|
||||
"assistant-follow-up-offer",
|
||||
"assistant-revision-preface",
|
||||
"assistant-self-identification",
|
||||
"knowledge-cutoff-disclaimer",
|
||||
]
|
||||
|
|
@ -77,7 +93,10 @@ kind = "prose"
|
|||
expected_score = 0
|
||||
expected = []
|
||||
near_miss_for = [
|
||||
"editorial-insertion-placeholder",
|
||||
"excess-vocabulary-cluster",
|
||||
"repeated-transition",
|
||||
"stock-helpful-closing",
|
||||
"templated-conclusion",
|
||||
"templated-filler",
|
||||
]
|
||||
|
|
@ -87,4 +106,9 @@ path = "negative-source.rs"
|
|||
kind = "source-code"
|
||||
expected_score = 0
|
||||
expected = []
|
||||
near_miss_for = ["source-placeholder-todo", "source-tutorial-narration"]
|
||||
near_miss_for = [
|
||||
"source-markdown-fence",
|
||||
"source-omitted-code-placeholder",
|
||||
"source-placeholder-todo",
|
||||
"source-tutorial-narration",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
The editor described a model disclosure in original language and verified the
|
||||
browser limitation against the tool documentation. The dataset date was cited
|
||||
directly rather than presented as a generic cutoff disclaimer.
|
||||
|
||||
The editor asked whether readers would like a revised draft, but the published
|
||||
article begins directly with its findings rather than retaining that exchange.
|
||||
|
|
|
|||
|
|
@ -3,3 +3,6 @@
|
|||
fn parse_identifier(input: &str) -> &str {
|
||||
input.trim()
|
||||
}
|
||||
|
||||
const MARKDOWN_FENCE: &str = "three backticks";
|
||||
// The remaining code validates the parsed identifier below.
|
||||
|
|
|
|||
|
|
@ -1,2 +1,5 @@
|
|||
Moreover, the first measurement includes its sample size. The conclusion names
|
||||
the retained limit and the observed latency instead of repeating a template.
|
||||
|
||||
The landscape photograph shows a complex rock formation. The editor resolved
|
||||
the citation request, and the support agent wrote that the answer should help.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
schema_version = 1
|
||||
release = "0.1.25"
|
||||
release = "0.1.26"
|
||||
reviewed_on = "2026-09-30"
|
||||
status = "accepted-with-runtime-catalog-boundary"
|
||||
catalog_sha256 = "fa1a26bfa40e7336a4930f46c5511a2ea3223cf3e7a373bb830372eaa523a020"
|
||||
catalog_sha256 = "da34ce00dfd2d801e772fa440d932abb98c929f5ad08ff3ad5e47e0ed4d84d5a"
|
||||
limitations_acknowledged = true
|
||||
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Keep every diagnostic category enabled by default while allowing layered category gates and individual rule switches. Permit declarative runtime catalogs to add exact-phrase, regex, and repeated-regex rules, but reject code-backed matcher engines. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Give reviewed fullwidth ASCII compatibility folds precedence over visual UTS #39 skeletons under every confusable policy. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
|
||||
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Keep every diagnostic category enabled by default while allowing layered category gates and individual rule switches. Permit declarative runtime catalogs to add exact-phrase, regex, and repeated-regex rules, but reject code-backed matcher engines. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Give reviewed fullwidth ASCII compatibility folds precedence over visual UTS #39 skeletons under every confusable policy. Admit direct response and unfinished-editing residue with narrow patterns, but keep population-level excess vocabulary and stock closings at low weight; reject fluency, grammar, dialect, and perplexity proxies. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
|
||||
|
|
|
|||
|
|
@ -4,3 +4,7 @@ Moreover, the first observation repeats the input.
|
|||
Additionally, the second observation repeats it again.
|
||||
|
||||
In conclusion, we can keep the existing limit.
|
||||
|
||||
The report delves into a multifaceted landscape. [Insert citation here]
|
||||
|
||||
I hope this helps.
|
||||
|
|
|
|||
|
|
@ -62,6 +62,18 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
|||
near_miss: "The assistant: generated a response.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "assistant-follow-up-offer",
|
||||
positive: "Let me know if you'd like me to expand this.",
|
||||
near_miss: "Let me know whether the expansion is useful.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "assistant-revision-preface",
|
||||
positive: "Here's the polished draft for publication.",
|
||||
near_miss: "The polished draft was accepted for publication.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "templated-conclusion",
|
||||
positive: "In conclusion, we can retain this design.",
|
||||
|
|
@ -80,6 +92,24 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
|||
near_miss: "Moreover, the only transition is relevant.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "editorial-insertion-placeholder",
|
||||
positive: "The claim needs support. [Add reference here]",
|
||||
near_miss: "The claim cites [Smith 2024].",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "stock-helpful-closing",
|
||||
positive: "The command exits successfully.\nI hope this helps.",
|
||||
near_miss: "We hope this helps readers reproduce the result.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "excess-vocabulary-cluster",
|
||||
positive: "The report delves into a multifaceted problem.",
|
||||
near_miss: "The landscape photograph records the site.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "source-placeholder-todo",
|
||||
positive: "fn parse() { /* TODO: implement the parser */ }",
|
||||
|
|
@ -92,6 +122,18 @@ fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
|||
near_miss: "// First input byte stores the version.\nlet value = parse();",
|
||||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "source-omitted-code-placeholder",
|
||||
positive: "fn parse() {}\n// ... remaining code here",
|
||||
near_miss: "// The remaining code validates the parsed value.",
|
||||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "source-markdown-fence",
|
||||
positive: "```rust\nfn parse() {}",
|
||||
near_miss: "const FENCE: &str = \"three backticks\";",
|
||||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "unicode-ascii-replacement",
|
||||
positive: "Review this—then continue.",
|
||||
|
|
|
|||
77
tests/lsp.rs
77
tests/lsp.rs
|
|
@ -4,7 +4,9 @@ use std::{fs, ops::Range as ByteRange, path::PathBuf};
|
|||
|
||||
use futures::{FutureExt, StreamExt};
|
||||
use john_slop::{
|
||||
Config, ConfigLoader, ConfigPaths, apply_replacements, lsp::LspBackend, plan_replacements,
|
||||
Config, ConfigLoader, ConfigPaths, apply_replacements,
|
||||
lsp::{DiagnosticFilter, LspBackend},
|
||||
plan_replacements,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tempfile::TempDir;
|
||||
|
|
@ -24,12 +26,24 @@ struct Harness {
|
|||
|
||||
impl Harness {
|
||||
async fn new(explicit_config: Option<&str>) -> Self {
|
||||
Self::new_with_diagnostics(explicit_config, None).await
|
||||
Self::new_with_filter(explicit_config, None, DiagnosticFilter::UnicodeOnly).await
|
||||
}
|
||||
|
||||
async fn new_all(explicit_config: Option<&str>) -> Self {
|
||||
Self::new_with_filter(explicit_config, None, DiagnosticFilter::All).await
|
||||
}
|
||||
|
||||
async fn new_with_diagnostics(
|
||||
explicit_config: Option<&str>,
|
||||
diagnostic_catalog: Option<&str>,
|
||||
) -> Self {
|
||||
Self::new_with_filter(explicit_config, diagnostic_catalog, DiagnosticFilter::All).await
|
||||
}
|
||||
|
||||
async fn new_with_filter(
|
||||
explicit_config: Option<&str>,
|
||||
diagnostic_catalog: Option<&str>,
|
||||
diagnostic_filter: DiagnosticFilter,
|
||||
) -> Self {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let explicit = explicit_config.map(|contents| {
|
||||
|
|
@ -52,7 +66,9 @@ impl Harness {
|
|||
ConfigLoader::from_paths(paths).with_diagnostic_files(diagnostics.iter().cloned());
|
||||
let config = loader.load().unwrap();
|
||||
let (service, socket) = LspService::new(|client| {
|
||||
LspBackend::new(client, loader, config).expect("build test LSP backend")
|
||||
LspBackend::new(client, loader, config)
|
||||
.expect("build test LSP backend")
|
||||
.with_diagnostic_filter(diagnostic_filter)
|
||||
});
|
||||
let mut harness = Self {
|
||||
service,
|
||||
|
|
@ -163,7 +179,7 @@ fn open_params_with_language(uri: &str, language_id: &str, version: i32, text: &
|
|||
|
||||
#[tokio::test]
|
||||
async fn publishes_style_and_source_findings_without_non_fixable_actions() {
|
||||
let mut harness = Harness::new(None).await;
|
||||
let mut harness = Harness::new_all(None).await;
|
||||
let prose_uri = "file:///tmp/article.txt";
|
||||
let prose = "Moreover, the first claim.\nAdditionally, the second claim.";
|
||||
harness
|
||||
|
|
@ -256,6 +272,57 @@ async fn publishes_style_and_source_findings_without_non_fixable_actions() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_lsp_publishes_unicode_but_suppresses_other_findings() {
|
||||
let mut harness = Harness::new(None).await;
|
||||
let uri = "file:///tmp/quiet-default.txt";
|
||||
harness
|
||||
.notify(
|
||||
"textDocument/didOpen",
|
||||
open_params(uri, 1, "As an AI language model—I hope this helps."),
|
||||
)
|
||||
.await;
|
||||
|
||||
let published = harness
|
||||
.notification("textDocument/publishDiagnostics")
|
||||
.await;
|
||||
let diagnostics = published["diagnostics"].as_array().unwrap();
|
||||
assert_eq!(diagnostics.len(), 1);
|
||||
assert_eq!(diagnostics[0]["code"], "unicode-ascii-replacement");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn minimum_weight_keeps_unicode_and_only_heavier_non_unicode_findings() {
|
||||
let mut harness =
|
||||
Harness::new_with_filter(None, None, DiagnosticFilter::MinimumWeight(10.0)).await;
|
||||
harness
|
||||
.notify(
|
||||
"textDocument/didOpen",
|
||||
open_params(
|
||||
"file:///tmp/weighted.txt",
|
||||
1,
|
||||
"As an AI language model—I hope this helps.",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
let published = harness
|
||||
.notification("textDocument/publishDiagnostics")
|
||||
.await;
|
||||
let codes = published["diagnostics"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|diagnostic| diagnostic["code"].as_str().unwrap())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
codes,
|
||||
["assistant-self-identification", "unicode-ascii-replacement"]
|
||||
.into_iter()
|
||||
.collect()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unicode_review_findings_separate_safe_fixes_from_controls() {
|
||||
let mut harness = Harness::new(None).await;
|
||||
|
|
@ -630,7 +697,7 @@ enabled = true
|
|||
#[tokio::test]
|
||||
async fn disabled_category_suppresses_lsp_diagnostics() {
|
||||
let mut harness =
|
||||
Harness::new(Some("[diagnostics.categories]\nprompt-leakage = false\n")).await;
|
||||
Harness::new_all(Some("[diagnostics.categories]\nprompt-leakage = false\n")).await;
|
||||
harness
|
||||
.notify(
|
||||
"textDocument/didOpen",
|
||||
|
|
|
|||
Loading…
Reference in New Issue