feat: add research-backed review signals

This commit is contained in:
Dandaedre 2026-09-30 02:23:59 +02:00
parent 22308219c8
commit aedc130082
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
18 changed files with 324 additions and 21 deletions

View File

@ -93,8 +93,9 @@ Start with the guide closest to what you want to do:
with [commented configuration examples](examples/config/).
4. [Run the Rust and expected-failure examples](examples/README.md).
5. [Connect the formatter and LSP to Helix or Vim](docs/editors.md).
6. [Understand the heuristics and evidence score](docs/heuristics.md), then
review the [interpretation limits](docs/limitations.md).
6. [Understand the review-signal methodology](docs/review-signals.md), the
[heuristics and evidence score](docs/heuristics.md), and then the
[interpretation limits](docs/limitations.md).
The [documentation index](docs/README.md) offers the same guided path with a
little more context.

View File

@ -87,6 +87,38 @@ enabled = true
content = ["prose", "source-code"]
extensions = []
[[rules]]
id = "assistant-follow-up-offer"
category = "prompt-leakage"
severity = "warning"
matcher = "regex"
pattern = "(?i)\\b(?:let me know|tell me) if you(?:'d| would) like me to\\b"
weight = 12.0
message = "a conversational assistant follow-up offer appears in the source"
help = "Review whether a response-closing offer was copied into the document."
occurrence_cap = 1
enabled = true
[rules.scope]
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "assistant-revision-preface"
category = "prompt-leakage"
severity = "warning"
matcher = "regex"
pattern = "(?i)\\bhere(?:'s| is) (?:a|the) (?:revised|polished|rewritten) (?:version|draft)\\b"
weight = 12.0
message = "an assistant-style revision preface appears in the source"
help = "Remove conversational revision scaffolding if it is not part of the document."
occurrence_cap = 1
enabled = true
[rules.scope]
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "templated-conclusion"
category = "style"
@ -135,6 +167,54 @@ enabled = true
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "editorial-insertion-placeholder"
category = "style"
severity = "warning"
matcher = "regex"
pattern = "(?i)\\[(?:insert|add) (?:citation|reference|source|example|details?)(?: here)?\\]"
weight = 10.0
message = "an unresolved editorial insertion placeholder appears in the source"
help = "Supply the requested material or remove the placeholder before publishing."
occurrence_cap = 2
enabled = true
[rules.scope]
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "stock-helpful-closing"
category = "style"
severity = "advice"
matcher = "regex"
pattern = "(?im)^[ \\t]*I hope this helps[.!][ \\t]*$"
weight = 2.0
message = "a stock conversational closing appears in the source"
help = "Keep the closing only when the document is intentionally conversational."
occurrence_cap = 1
enabled = true
[rules.scope]
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "excess-vocabulary-cluster"
category = "style"
severity = "advice"
matcher = "repeated-regex"
pattern = "(?i)\\b(?:delv(?:e|es|ed|ing)|intricate|multifaceted|pivotal|underscore[sd]?|realm|landscape)\\b"
weight = 1.0
message = "several terms from a population-level excess-vocabulary cluster appear"
help = "This weak corpus-level clue is not authorship evidence; review only for fit and specificity."
occurrence_cap = 3
enabled = true
[rules.scope]
content = ["prose"]
extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "source-placeholder-todo"
category = "source-code"
@ -166,3 +246,35 @@ enabled = true
[rules.scope]
content = ["source-code"]
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
[[rules]]
id = "source-omitted-code-placeholder"
category = "source-code"
severity = "warning"
matcher = "regex"
pattern = "(?im)^[ \\t]*(?://|#)[ \\t]*(?:\\.\\.\\.[ \\t]*)?(?:existing|remaining|rest of (?:the )?)[ \\t]+code(?:[ \\t]+(?:here|remains?|unchanged))?[ \\t]*$"
weight = 10.0
message = "an omitted-code placeholder remains in source code"
help = "Restore the omitted implementation or remove the placeholder."
occurrence_cap = 2
enabled = true
[rules.scope]
content = ["source-code"]
extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
[[rules]]
id = "source-markdown-fence"
category = "source-code"
severity = "advice"
matcher = "regex"
pattern = "(?m)^[ \\t]*```(?:[A-Za-z0-9_+-]+)?[ \\t]*$"
weight = 6.0
message = "a Markdown code fence appears in a source-code file"
help = "Remove response-formatting fences unless they are intentionally embedded data."
occurrence_cap = 2
enabled = true
[rules.scope]
content = ["source-code"]
extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]

View File

@ -1466,7 +1466,7 @@ mod tests {
fn embedded_heuristics_have_complete_metadata() {
let config = Config::embedded();
assert_eq!(config.rules.len(), 12);
assert_eq!(config.rules.len(), 19);
assert_eq!(config.thresholds.low, 20);
assert_eq!(config.thresholds.medium, 50);
assert_eq!(config.thresholds.high, 75);
@ -1572,7 +1572,7 @@ scope = { content = ["prose", "source-code"], extensions = [] }
assert!(!rule.enabled);
assert_eq!(rule.weight, 11.0);
assert_eq!(config.rules.len(), 13);
assert_eq!(config.rules.len(), 20);
}
#[test]
@ -1788,7 +1788,7 @@ weight = 9.5
.enabled
);
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
assert_eq!(config.rules.len(), 12);
assert_eq!(config.rules.len(), 19);
let heuristic = config
.rules
.iter()

View File

@ -21,9 +21,11 @@ internals to use the tools.
diagnostic showcase.
5. [Editor setup](editors.md) provides working Helix (`hx`) and Vim formatter
and LSP configurations with validation steps.
6. [Heuristics and evidence scoring](heuristics.md) explains what each rule
6. [Review signals and evidence methodology](review-signals.md) explains the
research boundary, evidence tiers, and deliberately excluded proxies.
7. [Heuristics and evidence scoring](heuristics.md) explains what each rule
contributes and how the bounded score is calculated.
7. [Limitations](limitations.md) covers false positives, evasion, domain shift,
8. [Limitations](limitations.md) covers false positives, evasion, domain shift,
score semantics, and safe interpretation.
## Go deeper

View File

@ -10,6 +10,8 @@ points for day-to-day use.
- [Heuristics and evidence scoring](heuristics.md) describes applicability,
occurrence damping, normalization, and why the score is not authorship
confidence.
- [Review signals and evidence methodology](review-signals.md) records the
research basis, evidence tiers, and signals deliberately kept out.
- [Evaluation](evaluation.md) records corpus provenance, per-rule measurements,
aggregate false positives, and the limits of the current metrics.
- [Limitations](limitations.md) covers evasion, domain shift, interpretation,

View File

@ -14,15 +14,22 @@ With the embedded schema-version 1 catalog, the corpus currently reports:
| assistant-answer-preface | 1 | 0 | 0 | 100% | 100% |
| knowledge-cutoff-disclaimer | 1 | 0 | 0 | 100% | 100% |
| prompt-response-scaffolding | 1 | 0 | 0 | 100% | 100% |
| assistant-follow-up-offer | 1 | 0 | 0 | 100% | 100% |
| assistant-revision-preface | 1 | 0 | 0 | 100% | 100% |
| templated-conclusion | 1 | 0 | 0 | 100% | 100% |
| templated-filler | 1 | 0 | 0 | 100% | 100% |
| repeated-transition | 1 | 0 | 0 | 100% | 100% |
| editorial-insertion-placeholder | 1 | 0 | 0 | 100% | 100% |
| stock-helpful-closing | 1 | 0 | 0 | 100% | 100% |
| excess-vocabulary-cluster | 1 | 0 | 0 | 100% | 100% |
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
| source-omitted-code-placeholder | 1 | 0 | 0 | 100% | 100% |
| source-markdown-fence | 1 | 0 | 0 | 100% | 100% |
| unicode-ascii-replacement | 2 | 0 | 0 | 100% | 100% |
| unicode-invisible-control | 1 | 0 | 0 | 100% | 100% |
Aggregate labeled-rule counts are 13 true positives, 0 false positives, and 0
Aggregate labeled-rule counts are 20 true positives, 0 false positives, and 0
false negatives. The clean-document false-positive rate is 0/5 (0%). The test
suite recomputes these values from the manifest and fails on drift.
@ -30,10 +37,10 @@ The same manifest pins evidence scores for regression characterization:
| Corpus case | Score |
| --- | ---: |
| four assistant disclaimer/preface rules | 60 |
| six assistant disclaimer/preface rules | 64 |
| one prompt/response marker | 33 |
| style signals | 19 |
| source-code signals | 24 |
| style signals | 25 |
| source-code signals | 33 |
| typography signal | 17 |
| one ASCII lookalike and one invisible character | 28 |
| each of five negative documents | 0 |
@ -54,6 +61,9 @@ replaceable by default, retains a non-mutating `review` policy, and keeps the
Gitea-compatible invisible/control profile diagnostic-only, with explicit
multilingual limitations. The Milestone 25 review gives one-to-one fullwidth
ASCII compatibility folds precedence over visual skeletons under every policy.
The Milestone 26 review adds narrow response and unfinished-editing residue,
keeps population-level vocabulary and stock closings deliberately low-weight,
and excludes language-proficiency proxies.
The checked-in `signoff.toml` records those decisions
and contains the SHA-256 digest of `config.toml`, `heuristics.toml`, and
`unicode.toml`. This binds the reviewed scoring behavior to both rule catalogs

View File

@ -19,7 +19,8 @@ and LSP publication as embedded rules. New matcher algorithms and categories
remain code changes rather than executable configuration.
The generated Unicode 16.0 profile maps every non-ASCII scalar with a printable
ASCII UTS #39 skeleton, alongside ten manually reviewed punctuation mappings.
ASCII UTS #39 skeleton after explicitly reviewed compatibility-width and
punctuation mappings.
The default `replace` policy gives each of these findings a formatter/LSP edit;
`review` keeps the same exact diagnostics without edits, and `off` suppresses
the generated lookalikes. Invisible and direction-changing controls remain

106
docs/review-signals.md Normal file
View File

@ -0,0 +1,106 @@
# Review signals and evidence methodology
John Slop finds concrete text that may deserve review. It does **not** determine
whether AI wrote a document, identify a model, or assign an authorship
probability. A finding names the exact matched text and the configured reason
for reviewing it; the evidence score is only a bounded aggregation of those
findings.
This boundary matters because general-purpose AI-text detection is not reliable
enough to support an authorship claim. Sadasivan et al. show both practical
paraphrasing attacks and a theoretical trade-off between detection and false
positives as human and model distributions become closer [1]. A comparative
evaluation of 14 detection tools likewise found them neither accurate nor
reliable, with performance worsening after obfuscation [2]. Liang et al. found
that perplexity-based detectors disproportionately classified non-native English
writing as AI-generated [3]. John Slop therefore does not use perplexity,
burstiness, grammar, spelling, vocabulary richness, dialect, or English fluency
as evidence.
## Signal tiers
### Direct response residue
These are relatively strong, locally explainable signs that conversational
material may have been pasted into a document:
- assistant self-identification, capability, or knowledge-cutoff disclaimers;
- `User:` / `Assistant:` transcript scaffolding;
- answer or revision prefaces; and
- follow-up offers such as asking whether the reader wants an expansion.
They still do not prove authorship. A quotation, article about assistants, or
intentional transcript can contain the same text, so applicability scopes and
near-miss fixtures are part of every rule's review.
### Unfinished editing and source residue
Unresolved insertion notes, implementation placeholders, omitted-code comments,
tutorial narration, and stray Markdown fences are useful low-care signals
regardless of how they were produced. These rules ask whether work is complete;
they do not infer who or what produced it.
### Subtle population-level clues
Some patterns are meaningful across a large corpus but weak in one document.
Kobak et al. measured abrupt, population-level increases in selected style words
across more than 15 million biomedical abstracts [4]. John Slop represents this
only as `excess-vocabulary-cluster`: it requires multiple matches, has weight
`1.0`, is occurrence-capped, and says explicitly that it is not authorship
evidence. A stock closing has similarly low weight. These clues should prompt a
specificity edit at most, never an accusation.
Repeated transitions and templated phrases sit between the tiers. They activate
only on an exact conservative pattern; repetition from one rule is damped and
cannot alone create a high evidence band.
### Verification targets, not offline shape rules
Generated references and package names can be wrong. Walters and Wilder found
fabricated and substantially erroneous citations in a controlled set of
generated bibliographies [5]. Spracklen et al. measured package hallucinations
in a large code-generation study [6]. But a citation's typography or a package
name's shape cannot establish that it is fictitious. Reliable checks require an
authoritative catalog and contextual validation, while John Slop analysis is
local and makes no network calls. These remain human review targets rather than
shipped heuristic rules.
## How a rule is admitted
A shipped rule must have a stable ID, a narrow pattern, a rationale, review
help, an applicability scope, a finite non-negative weight, and an occurrence
cap. It also needs:
1. a positive fixture showing the intended match;
2. a realistic near miss showing what must not match;
3. deterministic byte spans and ordering; and
4. review of likely harm, especially language-proficiency and demographic
proxies.
Exact residue and fixable Unicode receive higher confidence than broad stylistic
clues. New research is used to propose testable review signals, not to import a
paper's population-level conclusion into an individual-document verdict.
## Scoring and editor visibility
Rule weight contributes bounded evidence to the shared score. Per-rule caps and
correlation damping prevent repeated copies of one pattern from dominating it.
The number is neither accuracy nor probability; see
[Heuristics and evidence scoring](heuristics.md) for the formula.
LSP visibility is a separate presentation choice. By default the editor shows
only configured Unicode findings, which are exact and may have safe replacement
edits. `--all-diagnostics` enables the complete detector catalog for testing or
deliberate review. `--minimum-diagnostic-weight <WEIGHT>` is the quieter opt-in:
it keeps Unicode findings and adds only non-Unicode findings whose configured
weight meets the threshold. Filtering affects editor diagnostics, not CLI
analysis or scoring.
## References
1. Sadasivan et al., [Can AI-Generated Text be Reliably Detected?](https://arxiv.org/abs/2303.11156)
2. Weber-Wulff et al., [Testing of detection tools for AI-generated text](https://link.springer.com/article/10.1007/s40979-023-00146-z)
3. Liang et al., [GPT detectors are biased against non-native English writers](https://pmc.ncbi.nlm.nih.gov/articles/PMC10382961/)
4. Kobak et al., [Delving into LLM-assisted writing in biomedical publications through excess vocabulary](https://arxiv.org/abs/2406.07016)
5. Walters and Wilder, [Fabrication and errors in the bibliographic citations generated by ChatGPT](https://pmc.ncbi.nlm.nih.gov/articles/PMC10484980/)
6. Spracklen et al., [We Have a Package for You! A Comprehensive Analysis of Package Hallucinations by Code Generating LLMs](https://www.usenix.org/conference/usenixsecurity25/presentation/spracklen)

View File

@ -666,6 +666,33 @@ evaluation, dependency policy/audit checks, Unicode-catalog reproducibility,
and workspace package assembly pass; coverage is 92.84% (1,583/1,705 lines),
and all release metadata and binary versions agree on `0.1.25`.
## Milestone 26 — Evidence tiers and quiet editor defaults
- [x] Document the research boundary between directly reviewable residue,
low-confidence population-level signals, and unreliable authorship inference.
Add conservative data-driven rules with positive and near-miss evaluation
fixtures, while explicitly excluding fluency and perplexity proxies.
- [ ] Make LSP diagnostics Unicode-only by default. Add an
`--all-diagnostics` testing switch and a minimum-weight opt-in that always
retains Unicode findings while suppressing lighter non-Unicode signals.
Cover the policy through unit, protocol, and CLI-help tests.
- [ ] Update editor/configuration guidance, changelog, evaluation sign-off,
release metadata, coverage, badges, and recap; run all quality, audit,
regeneration, and package gates.
Decision: rule weight remains bounded evidence for scoring, not a probability
of AI authorship. LSP visibility is a separate presentation policy: Unicode
findings are shown by default because they are exact and fixable; non-Unicode
rules require an explicit all-diagnostics switch or minimum-weight threshold.
This keeps subtle research-derived vocabulary and style clues out of the
default editor experience without weakening CLI analysis or configuration.
Acceptance: the methodology cites primary evidence and states what the tool
will not infer; each new rule has reviewed positive and near-miss coverage; the
default LSP publishes only Unicode findings, both opt-in modes are deterministic,
and all release artifacts agree on `0.1.26` before the immutable `v0.1.26` tag
is created.
## Deferred by design
- Statistical or model-based authorship classification.

View File

@ -111,7 +111,7 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
);
assert_eq!((total_tp, total_fp, total_fn), (13, 0, 0));
assert_eq!((total_tp, total_fp, total_fn), (20, 0, 0));
assert_eq!((false_positive_cases, negative_cases), (0, 5));
assert_eq!(
reviewed_near_misses,

View File

@ -2,3 +2,5 @@ Certainly! Here's a compact answer.
As an AI language model, I cannot browse the internet. My knowledge cutoff may
also make current details unavailable.
Here's the revised version. Let me know if you'd like me to expand it.

View File

@ -2,3 +2,6 @@
fn parse() {
// TODO: implement the parser
}
// ... remaining code here
```rust

View File

@ -4,10 +4,12 @@ license = "MIT"
[[cases]]
path = "assistant.md"
kind = "prose"
expected_score = 60
expected_score = 64
expected = [
"assistant-answer-preface",
"assistant-capability-disclaimer",
"assistant-follow-up-offer",
"assistant-revision-preface",
"assistant-self-identification",
"knowledge-cutoff-disclaimer",
]
@ -21,14 +23,26 @@ expected = ["prompt-response-scaffolding"]
[[cases]]
path = "style.md"
kind = "prose"
expected_score = 19
expected = ["repeated-transition", "templated-conclusion", "templated-filler"]
expected_score = 25
expected = [
"editorial-insertion-placeholder",
"excess-vocabulary-cluster",
"repeated-transition",
"stock-helpful-closing",
"templated-conclusion",
"templated-filler",
]
[[cases]]
path = "example.rs"
kind = "source-code"
expected_score = 24
expected = ["source-placeholder-todo", "source-tutorial-narration"]
expected_score = 33
expected = [
"source-markdown-fence",
"source-omitted-code-placeholder",
"source-placeholder-todo",
"source-tutorial-narration",
]
[[cases]]
path = "typography.md"
@ -60,6 +74,8 @@ expected = []
near_miss_for = [
"assistant-answer-preface",
"assistant-capability-disclaimer",
"assistant-follow-up-offer",
"assistant-revision-preface",
"assistant-self-identification",
"knowledge-cutoff-disclaimer",
]
@ -77,7 +93,10 @@ kind = "prose"
expected_score = 0
expected = []
near_miss_for = [
"editorial-insertion-placeholder",
"excess-vocabulary-cluster",
"repeated-transition",
"stock-helpful-closing",
"templated-conclusion",
"templated-filler",
]
@ -87,4 +106,9 @@ path = "negative-source.rs"
kind = "source-code"
expected_score = 0
expected = []
near_miss_for = ["source-placeholder-todo", "source-tutorial-narration"]
near_miss_for = [
"source-markdown-fence",
"source-omitted-code-placeholder",
"source-placeholder-todo",
"source-tutorial-narration",
]

View File

@ -1,3 +1,6 @@
The editor described a model disclosure in original language and verified the
browser limitation against the tool documentation. The dataset date was cited
directly rather than presented as a generic cutoff disclaimer.
The editor asked whether readers would like a revised draft, but the published
article begins directly with its findings rather than retaining that exchange.

View File

@ -3,3 +3,6 @@
fn parse_identifier(input: &str) -> &str {
input.trim()
}
const MARKDOWN_FENCE: &str = "three backticks";
// The remaining code validates the parsed identifier below.

View File

@ -1,2 +1,5 @@
Moreover, the first measurement includes its sample size. The conclusion names
the retained limit and the observed latency instead of repeating a template.
The landscape photograph shows a complex rock formation. The editor resolved
the citation request, and the support agent wrote that the answer should help.

View File

@ -2,6 +2,6 @@ schema_version = 1
release = "0.1.25"
reviewed_on = "2026-09-30"
status = "accepted-with-runtime-catalog-boundary"
catalog_sha256 = "fa1a26bfa40e7336a4930f46c5511a2ea3223cf3e7a373bb830372eaa523a020"
catalog_sha256 = "da34ce00dfd2d801e772fa440d932abb98c929f5ad08ff3ad5e47e0ed4d84d5a"
limitations_acknowledged = true
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Keep every diagnostic category enabled by default while allowing layered category gates and individual rule switches. Permit declarative runtime catalogs to add exact-phrase, regex, and repeated-regex rules, but reject code-backed matcher engines. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Give reviewed fullwidth ASCII compatibility folds precedence over visual UTS #39 skeletons under every confusable policy. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."
decision = "Retain category-specific correlation damping: prompt-leakage uses 0.90 per later distinct rule while other families retain 0.75. Keep every diagnostic category enabled by default while allowing layered category gates and individual rule switches. Permit declarative runtime catalogs to add exact-phrase, regex, and repeated-regex rules, but reject code-backed matcher engines. Use generated Unicode 16.0 UTS #39 printable-ASCII mappings in replace mode by default, retain review and off policies for multilingual usage, and keep pinned Gitea invisible/control findings non-mutating. Give reviewed fullwidth ASCII compatibility folds precedence over visual UTS #39 skeletons under every confusable policy. Admit direct response and unfinished-editing residue with narrow patterns, but keep population-level excess vocabulary and stock closings at low weight; reject fluency, grammar, dialect, and perplexity proxies. Reviewed corpus scores are pinned with 0/5 observed false-positive documents; the small authored sample does not establish real-world or multilingual accuracy."

View File

@ -4,3 +4,7 @@ Moreover, the first observation repeats the input.
Additionally, the second observation repeats it again.
In conclusion, we can keep the existing limit.
The report delves into a multifaceted landscape. [Insert citation here]
I hope this helps.