test: add reviewed heuristic corpus
This commit is contained in:
parent
f8e2c9ad8e
commit
f1fe9f5d8d
|
|
@ -11,6 +11,8 @@ the details needed to use, embed, and develop John Slop:
|
|||
precedence, stable-ID merges, validation, and inspection commands.
|
||||
- [Heuristics and confidence](heuristics.md) documents rule execution,
|
||||
applicability, evidence damping, normalization, and score interpretation.
|
||||
- [Evaluation corpus](evaluation.md) records fixture provenance, per-rule
|
||||
precision/recall, aggregate false positives, and benchmark caveats.
|
||||
- [Development](development.md) covers the supported Rust version, feature
|
||||
matrix, snapshots, coverage, badges, and CI quality gates.
|
||||
- [Background](background.md) collects external context on observable signs of
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
# Evaluation corpus
|
||||
|
||||
Milestone 3 ships a small, reviewed characterization corpus under
|
||||
`tests/fixtures/evaluation`. Its text was authored for this repository and is
|
||||
covered by the MIT license. Labels name observable configured signals, never a
|
||||
presumed author or generation method.
|
||||
|
||||
With the embedded schema-version 1 catalog, the corpus currently reports:
|
||||
|
||||
| Rule | True positive | False positive | False negative | Precision | Recall |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| assistant-self-identification | 1 | 0 | 0 | 100% | 100% |
|
||||
| assistant-capability-disclaimer | 1 | 0 | 0 | 100% | 100% |
|
||||
| assistant-answer-preface | 1 | 0 | 0 | 100% | 100% |
|
||||
| knowledge-cutoff-disclaimer | 1 | 0 | 0 | 100% | 100% |
|
||||
| prompt-response-scaffolding | 1 | 0 | 0 | 100% | 100% |
|
||||
| templated-conclusion | 1 | 0 | 0 | 100% | 100% |
|
||||
| templated-filler | 1 | 0 | 0 | 100% | 100% |
|
||||
| repeated-transition | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% |
|
||||
| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% |
|
||||
| typography-non-ascii-hyphen | 1 | 0 | 0 | 100% | 100% |
|
||||
|
||||
Aggregate labeled-rule counts are 11 true positives, 0 false positives, and 0
|
||||
false negatives. The clean-document false-positive rate is 0/1 (0%). The test
|
||||
suite recomputes these values from the manifest and fails on drift.
|
||||
|
||||
These perfect values describe a tiny purpose-built regression set, not expected
|
||||
performance on real documents. They must not be used as an accuracy or
|
||||
authorship claim. Broader, independently sourced data and domain-specific review
|
||||
are required before tuning default weights or thresholds.
|
||||
2
tasks.md
2
tasks.md
|
|
@ -126,7 +126,7 @@ configuration is explainable, and all precedence/validation tests pass.
|
|||
- [x] Add table-driven unit tests for every shipped rule with positive examples
|
||||
and realistic near misses. Add tests for disabled rules, overrides, scope,
|
||||
caps, damping, threshold boundaries, and multilingual text.
|
||||
- [ ] Build a reviewed evaluation corpus that is legally distributable and labels
|
||||
- [x] Build a reviewed evaluation corpus that is legally distributable and labels
|
||||
observable low-care signals, not presumed author identity. Report per-rule
|
||||
precision/recall and aggregate false-positive rates before tuning defaults.
|
||||
- [ ] Document limitations, adversarial evasion, domain shift, and why the score
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
use std::{collections::BTreeSet, fs, path::Path};
|
||||
|
||||
use john_slop::{AnalysisContext, Config, Detector};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Manifest {
|
||||
schema_version: u32,
|
||||
license: String,
|
||||
cases: Vec<Case>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Case {
|
||||
path: String,
|
||||
kind: String,
|
||||
expected: BTreeSet<String>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation");
|
||||
let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap();
|
||||
let manifest: Manifest = toml::from_str(&manifest_text).unwrap();
|
||||
assert_eq!(manifest.schema_version, 1);
|
||||
assert_eq!(manifest.license, "MIT");
|
||||
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let mut observations = Vec::new();
|
||||
for case in manifest.cases {
|
||||
let text = fs::read_to_string(root.join(&case.path)).unwrap();
|
||||
let context = match case.kind.as_str() {
|
||||
"prose" | "source-code" => AnalysisContext::for_path(&case.path),
|
||||
other => panic!("unknown corpus kind {other}"),
|
||||
};
|
||||
let actual = detector
|
||||
.analyze_with(&text, &context)
|
||||
.findings
|
||||
.into_iter()
|
||||
.map(|finding| finding.rule_id.to_string())
|
||||
.collect::<BTreeSet<_>>();
|
||||
observations.push((case.path, case.expected, actual));
|
||||
}
|
||||
|
||||
let mut total_tp = 0;
|
||||
let mut total_fp = 0;
|
||||
let mut total_fn = 0;
|
||||
for rule in detector.rules() {
|
||||
let id = rule.id();
|
||||
let mut tp = 0;
|
||||
let mut fp = 0;
|
||||
let mut false_negatives = 0;
|
||||
for (_, expected, actual) in &observations {
|
||||
match (expected.contains(id), actual.contains(id)) {
|
||||
(true, true) => tp += 1,
|
||||
(false, true) => fp += 1,
|
||||
(true, false) => false_negatives += 1,
|
||||
(false, false) => {}
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"{id}: precision={}/{} recall={}/{}",
|
||||
tp,
|
||||
tp + fp,
|
||||
tp,
|
||||
tp + false_negatives
|
||||
);
|
||||
assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}");
|
||||
total_tp += tp;
|
||||
total_fp += fp;
|
||||
total_fn += false_negatives;
|
||||
}
|
||||
|
||||
let negative_cases = observations
|
||||
.iter()
|
||||
.filter(|(_, expected, _)| expected.is_empty())
|
||||
.count();
|
||||
let false_positive_cases = observations
|
||||
.iter()
|
||||
.filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty())
|
||||
.count();
|
||||
eprintln!(
|
||||
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
|
||||
);
|
||||
|
||||
assert_eq!((total_tp, total_fp, total_fn), (11, 0, 0));
|
||||
assert_eq!((false_positive_cases, negative_cases), (0, 1));
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# Evaluation fixtures
|
||||
|
||||
These short fixtures were authored specifically for John Slop and are
|
||||
distributed under the repository's MIT license. They label observable rule
|
||||
signals only; no label claims that a human or an AI system authored the text.
|
||||
|
||||
This is a characterization corpus for regression testing, not a representative
|
||||
benchmark. Additions require review for provenance, licensing, expected rule
|
||||
IDs, and realistic negative examples.
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Certainly! Here's a compact answer.
|
||||
|
||||
As an AI language model, I cannot browse the internet. My knowledge cutoff may
|
||||
also make current details unavailable.
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
// First, we parse the incoming value.
|
||||
fn parse() {
|
||||
// TODO: implement the parser
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
schema_version = 1
|
||||
license = "MIT"
|
||||
|
||||
[[cases]]
|
||||
path = "assistant.md"
|
||||
kind = "prose"
|
||||
expected = [
|
||||
"assistant-answer-preface",
|
||||
"assistant-capability-disclaimer",
|
||||
"assistant-self-identification",
|
||||
"knowledge-cutoff-disclaimer",
|
||||
]
|
||||
|
||||
[[cases]]
|
||||
path = "prompt.md"
|
||||
kind = "prose"
|
||||
expected = ["prompt-response-scaffolding"]
|
||||
|
||||
[[cases]]
|
||||
path = "style.md"
|
||||
kind = "prose"
|
||||
expected = ["repeated-transition", "templated-conclusion", "templated-filler"]
|
||||
|
||||
[[cases]]
|
||||
path = "example.rs"
|
||||
kind = "source-code"
|
||||
expected = ["source-placeholder-todo", "source-tutorial-narration"]
|
||||
|
||||
[[cases]]
|
||||
path = "typography.md"
|
||||
kind = "prose"
|
||||
expected = ["typography-non-ascii-hyphen"]
|
||||
|
||||
[[cases]]
|
||||
path = "negative.md"
|
||||
kind = "prose"
|
||||
expected = []
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
The measured request latency was 14 milliseconds in the local test run.
|
||||
|
||||
Le résumé français et la courte note japonaise 日本語 describe the same result.
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
User: Summarize the decision.
|
||||
The selected option keeps the parser deterministic.
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
It is worth noting that the cache is local.
|
||||
|
||||
Moreover, the first observation repeats the input.
|
||||
Additionally, the second observation repeats it again.
|
||||
|
||||
In conclusion, we can keep the existing limit.
|
||||
|
|
@ -0,0 +1 @@
|
|||
The draft joins two independent clauses—without reviewing the punctuation.
|
||||
Loading…
Reference in New Issue