diff --git a/docs/README.md b/docs/README.md index a80c0d0..7eca845 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,8 @@ the details needed to use, embed, and develop John Slop: precedence, stable-ID merges, validation, and inspection commands. - [Heuristics and confidence](heuristics.md) documents rule execution, applicability, evidence damping, normalization, and score interpretation. +- [Evaluation corpus](evaluation.md) records fixture provenance, per-rule + precision/recall, aggregate false positives, and benchmark caveats. - [Development](development.md) covers the supported Rust version, feature matrix, snapshots, coverage, badges, and CI quality gates. - [Background](background.md) collects external context on observable signs of diff --git a/docs/evaluation.md b/docs/evaluation.md new file mode 100644 index 0000000..f15eb64 --- /dev/null +++ b/docs/evaluation.md @@ -0,0 +1,31 @@ +# Evaluation corpus + +Milestone 3 ships a small, reviewed characterization corpus under +`tests/fixtures/evaluation`. Its text was authored for this repository and is +covered by the MIT license. Labels name observable configured signals, never a +presumed author or generation method. + +With the embedded schema-version 1 catalog, the corpus currently reports: + +| Rule | True positive | False positive | False negative | Precision | Recall | +| --- | ---: | ---: | ---: | ---: | ---: | +| assistant-self-identification | 1 | 0 | 0 | 100% | 100% | +| assistant-capability-disclaimer | 1 | 0 | 0 | 100% | 100% | +| assistant-answer-preface | 1 | 0 | 0 | 100% | 100% | +| knowledge-cutoff-disclaimer | 1 | 0 | 0 | 100% | 100% | +| prompt-response-scaffolding | 1 | 0 | 0 | 100% | 100% | +| templated-conclusion | 1 | 0 | 0 | 100% | 100% | +| templated-filler | 1 | 0 | 0 | 100% | 100% | +| repeated-transition | 1 | 0 | 0 | 100% | 100% | +| source-placeholder-todo | 1 | 0 | 0 | 100% | 100% | +| source-tutorial-narration | 1 | 0 | 0 | 100% | 100% | +| typography-non-ascii-hyphen | 1 | 0 | 0 | 100% | 100% | + +Aggregate labeled-rule counts are 11 true positives, 0 false positives, and 0 +false negatives. The clean-document false-positive rate is 0/1 (0%). The test +suite recomputes these values from the manifest and fails on drift. + +These perfect values describe a tiny purpose-built regression set, not expected +performance on real documents. They must not be used as an accuracy or +authorship claim. Broader, independently sourced data and domain-specific review +are required before tuning default weights or thresholds. diff --git a/tasks.md b/tasks.md index a0dd38b..accaf89 100644 --- a/tasks.md +++ b/tasks.md @@ -126,7 +126,7 @@ configuration is explainable, and all precedence/validation tests pass. - [x] Add table-driven unit tests for every shipped rule with positive examples and realistic near misses. Add tests for disabled rules, overrides, scope, caps, damping, threshold boundaries, and multilingual text. -- [ ] Build a reviewed evaluation corpus that is legally distributable and labels +- [x] Build a reviewed evaluation corpus that is legally distributable and labels observable low-care signals, not presumed author identity. Report per-rule precision/recall and aggregate false-positive rates before tuning defaults. - [ ] Document limitations, adversarial evasion, domain shift, and why the score diff --git a/tests/evaluation.rs b/tests/evaluation.rs new file mode 100644 index 0000000..268546f --- /dev/null +++ b/tests/evaluation.rs @@ -0,0 +1,88 @@ +use std::{collections::BTreeSet, fs, path::Path}; + +use john_slop::{AnalysisContext, Config, Detector}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Manifest { + schema_version: u32, + license: String, + cases: Vec, +} + +#[derive(Deserialize)] +struct Case { + path: String, + kind: String, + expected: BTreeSet, +} + +#[test] +fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation"); + let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap(); + let manifest: Manifest = toml::from_str(&manifest_text).unwrap(); + assert_eq!(manifest.schema_version, 1); + assert_eq!(manifest.license, "MIT"); + + let detector = Detector::new(&Config::embedded()).unwrap(); + let mut observations = Vec::new(); + for case in manifest.cases { + let text = fs::read_to_string(root.join(&case.path)).unwrap(); + let context = match case.kind.as_str() { + "prose" | "source-code" => AnalysisContext::for_path(&case.path), + other => panic!("unknown corpus kind {other}"), + }; + let actual = detector + .analyze_with(&text, &context) + .findings + .into_iter() + .map(|finding| finding.rule_id.to_string()) + .collect::>(); + observations.push((case.path, case.expected, actual)); + } + + let mut total_tp = 0; + let mut total_fp = 0; + let mut total_fn = 0; + for rule in detector.rules() { + let id = rule.id(); + let mut tp = 0; + let mut fp = 0; + let mut false_negatives = 0; + for (_, expected, actual) in &observations { + match (expected.contains(id), actual.contains(id)) { + (true, true) => tp += 1, + (false, true) => fp += 1, + (true, false) => false_negatives += 1, + (false, false) => {} + } + } + eprintln!( + "{id}: precision={}/{} recall={}/{}", + tp, + tp + fp, + tp, + tp + false_negatives + ); + assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}"); + total_tp += tp; + total_fp += fp; + total_fn += false_negatives; + } + + let negative_cases = observations + .iter() + .filter(|(_, expected, _)| expected.is_empty()) + .count(); + let false_positive_cases = observations + .iter() + .filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty()) + .count(); + eprintln!( + "aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}" + ); + + assert_eq!((total_tp, total_fp, total_fn), (11, 0, 0)); + assert_eq!((false_positive_cases, negative_cases), (0, 1)); +} diff --git a/tests/fixtures/evaluation/README.md b/tests/fixtures/evaluation/README.md new file mode 100644 index 0000000..58cb248 --- /dev/null +++ b/tests/fixtures/evaluation/README.md @@ -0,0 +1,9 @@ +# Evaluation fixtures + +These short fixtures were authored specifically for John Slop and are +distributed under the repository's MIT license. They label observable rule +signals only; no label claims that a human or an AI system authored the text. + +This is a characterization corpus for regression testing, not a representative +benchmark. Additions require review for provenance, licensing, expected rule +IDs, and realistic negative examples. diff --git a/tests/fixtures/evaluation/assistant.md b/tests/fixtures/evaluation/assistant.md new file mode 100644 index 0000000..dfafeee --- /dev/null +++ b/tests/fixtures/evaluation/assistant.md @@ -0,0 +1,4 @@ +Certainly! Here's a compact answer. + +As an AI language model, I cannot browse the internet. My knowledge cutoff may +also make current details unavailable. diff --git a/tests/fixtures/evaluation/example.rs b/tests/fixtures/evaluation/example.rs new file mode 100644 index 0000000..d53dd02 --- /dev/null +++ b/tests/fixtures/evaluation/example.rs @@ -0,0 +1,4 @@ +// First, we parse the incoming value. +fn parse() { + // TODO: implement the parser +} diff --git a/tests/fixtures/evaluation/manifest.toml b/tests/fixtures/evaluation/manifest.toml new file mode 100644 index 0000000..05389df --- /dev/null +++ b/tests/fixtures/evaluation/manifest.toml @@ -0,0 +1,37 @@ +schema_version = 1 +license = "MIT" + +[[cases]] +path = "assistant.md" +kind = "prose" +expected = [ + "assistant-answer-preface", + "assistant-capability-disclaimer", + "assistant-self-identification", + "knowledge-cutoff-disclaimer", +] + +[[cases]] +path = "prompt.md" +kind = "prose" +expected = ["prompt-response-scaffolding"] + +[[cases]] +path = "style.md" +kind = "prose" +expected = ["repeated-transition", "templated-conclusion", "templated-filler"] + +[[cases]] +path = "example.rs" +kind = "source-code" +expected = ["source-placeholder-todo", "source-tutorial-narration"] + +[[cases]] +path = "typography.md" +kind = "prose" +expected = ["typography-non-ascii-hyphen"] + +[[cases]] +path = "negative.md" +kind = "prose" +expected = [] diff --git a/tests/fixtures/evaluation/negative.md b/tests/fixtures/evaluation/negative.md new file mode 100644 index 0000000..4eaf848 --- /dev/null +++ b/tests/fixtures/evaluation/negative.md @@ -0,0 +1,3 @@ +The measured request latency was 14 milliseconds in the local test run. + +Le résumé français et la courte note japonaise 日本語 describe the same result. diff --git a/tests/fixtures/evaluation/prompt.md b/tests/fixtures/evaluation/prompt.md new file mode 100644 index 0000000..91ad3c9 --- /dev/null +++ b/tests/fixtures/evaluation/prompt.md @@ -0,0 +1,2 @@ +User: Summarize the decision. +The selected option keeps the parser deterministic. diff --git a/tests/fixtures/evaluation/style.md b/tests/fixtures/evaluation/style.md new file mode 100644 index 0000000..2439615 --- /dev/null +++ b/tests/fixtures/evaluation/style.md @@ -0,0 +1,6 @@ +It is worth noting that the cache is local. + +Moreover, the first observation repeats the input. +Additionally, the second observation repeats it again. + +In conclusion, we can keep the existing limit. diff --git a/tests/fixtures/evaluation/typography.md b/tests/fixtures/evaluation/typography.md new file mode 100644 index 0000000..5a00467 --- /dev/null +++ b/tests/fixtures/evaluation/typography.md @@ -0,0 +1 @@ +The draft joins two independent clauses—without reviewing the punctuation.