john-slop/tests/evaluation.rs

157 lines
5.0 KiB
Rust

use std::{collections::BTreeSet, fs, path::Path};
use john_slop::{AnalysisContext, Config, Detector};
use serde::Deserialize;
use sha2::{Digest, Sha256};
#[derive(Deserialize)]
struct Manifest {
schema_version: u32,
license: String,
cases: Vec<Case>,
}
#[derive(Deserialize)]
struct Case {
path: String,
kind: String,
expected_score: u8,
expected: BTreeSet<String>,
#[serde(default)]
near_miss_for: BTreeSet<String>,
}
#[derive(Deserialize)]
struct Signoff {
schema_version: u32,
release: String,
reviewed_on: String,
status: String,
catalog_sha256: String,
limitations_acknowledged: bool,
decision: String,
}
#[test]
fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation");
let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap();
let manifest: Manifest = toml::from_str(&manifest_text).unwrap();
assert_eq!(manifest.schema_version, 2);
assert_eq!(manifest.license, "MIT");
let detector = Detector::new(&Config::embedded()).unwrap();
let mut observations = Vec::new();
let mut reviewed_near_misses = BTreeSet::new();
for case in manifest.cases {
let text = fs::read_to_string(root.join(&case.path)).unwrap();
let context = match case.kind.as_str() {
"prose" | "source-code" => AnalysisContext::for_path(&case.path),
other => panic!("unknown corpus kind {other}"),
};
let analysis = detector.analyze_with(&text, &context);
assert_eq!(
analysis.evidence_score.score(),
case.expected_score,
"evidence score for {}",
case.path
);
let actual = analysis
.findings
.into_iter()
.map(|finding| finding.rule_id.to_string())
.collect::<BTreeSet<_>>();
reviewed_near_misses.extend(case.near_miss_for);
observations.push((case.path, case.expected, actual));
}
let mut total_tp = 0;
let mut total_fp = 0;
let mut total_fn = 0;
for rule in detector.rules() {
let id = rule.id();
let mut tp = 0;
let mut fp = 0;
let mut false_negatives = 0;
for (_, expected, actual) in &observations {
match (expected.contains(id), actual.contains(id)) {
(true, true) => tp += 1,
(false, true) => fp += 1,
(true, false) => false_negatives += 1,
(false, false) => {}
}
}
eprintln!(
"{id}: precision={}/{} recall={}/{}",
tp,
tp + fp,
tp,
tp + false_negatives
);
let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1;
assert_eq!(
(tp, fp, false_negatives),
(expected_true_positives, 0, 0),
"metrics for {id}"
);
total_tp += tp;
total_fp += fp;
total_fn += false_negatives;
}
let negative_cases = observations
.iter()
.filter(|(_, expected, _)| expected.is_empty())
.count();
let false_positive_cases = observations
.iter()
.filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty())
.count();
eprintln!(
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
);
assert_eq!((total_tp, total_fp, total_fn), (13, 0, 0));
assert_eq!((false_positive_cases, negative_cases), (0, 5));
assert_eq!(
reviewed_near_misses,
detector
.rules()
.iter()
.map(|rule| rule.id().to_owned())
.collect()
);
let signoff: Signoff = toml::from_str(
&fs::read_to_string(root.join("signoff.toml")).expect("read evaluation sign-off"),
)
.expect("parse evaluation sign-off");
let mut catalog = Sha256::new();
catalog.update(include_bytes!(
"../crates/john-slop-core/assets/config.toml"
));
catalog.update([0]);
catalog.update(include_bytes!(
"../crates/john-slop-core/assets/heuristics.toml"
));
catalog.update([0]);
catalog.update(include_bytes!(
"../crates/john-slop-core/assets/unicode.toml"
));
let catalog_hash = catalog
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
assert_eq!(signoff.schema_version, 1);
assert_eq!(signoff.release, env!("CARGO_PKG_VERSION"));
assert_eq!(signoff.status, "accepted-after-diagnostic-policy");
assert_eq!(signoff.catalog_sha256, catalog_hash);
assert!(signoff.limitations_acknowledged);
assert_eq!(signoff.reviewed_on, "2026-09-30");
assert!(signoff.decision.contains("prompt-leakage"));
assert!(signoff.decision.contains("0.90"));
assert!(signoff.decision.contains("UTS #39"));
assert!(signoff.decision.contains("review"));
}