use std::{collections::BTreeSet, fs, path::Path}; use john_slop::{AnalysisContext, Config, Detector}; use serde::Deserialize; #[derive(Deserialize)] struct Manifest { schema_version: u32, license: String, cases: Vec, } #[derive(Deserialize)] struct Case { path: String, kind: String, expected: BTreeSet, } #[test] fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation"); let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap(); let manifest: Manifest = toml::from_str(&manifest_text).unwrap(); assert_eq!(manifest.schema_version, 1); assert_eq!(manifest.license, "MIT"); let detector = Detector::new(&Config::embedded()).unwrap(); let mut observations = Vec::new(); for case in manifest.cases { let text = fs::read_to_string(root.join(&case.path)).unwrap(); let context = match case.kind.as_str() { "prose" | "source-code" => AnalysisContext::for_path(&case.path), other => panic!("unknown corpus kind {other}"), }; let actual = detector .analyze_with(&text, &context) .findings .into_iter() .map(|finding| finding.rule_id.to_string()) .collect::>(); observations.push((case.path, case.expected, actual)); } let mut total_tp = 0; let mut total_fp = 0; let mut total_fn = 0; for rule in detector.rules() { let id = rule.id(); let mut tp = 0; let mut fp = 0; let mut false_negatives = 0; for (_, expected, actual) in &observations { match (expected.contains(id), actual.contains(id)) { (true, true) => tp += 1, (false, true) => fp += 1, (true, false) => false_negatives += 1, (false, false) => {} } } eprintln!( "{id}: precision={}/{} recall={}/{}", tp, tp + fp, tp, tp + false_negatives ); assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}"); total_tp += tp; total_fp += fp; total_fn += false_negatives; } let negative_cases = observations .iter() .filter(|(_, expected, _)| expected.is_empty()) .count(); let false_positive_cases = observations .iter() .filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty()) .count(); eprintln!( "aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}" ); assert_eq!((total_tp, total_fp, total_fn), (11, 0, 0)); assert_eq!((false_positive_cases, negative_cases), (0, 1)); }