john-slop/tests/evaluation.rs

89 lines
2.8 KiB
Rust

use std::{collections::BTreeSet, fs, path::Path};
use john_slop::{AnalysisContext, Config, Detector};
use serde::Deserialize;
#[derive(Deserialize)]
struct Manifest {
schema_version: u32,
license: String,
cases: Vec<Case>,
}
#[derive(Deserialize)]
struct Case {
path: String,
kind: String,
expected: BTreeSet<String>,
}
#[test]
fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation");
let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap();
let manifest: Manifest = toml::from_str(&manifest_text).unwrap();
assert_eq!(manifest.schema_version, 1);
assert_eq!(manifest.license, "MIT");
let detector = Detector::new(&Config::embedded()).unwrap();
let mut observations = Vec::new();
for case in manifest.cases {
let text = fs::read_to_string(root.join(&case.path)).unwrap();
let context = match case.kind.as_str() {
"prose" | "source-code" => AnalysisContext::for_path(&case.path),
other => panic!("unknown corpus kind {other}"),
};
let actual = detector
.analyze_with(&text, &context)
.findings
.into_iter()
.map(|finding| finding.rule_id.to_string())
.collect::<BTreeSet<_>>();
observations.push((case.path, case.expected, actual));
}
let mut total_tp = 0;
let mut total_fp = 0;
let mut total_fn = 0;
for rule in detector.rules() {
let id = rule.id();
let mut tp = 0;
let mut fp = 0;
let mut false_negatives = 0;
for (_, expected, actual) in &observations {
match (expected.contains(id), actual.contains(id)) {
(true, true) => tp += 1,
(false, true) => fp += 1,
(true, false) => false_negatives += 1,
(false, false) => {}
}
}
eprintln!(
"{id}: precision={}/{} recall={}/{}",
tp,
tp + fp,
tp,
tp + false_negatives
);
assert_eq!((tp, fp, false_negatives), (1, 0, 0), "metrics for {id}");
total_tp += tp;
total_fp += fp;
total_fn += false_negatives;
}
let negative_cases = observations
.iter()
.filter(|(_, expected, _)| expected.is_empty())
.count();
let false_positive_cases = observations
.iter()
.filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty())
.count();
eprintln!(
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
);
assert_eq!((total_tp, total_fp, total_fn), (11, 0, 0));
assert_eq!((false_positive_cases, negative_cases), (0, 1));
}