153 lines
4.9 KiB
Rust
153 lines
4.9 KiB
Rust
use std::{collections::BTreeSet, fs, path::Path};
|
|
|
|
use john_slop::{AnalysisContext, Config, Detector};
|
|
use serde::Deserialize;
|
|
use sha2::{Digest, Sha256};
|
|
|
|
#[derive(Deserialize)]
|
|
struct Manifest {
|
|
schema_version: u32,
|
|
license: String,
|
|
cases: Vec<Case>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct Case {
|
|
path: String,
|
|
kind: String,
|
|
expected_score: u8,
|
|
expected: BTreeSet<String>,
|
|
#[serde(default)]
|
|
near_miss_for: BTreeSet<String>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct Signoff {
|
|
schema_version: u32,
|
|
release: String,
|
|
reviewed_on: String,
|
|
status: String,
|
|
catalog_sha256: String,
|
|
limitations_acknowledged: bool,
|
|
decision: String,
|
|
}
|
|
|
|
#[test]
|
|
fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/evaluation");
|
|
let manifest_text = fs::read_to_string(root.join("manifest.toml")).unwrap();
|
|
let manifest: Manifest = toml::from_str(&manifest_text).unwrap();
|
|
assert_eq!(manifest.schema_version, 2);
|
|
assert_eq!(manifest.license, "MIT");
|
|
|
|
let detector = Detector::new(&Config::embedded()).unwrap();
|
|
let mut observations = Vec::new();
|
|
let mut reviewed_near_misses = BTreeSet::new();
|
|
for case in manifest.cases {
|
|
let text = fs::read_to_string(root.join(&case.path)).unwrap();
|
|
let context = match case.kind.as_str() {
|
|
"prose" | "source-code" => AnalysisContext::for_path(&case.path),
|
|
other => panic!("unknown corpus kind {other}"),
|
|
};
|
|
let analysis = detector.analyze_with(&text, &context);
|
|
assert_eq!(
|
|
analysis.evidence_score.score(),
|
|
case.expected_score,
|
|
"evidence score for {}",
|
|
case.path
|
|
);
|
|
let actual = analysis
|
|
.findings
|
|
.into_iter()
|
|
.map(|finding| finding.rule_id.to_string())
|
|
.collect::<BTreeSet<_>>();
|
|
reviewed_near_misses.extend(case.near_miss_for);
|
|
observations.push((case.path, case.expected, actual));
|
|
}
|
|
|
|
let mut total_tp = 0;
|
|
let mut total_fp = 0;
|
|
let mut total_fn = 0;
|
|
for rule in detector.rules() {
|
|
let id = rule.id();
|
|
let mut tp = 0;
|
|
let mut fp = 0;
|
|
let mut false_negatives = 0;
|
|
for (_, expected, actual) in &observations {
|
|
match (expected.contains(id), actual.contains(id)) {
|
|
(true, true) => tp += 1,
|
|
(false, true) => fp += 1,
|
|
(true, false) => false_negatives += 1,
|
|
(false, false) => {}
|
|
}
|
|
}
|
|
eprintln!(
|
|
"{id}: precision={}/{} recall={}/{}",
|
|
tp,
|
|
tp + fp,
|
|
tp,
|
|
tp + false_negatives
|
|
);
|
|
let expected_true_positives = usize::from(id == "unicode-ascii-replacement") + 1;
|
|
assert_eq!(
|
|
(tp, fp, false_negatives),
|
|
(expected_true_positives, 0, 0),
|
|
"metrics for {id}"
|
|
);
|
|
total_tp += tp;
|
|
total_fp += fp;
|
|
total_fn += false_negatives;
|
|
}
|
|
|
|
let negative_cases = observations
|
|
.iter()
|
|
.filter(|(_, expected, _)| expected.is_empty())
|
|
.count();
|
|
let false_positive_cases = observations
|
|
.iter()
|
|
.filter(|(_, expected, actual)| expected.is_empty() && !actual.is_empty())
|
|
.count();
|
|
eprintln!(
|
|
"aggregate: tp={total_tp} fp={total_fp} fn={total_fn}; false-positive cases={false_positive_cases}/{negative_cases}"
|
|
);
|
|
|
|
assert_eq!((total_tp, total_fp, total_fn), (13, 0, 0));
|
|
assert_eq!((false_positive_cases, negative_cases), (0, 5));
|
|
assert_eq!(
|
|
reviewed_near_misses,
|
|
detector
|
|
.rules()
|
|
.iter()
|
|
.map(|rule| rule.id().to_owned())
|
|
.collect()
|
|
);
|
|
|
|
let signoff: Signoff = toml::from_str(
|
|
&fs::read_to_string(root.join("signoff.toml")).expect("read evaluation sign-off"),
|
|
)
|
|
.expect("parse evaluation sign-off");
|
|
let mut catalog = Sha256::new();
|
|
catalog.update(include_bytes!(
|
|
"../crates/john-slop-core/assets/heuristics.toml"
|
|
));
|
|
catalog.update([0]);
|
|
catalog.update(include_bytes!(
|
|
"../crates/john-slop-core/assets/unicode.toml"
|
|
));
|
|
let catalog_hash = catalog
|
|
.finalize()
|
|
.iter()
|
|
.map(|byte| format!("{byte:02x}"))
|
|
.collect::<String>();
|
|
assert_eq!(signoff.schema_version, 1);
|
|
assert_eq!(signoff.release, env!("CARGO_PKG_VERSION"));
|
|
assert_eq!(signoff.status, "accepted-after-unicode-replacement-profile");
|
|
assert_eq!(signoff.catalog_sha256, catalog_hash);
|
|
assert!(signoff.limitations_acknowledged);
|
|
assert_eq!(signoff.reviewed_on, "2026-09-30");
|
|
assert!(signoff.decision.contains("prompt-leakage"));
|
|
assert!(signoff.decision.contains("0.90"));
|
|
assert!(signoff.decision.contains("UTS #39"));
|
|
assert!(signoff.decision.contains("review"));
|
|
}
|