diff --git a/tasks.md b/tasks.md index 23e5aa6..a0dd38b 100644 --- a/tasks.md +++ b/tasks.md @@ -123,7 +123,7 @@ configuration is explainable, and all precedence/validation tests pass. - [x] Return zero confidence and no verdict for empty/no-evidence input. Keep unsupported/binary input distinct from a clean analysis. - [x] Sort findings deterministically and deduplicate identical rule/span pairs. -- [ ] Add table-driven unit tests for every shipped rule with positive examples +- [x] Add table-driven unit tests for every shipped rule with positive examples and realistic near misses. Add tests for disabled rules, overrides, scope, caps, damping, threshold boundaries, and multilingual text. - [ ] Build a reviewed evaluation corpus that is legally distributable and labels diff --git a/tests/heuristics.rs b/tests/heuristics.rs new file mode 100644 index 0000000..5b32a65 --- /dev/null +++ b/tests/heuristics.rs @@ -0,0 +1,159 @@ +use john_slop::{AnalysisContext, Config, Detector}; + +#[derive(Clone, Copy)] +enum FixtureKind { + Prose, + Rust, +} + +struct RuleCase { + id: &'static str, + positive: &'static str, + near_miss: &'static str, + kind: FixtureKind, +} + +fn context(kind: FixtureKind) -> AnalysisContext { + match kind { + FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"), + FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"), + } +} + +fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool { + detector + .analyze_with(text, &context(kind)) + .findings + .iter() + .any(|finding| finding.rule_id.as_str() == id) +} + +#[test] +fn every_shipped_rule_has_a_positive_and_realistic_near_miss() { + let detector = Detector::new(&Config::embedded()).unwrap(); + let cases = [ + RuleCase { + id: "assistant-self-identification", + positive: "As an AI language model, I can summarize this.", + near_miss: "AI language models tokenize their inputs.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "assistant-capability-disclaimer", + positive: "I cannot browse the internet for current prices.", + near_miss: "The browser cannot access the private network.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "assistant-answer-preface", + positive: "Certainly! Here's the requested overview.", + near_miss: "Here is the requested overview.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "knowledge-cutoff-disclaimer", + positive: "My knowledge cutoff prevents a current answer.", + near_miss: "The archive has a documented cutoff date.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "prompt-response-scaffolding", + positive: "Assistant: generated response", + near_miss: "The assistant: generated a response.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "templated-conclusion", + positive: "In conclusion, we can retain this design.", + near_miss: "The conclusion retains this design.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "templated-filler", + positive: "It is important to note that the value is cached.", + near_miss: "The important value is cached.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "repeated-transition", + positive: "Moreover, the first result holds.\nAdditionally, the second follows.", + near_miss: "Moreover, the only transition is relevant.", + kind: FixtureKind::Prose, + }, + RuleCase { + id: "source-placeholder-todo", + positive: "fn parse() { /* TODO: implement the parser */ }", + near_miss: "fn parse() { /* TODO: simplify error mapping */ }", + kind: FixtureKind::Rust, + }, + RuleCase { + id: "source-tutorial-narration", + positive: "// First, we parse the input.\nlet value = parse();", + near_miss: "// First input byte stores the version.\nlet value = parse();", + kind: FixtureKind::Rust, + }, + RuleCase { + id: "typography-non-ascii-hyphen", + positive: "Review this—then continue.", + near_miss: "Review this-then continue.", + kind: FixtureKind::Prose, + }, + ]; + + assert_eq!(cases.len(), detector.rules().len()); + for case in cases { + assert!( + contains_rule(&detector, case.positive, case.kind, case.id), + "positive fixture did not trigger {}", + case.id + ); + assert!( + !contains_rule(&detector, case.near_miss, case.kind, case.id), + "near miss unexpectedly triggered {}", + case.id + ); + } +} + +#[test] +fn disabled_and_overridden_rules_change_execution_without_code_changes() { + let mut config = Config::embedded(); + config + .rules + .iter_mut() + .find(|rule| rule.id == "assistant-self-identification") + .unwrap() + .enabled = false; + assert!(!contains_rule( + &Detector::new(&config).unwrap(), + "As an AI language model, I can help.", + FixtureKind::Prose, + "assistant-self-identification" + )); + + let rule = config + .rules + .iter_mut() + .find(|rule| rule.id == "assistant-self-identification") + .unwrap(); + rule.enabled = true; + rule.pattern = "Project-specific marker".to_owned(); + rule.weight = 42.0; + let analysis = Detector::new(&config) + .unwrap() + .analyze_with("Project-specific marker", &context(FixtureKind::Prose)); + + assert_eq!(analysis.findings.len(), 1); + assert_eq!(analysis.findings[0].weight, 42.0); +} + +#[test] +fn ordinary_multilingual_text_is_not_evidence() { + let detector = Detector::new(&Config::embedded()).unwrap(); + let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español."; + let analysis = detector.analyze_with(text, &context(FixtureKind::Prose)); + + assert!(analysis.findings.is_empty()); + assert_eq!(analysis.confidence.score(), 0); + assert!(analysis.confidence_band.is_none()); +}