test: pin shipped heuristic behavior

This commit is contained in:
Dandaedre 2026-09-29 15:30:18 +02:00
parent 0c9c117d5e
commit f8e2c9ad8e
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
2 changed files with 160 additions and 1 deletions

View File

@ -123,7 +123,7 @@ configuration is explainable, and all precedence/validation tests pass.
- [x] Return zero confidence and no verdict for empty/no-evidence input. Keep
unsupported/binary input distinct from a clean analysis.
- [x] Sort findings deterministically and deduplicate identical rule/span pairs.
- [ ] Add table-driven unit tests for every shipped rule with positive examples
- [x] Add table-driven unit tests for every shipped rule with positive examples
and realistic near misses. Add tests for disabled rules, overrides, scope,
caps, damping, threshold boundaries, and multilingual text.
- [ ] Build a reviewed evaluation corpus that is legally distributable and labels

159
tests/heuristics.rs Normal file
View File

@ -0,0 +1,159 @@
use john_slop::{AnalysisContext, Config, Detector};
#[derive(Clone, Copy)]
enum FixtureKind {
Prose,
Rust,
}
struct RuleCase {
id: &'static str,
positive: &'static str,
near_miss: &'static str,
kind: FixtureKind,
}
fn context(kind: FixtureKind) -> AnalysisContext {
match kind {
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
}
}
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
detector
.analyze_with(text, &context(kind))
.findings
.iter()
.any(|finding| finding.rule_id.as_str() == id)
}
#[test]
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
let detector = Detector::new(&Config::embedded()).unwrap();
let cases = [
RuleCase {
id: "assistant-self-identification",
positive: "As an AI language model, I can summarize this.",
near_miss: "AI language models tokenize their inputs.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-capability-disclaimer",
positive: "I cannot browse the internet for current prices.",
near_miss: "The browser cannot access the private network.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-answer-preface",
positive: "Certainly! Here's the requested overview.",
near_miss: "Here is the requested overview.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "knowledge-cutoff-disclaimer",
positive: "My knowledge cutoff prevents a current answer.",
near_miss: "The archive has a documented cutoff date.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "prompt-response-scaffolding",
positive: "Assistant: generated response",
near_miss: "The assistant: generated a response.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-conclusion",
positive: "In conclusion, we can retain this design.",
near_miss: "The conclusion retains this design.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-filler",
positive: "It is important to note that the value is cached.",
near_miss: "The important value is cached.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "repeated-transition",
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
near_miss: "Moreover, the only transition is relevant.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "source-placeholder-todo",
positive: "fn parse() { /* TODO: implement the parser */ }",
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
kind: FixtureKind::Rust,
},
RuleCase {
id: "source-tutorial-narration",
positive: "// First, we parse the input.\nlet value = parse();",
near_miss: "// First input byte stores the version.\nlet value = parse();",
kind: FixtureKind::Rust,
},
RuleCase {
id: "typography-non-ascii-hyphen",
positive: "Review this—then continue.",
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
];
assert_eq!(cases.len(), detector.rules().len());
for case in cases {
assert!(
contains_rule(&detector, case.positive, case.kind, case.id),
"positive fixture did not trigger {}",
case.id
);
assert!(
!contains_rule(&detector, case.near_miss, case.kind, case.id),
"near miss unexpectedly triggered {}",
case.id
);
}
}
#[test]
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
let mut config = Config::embedded();
config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap()
.enabled = false;
assert!(!contains_rule(
&Detector::new(&config).unwrap(),
"As an AI language model, I can help.",
FixtureKind::Prose,
"assistant-self-identification"
));
let rule = config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap();
rule.enabled = true;
rule.pattern = "Project-specific marker".to_owned();
rule.weight = 42.0;
let analysis = Detector::new(&config)
.unwrap()
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
assert_eq!(analysis.findings.len(), 1);
assert_eq!(analysis.findings[0].weight, 42.0);
}
#[test]
fn ordinary_multilingual_text_is_not_evidence() {
let detector = Detector::new(&Config::embedded()).unwrap();
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
assert!(analysis.findings.is_empty());
assert_eq!(analysis.confidence.score(), 0);
assert!(analysis.confidence_band.is_none());
}