test: pin shipped heuristic behavior
This commit is contained in:
parent
0c9c117d5e
commit
f8e2c9ad8e
2
tasks.md
2
tasks.md
|
|
@ -123,7 +123,7 @@ configuration is explainable, and all precedence/validation tests pass.
|
|||
- [x] Return zero confidence and no verdict for empty/no-evidence input. Keep
|
||||
unsupported/binary input distinct from a clean analysis.
|
||||
- [x] Sort findings deterministically and deduplicate identical rule/span pairs.
|
||||
- [ ] Add table-driven unit tests for every shipped rule with positive examples
|
||||
- [x] Add table-driven unit tests for every shipped rule with positive examples
|
||||
and realistic near misses. Add tests for disabled rules, overrides, scope,
|
||||
caps, damping, threshold boundaries, and multilingual text.
|
||||
- [ ] Build a reviewed evaluation corpus that is legally distributable and labels
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
use john_slop::{AnalysisContext, Config, Detector};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum FixtureKind {
|
||||
Prose,
|
||||
Rust,
|
||||
}
|
||||
|
||||
struct RuleCase {
|
||||
id: &'static str,
|
||||
positive: &'static str,
|
||||
near_miss: &'static str,
|
||||
kind: FixtureKind,
|
||||
}
|
||||
|
||||
fn context(kind: FixtureKind) -> AnalysisContext {
|
||||
match kind {
|
||||
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
|
||||
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
|
||||
detector
|
||||
.analyze_with(text, &context(kind))
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.rule_id.as_str() == id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let cases = [
|
||||
RuleCase {
|
||||
id: "assistant-self-identification",
|
||||
positive: "As an AI language model, I can summarize this.",
|
||||
near_miss: "AI language models tokenize their inputs.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "assistant-capability-disclaimer",
|
||||
positive: "I cannot browse the internet for current prices.",
|
||||
near_miss: "The browser cannot access the private network.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "assistant-answer-preface",
|
||||
positive: "Certainly! Here's the requested overview.",
|
||||
near_miss: "Here is the requested overview.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "knowledge-cutoff-disclaimer",
|
||||
positive: "My knowledge cutoff prevents a current answer.",
|
||||
near_miss: "The archive has a documented cutoff date.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "prompt-response-scaffolding",
|
||||
positive: "Assistant: generated response",
|
||||
near_miss: "The assistant: generated a response.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "templated-conclusion",
|
||||
positive: "In conclusion, we can retain this design.",
|
||||
near_miss: "The conclusion retains this design.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "templated-filler",
|
||||
positive: "It is important to note that the value is cached.",
|
||||
near_miss: "The important value is cached.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "repeated-transition",
|
||||
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
|
||||
near_miss: "Moreover, the only transition is relevant.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
RuleCase {
|
||||
id: "source-placeholder-todo",
|
||||
positive: "fn parse() { /* TODO: implement the parser */ }",
|
||||
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
|
||||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "source-tutorial-narration",
|
||||
positive: "// First, we parse the input.\nlet value = parse();",
|
||||
near_miss: "// First input byte stores the version.\nlet value = parse();",
|
||||
kind: FixtureKind::Rust,
|
||||
},
|
||||
RuleCase {
|
||||
id: "typography-non-ascii-hyphen",
|
||||
positive: "Review this—then continue.",
|
||||
near_miss: "Review this-then continue.",
|
||||
kind: FixtureKind::Prose,
|
||||
},
|
||||
];
|
||||
|
||||
assert_eq!(cases.len(), detector.rules().len());
|
||||
for case in cases {
|
||||
assert!(
|
||||
contains_rule(&detector, case.positive, case.kind, case.id),
|
||||
"positive fixture did not trigger {}",
|
||||
case.id
|
||||
);
|
||||
assert!(
|
||||
!contains_rule(&detector, case.near_miss, case.kind, case.id),
|
||||
"near miss unexpectedly triggered {}",
|
||||
case.id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
|
||||
let mut config = Config::embedded();
|
||||
config
|
||||
.rules
|
||||
.iter_mut()
|
||||
.find(|rule| rule.id == "assistant-self-identification")
|
||||
.unwrap()
|
||||
.enabled = false;
|
||||
assert!(!contains_rule(
|
||||
&Detector::new(&config).unwrap(),
|
||||
"As an AI language model, I can help.",
|
||||
FixtureKind::Prose,
|
||||
"assistant-self-identification"
|
||||
));
|
||||
|
||||
let rule = config
|
||||
.rules
|
||||
.iter_mut()
|
||||
.find(|rule| rule.id == "assistant-self-identification")
|
||||
.unwrap();
|
||||
rule.enabled = true;
|
||||
rule.pattern = "Project-specific marker".to_owned();
|
||||
rule.weight = 42.0;
|
||||
let analysis = Detector::new(&config)
|
||||
.unwrap()
|
||||
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
|
||||
|
||||
assert_eq!(analysis.findings.len(), 1);
|
||||
assert_eq!(analysis.findings[0].weight, 42.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_multilingual_text_is_not_evidence() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
|
||||
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
|
||||
|
||||
assert!(analysis.findings.is_empty());
|
||||
assert_eq!(analysis.confidence.score(), 0);
|
||||
assert!(analysis.confidence_band.is_none());
|
||||
}
|
||||
Loading…
Reference in New Issue