john-slop/tests/heuristics.rs

208 lines
7.3 KiB
Rust

use john_slop::{AnalysisContext, Config, Detector};
#[derive(Clone, Copy)]
enum FixtureKind {
Prose,
Rust,
}
struct RuleCase {
id: &'static str,
positive: &'static str,
near_miss: &'static str,
kind: FixtureKind,
}
fn context(kind: FixtureKind) -> AnalysisContext {
match kind {
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
}
}
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
detector
.analyze_with(text, &context(kind))
.findings
.iter()
.any(|finding| finding.rule_id.as_str() == id)
}
#[test]
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
let detector = Detector::new(&Config::embedded()).unwrap();
let cases = [
RuleCase {
id: "assistant-self-identification",
positive: "As an AI language model, I can summarize this.",
near_miss: "AI language models tokenize their inputs.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-capability-disclaimer",
positive: "I cannot browse the internet for current prices.",
near_miss: "The browser cannot access the private network.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-answer-preface",
positive: "Certainly! Here's the requested overview.",
near_miss: "Here is the requested overview.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "knowledge-cutoff-disclaimer",
positive: "My knowledge cutoff prevents a current answer.",
near_miss: "The archive has a documented cutoff date.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "prompt-response-scaffolding",
positive: "Assistant: generated response",
near_miss: "The assistant: generated a response.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-follow-up-offer",
positive: "Let me know if you'd like me to expand this.",
near_miss: "Let me know whether the expansion is useful.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-revision-preface",
positive: "Here's the polished draft for publication.",
near_miss: "The polished draft was accepted for publication.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-conclusion",
positive: "In conclusion, we can retain this design.",
near_miss: "The conclusion retains this design.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-filler",
positive: "It is important to note that the value is cached.",
near_miss: "The important value is cached.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "repeated-transition",
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
near_miss: "Moreover, the only transition is relevant.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "editorial-insertion-placeholder",
positive: "The claim needs support. [Add reference here]",
near_miss: "The claim cites [Smith 2024].",
kind: FixtureKind::Prose,
},
RuleCase {
id: "stock-helpful-closing",
positive: "The command exits successfully.\nI hope this helps.",
near_miss: "We hope this helps readers reproduce the result.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "excess-vocabulary-cluster",
positive: "The report delves into a multifaceted problem.",
near_miss: "The landscape photograph records the site.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "source-placeholder-todo",
positive: "fn parse() { /* TODO: implement the parser */ }",
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
kind: FixtureKind::Rust,
},
RuleCase {
id: "source-tutorial-narration",
positive: "// First, we parse the input.\nlet value = parse();",
near_miss: "// First input byte stores the version.\nlet value = parse();",
kind: FixtureKind::Rust,
},
RuleCase {
id: "source-omitted-code-placeholder",
positive: "fn parse() {}\n// ... remaining code here",
near_miss: "// The remaining code validates the parsed value.",
kind: FixtureKind::Rust,
},
RuleCase {
id: "source-markdown-fence",
positive: "```rust\nfn parse() {}",
near_miss: "const FENCE: &str = \"three backticks\";",
kind: FixtureKind::Rust,
},
RuleCase {
id: "unicode-ascii-replacement",
positive: "Review this—then continue.",
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-invisible-control",
positive: "A zero-width\u{200b}space is hidden here.",
near_miss: "Ordinary spaces and\t tabs are visible structure.",
kind: FixtureKind::Prose,
},
];
assert_eq!(cases.len(), detector.rules().len());
for case in cases {
assert!(
contains_rule(&detector, case.positive, case.kind, case.id),
"positive fixture did not trigger {}",
case.id
);
assert!(
!contains_rule(&detector, case.near_miss, case.kind, case.id),
"near miss unexpectedly triggered {}",
case.id
);
}
}
#[test]
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
let mut config = Config::embedded();
config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap()
.enabled = false;
assert!(!contains_rule(
&Detector::new(&config).unwrap(),
"As an AI language model, I can help.",
FixtureKind::Prose,
"assistant-self-identification"
));
let rule = config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap();
rule.enabled = true;
rule.pattern = "Project-specific marker".to_owned();
rule.weight = 42.0;
let analysis = Detector::new(&config)
.unwrap()
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
assert_eq!(analysis.findings.len(), 1);
assert_eq!(analysis.findings[0].weight, 42.0);
}
#[test]
fn ordinary_multilingual_text_is_not_evidence() {
let detector = Detector::new(&Config::embedded()).unwrap();
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
assert!(analysis.findings.is_empty());
assert_eq!(analysis.evidence_score.score(), 0);
assert!(analysis.evidence_band.is_none());
}