john-slop/tests/heuristics.rs

172 lines
5.8 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use john_slop::{AnalysisContext, Config, Detector};
#[derive(Clone, Copy)]
enum FixtureKind {
Prose,
Rust,
}
struct RuleCase {
id: &'static str,
positive: &'static str,
near_miss: &'static str,
kind: FixtureKind,
}
fn context(kind: FixtureKind) -> AnalysisContext {
match kind {
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
}
}
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
detector
.analyze_with(text, &context(kind))
.findings
.iter()
.any(|finding| finding.rule_id.as_str() == id)
}
#[test]
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
let detector = Detector::new(&Config::embedded()).unwrap();
let cases = [
RuleCase {
id: "assistant-self-identification",
positive: "As an AI language model, I can summarize this.",
near_miss: "AI language models tokenize their inputs.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-capability-disclaimer",
positive: "I cannot browse the internet for current prices.",
near_miss: "The browser cannot access the private network.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "assistant-answer-preface",
positive: "Certainly! Here's the requested overview.",
near_miss: "Here is the requested overview.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "knowledge-cutoff-disclaimer",
positive: "My knowledge cutoff prevents a current answer.",
near_miss: "The archive has a documented cutoff date.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "prompt-response-scaffolding",
positive: "Assistant: generated response",
near_miss: "The assistant: generated a response.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-conclusion",
positive: "In conclusion, we can retain this design.",
near_miss: "The conclusion retains this design.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "templated-filler",
positive: "It is important to note that the value is cached.",
near_miss: "The important value is cached.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "repeated-transition",
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
near_miss: "Moreover, the only transition is relevant.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "source-placeholder-todo",
positive: "fn parse() { /* TODO: implement the parser */ }",
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
kind: FixtureKind::Rust,
},
RuleCase {
id: "source-tutorial-narration",
positive: "// First, we parse the input.\nlet value = parse();",
near_miss: "// First input byte stores the version.\nlet value = parse();",
kind: FixtureKind::Rust,
},
RuleCase {
id: "typography-non-ascii-hyphen",
positive: "Review this—then continue.",
near_miss: "Review this-then continue.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-ascii-confusable",
positive: "The pаssword contains a Cyrillic lookalike.",
near_miss: "Résumé précis en français.",
kind: FixtureKind::Prose,
},
RuleCase {
id: "unicode-invisible-control",
positive: "A zero-width\u{200b}space is hidden here.",
near_miss: "Ordinary spaces and\t tabs are visible structure.",
kind: FixtureKind::Prose,
},
];
assert_eq!(cases.len(), detector.rules().len());
for case in cases {
assert!(
contains_rule(&detector, case.positive, case.kind, case.id),
"positive fixture did not trigger {}",
case.id
);
assert!(
!contains_rule(&detector, case.near_miss, case.kind, case.id),
"near miss unexpectedly triggered {}",
case.id
);
}
}
#[test]
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
let mut config = Config::embedded();
config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap()
.enabled = false;
assert!(!contains_rule(
&Detector::new(&config).unwrap(),
"As an AI language model, I can help.",
FixtureKind::Prose,
"assistant-self-identification"
));
let rule = config
.rules
.iter_mut()
.find(|rule| rule.id == "assistant-self-identification")
.unwrap();
rule.enabled = true;
rule.pattern = "Project-specific marker".to_owned();
rule.weight = 42.0;
let analysis = Detector::new(&config)
.unwrap()
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
assert_eq!(analysis.findings.len(), 1);
assert_eq!(analysis.findings[0].weight, 42.0);
}
#[test]
fn ordinary_multilingual_text_is_not_evidence() {
let detector = Detector::new(&Config::embedded()).unwrap();
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
assert!(analysis.findings.is_empty());
assert_eq!(analysis.evidence_score.score(), 0);
assert!(analysis.evidence_band.is_none());
}