172 lines
5.8 KiB
Rust
172 lines
5.8 KiB
Rust
use john_slop::{AnalysisContext, Config, Detector};
|
||
|
||
#[derive(Clone, Copy)]
|
||
enum FixtureKind {
|
||
Prose,
|
||
Rust,
|
||
}
|
||
|
||
struct RuleCase {
|
||
id: &'static str,
|
||
positive: &'static str,
|
||
near_miss: &'static str,
|
||
kind: FixtureKind,
|
||
}
|
||
|
||
fn context(kind: FixtureKind) -> AnalysisContext {
|
||
match kind {
|
||
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
|
||
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
|
||
}
|
||
}
|
||
|
||
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
|
||
detector
|
||
.analyze_with(text, &context(kind))
|
||
.findings
|
||
.iter()
|
||
.any(|finding| finding.rule_id.as_str() == id)
|
||
}
|
||
|
||
#[test]
|
||
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||
let cases = [
|
||
RuleCase {
|
||
id: "assistant-self-identification",
|
||
positive: "As an AI language model, I can summarize this.",
|
||
near_miss: "AI language models tokenize their inputs.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "assistant-capability-disclaimer",
|
||
positive: "I cannot browse the internet for current prices.",
|
||
near_miss: "The browser cannot access the private network.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "assistant-answer-preface",
|
||
positive: "Certainly! Here's the requested overview.",
|
||
near_miss: "Here is the requested overview.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "knowledge-cutoff-disclaimer",
|
||
positive: "My knowledge cutoff prevents a current answer.",
|
||
near_miss: "The archive has a documented cutoff date.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "prompt-response-scaffolding",
|
||
positive: "Assistant: generated response",
|
||
near_miss: "The assistant: generated a response.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "templated-conclusion",
|
||
positive: "In conclusion, we can retain this design.",
|
||
near_miss: "The conclusion retains this design.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "templated-filler",
|
||
positive: "It is important to note that the value is cached.",
|
||
near_miss: "The important value is cached.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "repeated-transition",
|
||
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
|
||
near_miss: "Moreover, the only transition is relevant.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "source-placeholder-todo",
|
||
positive: "fn parse() { /* TODO: implement the parser */ }",
|
||
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
|
||
kind: FixtureKind::Rust,
|
||
},
|
||
RuleCase {
|
||
id: "source-tutorial-narration",
|
||
positive: "// First, we parse the input.\nlet value = parse();",
|
||
near_miss: "// First input byte stores the version.\nlet value = parse();",
|
||
kind: FixtureKind::Rust,
|
||
},
|
||
RuleCase {
|
||
id: "typography-non-ascii-hyphen",
|
||
positive: "Review this—then continue.",
|
||
near_miss: "Review this-then continue.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "unicode-ascii-confusable",
|
||
positive: "The pаssword contains a Cyrillic lookalike.",
|
||
near_miss: "Résumé précis en français.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
RuleCase {
|
||
id: "unicode-invisible-control",
|
||
positive: "A zero-width\u{200b}space is hidden here.",
|
||
near_miss: "Ordinary spaces and\t tabs are visible structure.",
|
||
kind: FixtureKind::Prose,
|
||
},
|
||
];
|
||
|
||
assert_eq!(cases.len(), detector.rules().len());
|
||
for case in cases {
|
||
assert!(
|
||
contains_rule(&detector, case.positive, case.kind, case.id),
|
||
"positive fixture did not trigger {}",
|
||
case.id
|
||
);
|
||
assert!(
|
||
!contains_rule(&detector, case.near_miss, case.kind, case.id),
|
||
"near miss unexpectedly triggered {}",
|
||
case.id
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
|
||
let mut config = Config::embedded();
|
||
config
|
||
.rules
|
||
.iter_mut()
|
||
.find(|rule| rule.id == "assistant-self-identification")
|
||
.unwrap()
|
||
.enabled = false;
|
||
assert!(!contains_rule(
|
||
&Detector::new(&config).unwrap(),
|
||
"As an AI language model, I can help.",
|
||
FixtureKind::Prose,
|
||
"assistant-self-identification"
|
||
));
|
||
|
||
let rule = config
|
||
.rules
|
||
.iter_mut()
|
||
.find(|rule| rule.id == "assistant-self-identification")
|
||
.unwrap();
|
||
rule.enabled = true;
|
||
rule.pattern = "Project-specific marker".to_owned();
|
||
rule.weight = 42.0;
|
||
let analysis = Detector::new(&config)
|
||
.unwrap()
|
||
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
|
||
|
||
assert_eq!(analysis.findings.len(), 1);
|
||
assert_eq!(analysis.findings[0].weight, 42.0);
|
||
}
|
||
|
||
#[test]
|
||
fn ordinary_multilingual_text_is_not_evidence() {
|
||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
|
||
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
|
||
|
||
assert!(analysis.findings.is_empty());
|
||
assert_eq!(analysis.evidence_score.score(), 0);
|
||
assert!(analysis.evidence_band.is_none());
|
||
}
|