208 lines
7.3 KiB
Rust
208 lines
7.3 KiB
Rust
use john_slop::{AnalysisContext, Config, Detector};
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum FixtureKind {
|
|
Prose,
|
|
Rust,
|
|
}
|
|
|
|
struct RuleCase {
|
|
id: &'static str,
|
|
positive: &'static str,
|
|
near_miss: &'static str,
|
|
kind: FixtureKind,
|
|
}
|
|
|
|
fn context(kind: FixtureKind) -> AnalysisContext {
|
|
match kind {
|
|
FixtureKind::Prose => AnalysisContext::for_path("docs/example.md"),
|
|
FixtureKind::Rust => AnalysisContext::for_path("src/example.rs"),
|
|
}
|
|
}
|
|
|
|
fn contains_rule(detector: &Detector, text: &str, kind: FixtureKind, id: &str) -> bool {
|
|
detector
|
|
.analyze_with(text, &context(kind))
|
|
.findings
|
|
.iter()
|
|
.any(|finding| finding.rule_id.as_str() == id)
|
|
}
|
|
|
|
#[test]
|
|
fn every_shipped_rule_has_a_positive_and_realistic_near_miss() {
|
|
let detector = Detector::new(&Config::embedded()).unwrap();
|
|
let cases = [
|
|
RuleCase {
|
|
id: "assistant-self-identification",
|
|
positive: "As an AI language model, I can summarize this.",
|
|
near_miss: "AI language models tokenize their inputs.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "assistant-capability-disclaimer",
|
|
positive: "I cannot browse the internet for current prices.",
|
|
near_miss: "The browser cannot access the private network.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "assistant-answer-preface",
|
|
positive: "Certainly! Here's the requested overview.",
|
|
near_miss: "Here is the requested overview.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "knowledge-cutoff-disclaimer",
|
|
positive: "My knowledge cutoff prevents a current answer.",
|
|
near_miss: "The archive has a documented cutoff date.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "prompt-response-scaffolding",
|
|
positive: "Assistant: generated response",
|
|
near_miss: "The assistant: generated a response.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "assistant-follow-up-offer",
|
|
positive: "Let me know if you'd like me to expand this.",
|
|
near_miss: "Let me know whether the expansion is useful.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "assistant-revision-preface",
|
|
positive: "Here's the polished draft for publication.",
|
|
near_miss: "The polished draft was accepted for publication.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "templated-conclusion",
|
|
positive: "In conclusion, we can retain this design.",
|
|
near_miss: "The conclusion retains this design.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "templated-filler",
|
|
positive: "It is important to note that the value is cached.",
|
|
near_miss: "The important value is cached.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "repeated-transition",
|
|
positive: "Moreover, the first result holds.\nAdditionally, the second follows.",
|
|
near_miss: "Moreover, the only transition is relevant.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "editorial-insertion-placeholder",
|
|
positive: "The claim needs support. [Add reference here]",
|
|
near_miss: "The claim cites [Smith 2024].",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "stock-helpful-closing",
|
|
positive: "The command exits successfully.\nI hope this helps.",
|
|
near_miss: "We hope this helps readers reproduce the result.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "excess-vocabulary-cluster",
|
|
positive: "The report delves into a multifaceted problem.",
|
|
near_miss: "The landscape photograph records the site.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "source-placeholder-todo",
|
|
positive: "fn parse() { /* TODO: implement the parser */ }",
|
|
near_miss: "fn parse() { /* TODO: simplify error mapping */ }",
|
|
kind: FixtureKind::Rust,
|
|
},
|
|
RuleCase {
|
|
id: "source-tutorial-narration",
|
|
positive: "// First, we parse the input.\nlet value = parse();",
|
|
near_miss: "// First input byte stores the version.\nlet value = parse();",
|
|
kind: FixtureKind::Rust,
|
|
},
|
|
RuleCase {
|
|
id: "source-omitted-code-placeholder",
|
|
positive: "fn parse() {}\n// ... remaining code here",
|
|
near_miss: "// The remaining code validates the parsed value.",
|
|
kind: FixtureKind::Rust,
|
|
},
|
|
RuleCase {
|
|
id: "source-markdown-fence",
|
|
positive: "```rust\nfn parse() {}",
|
|
near_miss: "const FENCE: &str = \"three backticks\";",
|
|
kind: FixtureKind::Rust,
|
|
},
|
|
RuleCase {
|
|
id: "unicode-ascii-replacement",
|
|
positive: "Review this—then continue.",
|
|
near_miss: "Review this-then continue.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
RuleCase {
|
|
id: "unicode-invisible-control",
|
|
positive: "A zero-width\u{200b}space is hidden here.",
|
|
near_miss: "Ordinary spaces and\t tabs are visible structure.",
|
|
kind: FixtureKind::Prose,
|
|
},
|
|
];
|
|
|
|
assert_eq!(cases.len(), detector.rules().len());
|
|
for case in cases {
|
|
assert!(
|
|
contains_rule(&detector, case.positive, case.kind, case.id),
|
|
"positive fixture did not trigger {}",
|
|
case.id
|
|
);
|
|
assert!(
|
|
!contains_rule(&detector, case.near_miss, case.kind, case.id),
|
|
"near miss unexpectedly triggered {}",
|
|
case.id
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_and_overridden_rules_change_execution_without_code_changes() {
|
|
let mut config = Config::embedded();
|
|
config
|
|
.rules
|
|
.iter_mut()
|
|
.find(|rule| rule.id == "assistant-self-identification")
|
|
.unwrap()
|
|
.enabled = false;
|
|
assert!(!contains_rule(
|
|
&Detector::new(&config).unwrap(),
|
|
"As an AI language model, I can help.",
|
|
FixtureKind::Prose,
|
|
"assistant-self-identification"
|
|
));
|
|
|
|
let rule = config
|
|
.rules
|
|
.iter_mut()
|
|
.find(|rule| rule.id == "assistant-self-identification")
|
|
.unwrap();
|
|
rule.enabled = true;
|
|
rule.pattern = "Project-specific marker".to_owned();
|
|
rule.weight = 42.0;
|
|
let analysis = Detector::new(&config)
|
|
.unwrap()
|
|
.analyze_with("Project-specific marker", &context(FixtureKind::Prose));
|
|
|
|
assert_eq!(analysis.findings.len(), 1);
|
|
assert_eq!(analysis.findings[0].weight, 42.0);
|
|
}
|
|
|
|
#[test]
|
|
fn ordinary_multilingual_text_is_not_evidence() {
|
|
let detector = Detector::new(&Config::embedded()).unwrap();
|
|
let text = "Résumé précis en français. 日本語の短い説明。 Explicación en español.";
|
|
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
|
|
|
|
assert!(analysis.findings.is_empty());
|
|
assert_eq!(analysis.evidence_score.score(), 0);
|
|
assert!(analysis.evidence_band.is_none());
|
|
}
|