feat: expand conservative heuristic catalog
This commit is contained in:
parent
90f64fd8ac
commit
aad482e8e4
|
|
@ -37,6 +37,36 @@ enabled = true
|
|||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "assistant-capability-disclaimer"
|
||||
category = "prompt-leakage"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "I cannot browse the internet"
|
||||
weight = 20.0
|
||||
message = "an assistant capability disclaimer appears in the source"
|
||||
help = "Replace generic capability boilerplate with a source-specific limitation if needed."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "assistant-answer-preface"
|
||||
category = "prompt-leakage"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "Certainly! Here's"
|
||||
weight = 15.0
|
||||
message = "a stock assistant answer preface appears in the source"
|
||||
help = "Review whether conversational response scaffolding was copied into the document."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "prompt-response-scaffolding"
|
||||
category = "prompt-leakage"
|
||||
|
|
@ -67,6 +97,36 @@ enabled = true
|
|||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "templated-filler"
|
||||
category = "style"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\bit is (?:important|worth) (?:to note|noting) that\\b"
|
||||
weight = 4.0
|
||||
message = "a generic filler transition appears in the source"
|
||||
help = "Prefer a direct, source-specific statement when the transition adds no meaning."
|
||||
occurrence_cap = 1
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "repeated-transition"
|
||||
category = "style"
|
||||
matcher = "repeated-regex"
|
||||
pattern = "(?im)^(?:Moreover|Furthermore|Additionally),[ \\t]"
|
||||
weight = 4.0
|
||||
message = "the same family of stock transitions is repeated"
|
||||
help = "Review repeated transitions and keep only those that clarify the argument."
|
||||
occurrence_cap = 3
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["prose"]
|
||||
extensions = ["md", "txt", "rst", "adoc"]
|
||||
|
||||
[[rules]]
|
||||
id = "source-placeholder-todo"
|
||||
category = "source-code"
|
||||
|
|
@ -82,6 +142,21 @@ enabled = true
|
|||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
|
||||
|
||||
[[rules]]
|
||||
id = "source-tutorial-narration"
|
||||
category = "source-code"
|
||||
matcher = "regex"
|
||||
pattern = "(?im)^[ \\t]*//[ \\t]*(?:first|next|finally),?[ \\t]+we\\b"
|
||||
weight = 8.0
|
||||
message = "a tutorial-style narration comment appears in source code"
|
||||
help = "Describe why the code exists rather than narrating the next obvious operation."
|
||||
occurrence_cap = 2
|
||||
enabled = true
|
||||
|
||||
[rules.scope]
|
||||
content = ["source-code"]
|
||||
extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
|
||||
|
||||
[[rules]]
|
||||
id = "typography-non-ascii-hyphen"
|
||||
category = "typography"
|
||||
|
|
|
|||
|
|
@ -750,6 +750,8 @@ pub enum MatcherType {
|
|||
ExactPhrase,
|
||||
/// Rust regular-expression matching.
|
||||
Regex,
|
||||
/// Regex matching that activates only after multiple occurrences.
|
||||
RepeatedRegex,
|
||||
/// Findings produced by the configured replacement catalog.
|
||||
Replacement,
|
||||
}
|
||||
|
|
@ -921,7 +923,7 @@ fn validate_rules(rules: &[HeuristicRule]) -> Result<(), ConfigValidationError>
|
|||
}
|
||||
match rule.matcher {
|
||||
MatcherType::ExactPhrase => {}
|
||||
MatcherType::Regex => {
|
||||
MatcherType::Regex | MatcherType::RepeatedRegex => {
|
||||
let regex = regex::Regex::new(&rule.pattern).map_err(|source| {
|
||||
ConfigValidationError::InvalidRegex {
|
||||
id: rule.id.clone(),
|
||||
|
|
@ -1047,7 +1049,7 @@ mod tests {
|
|||
fn embedded_heuristics_have_complete_metadata() {
|
||||
let config = Config::embedded();
|
||||
|
||||
assert_eq!(config.rules.len(), 6);
|
||||
assert_eq!(config.rules.len(), 11);
|
||||
assert_eq!(config.thresholds.low, 20);
|
||||
assert_eq!(config.thresholds.medium, 50);
|
||||
assert_eq!(config.thresholds.high, 75);
|
||||
|
|
@ -1190,7 +1192,7 @@ weight = 9.5
|
|||
assert_eq!(replacement.source, '—');
|
||||
assert_eq!(replacement.replacement, "-");
|
||||
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
|
||||
assert_eq!(config.rules.len(), 6);
|
||||
assert_eq!(config.rules.len(), 11);
|
||||
let heuristic = config
|
||||
.rules
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ impl Rule {
|
|||
match &self.matcher {
|
||||
CompiledMatcher::ExactPhrase(pattern) => pattern,
|
||||
CompiledMatcher::Regex(regex) => regex.as_str(),
|
||||
CompiledMatcher::RepeatedRegex(regex) => regex.as_str(),
|
||||
CompiledMatcher::Replacement => "configured-replacements",
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +46,7 @@ impl Rule {
|
|||
enum CompiledMatcher {
|
||||
ExactPhrase(String),
|
||||
Regex(Regex),
|
||||
RepeatedRegex(Regex),
|
||||
Replacement,
|
||||
}
|
||||
|
||||
|
|
@ -108,6 +110,15 @@ impl Rule {
|
|||
})?;
|
||||
CompiledMatcher::Regex(regex)
|
||||
}
|
||||
MatcherType::RepeatedRegex => {
|
||||
let regex = Regex::new(&definition.pattern).map_err(|source| {
|
||||
DetectorBuildError::InvalidRegex {
|
||||
id: definition.id.clone(),
|
||||
source,
|
||||
}
|
||||
})?;
|
||||
CompiledMatcher::RepeatedRegex(regex)
|
||||
}
|
||||
MatcherType::Replacement => CompiledMatcher::Replacement,
|
||||
};
|
||||
Ok(Self {
|
||||
|
|
@ -126,6 +137,17 @@ impl Rule {
|
|||
.find_iter(source)
|
||||
.map(|matched| self.finding(source, matched.range(), None))
|
||||
.collect(),
|
||||
CompiledMatcher::RepeatedRegex(regex) => {
|
||||
let matches = regex.find_iter(source).collect::<Vec<_>>();
|
||||
if matches.len() < 2 {
|
||||
Vec::new()
|
||||
} else {
|
||||
matches
|
||||
.into_iter()
|
||||
.map(|matched| self.finding(source, matched.range(), None))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
CompiledMatcher::Replacement => replacement::plan(config, source)
|
||||
.into_iter()
|
||||
.map(|edit| self.finding(source, edit.byte_span.clone(), Some(edit)))
|
||||
|
|
@ -253,4 +275,52 @@ mod tests {
|
|||
assert_eq!(finding.category, Category::PromptLeakage);
|
||||
assert_eq!(finding.weight, 25.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_catalog_covers_each_conservative_signal_family() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let ids = detector
|
||||
.rules()
|
||||
.iter()
|
||||
.map(Rule::id)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
|
||||
for expected in [
|
||||
"assistant-self-identification",
|
||||
"assistant-capability-disclaimer",
|
||||
"assistant-answer-preface",
|
||||
"knowledge-cutoff-disclaimer",
|
||||
"prompt-response-scaffolding",
|
||||
"templated-filler",
|
||||
"templated-conclusion",
|
||||
"repeated-transition",
|
||||
"source-placeholder-todo",
|
||||
"source-tutorial-narration",
|
||||
"typography-non-ascii-hyphen",
|
||||
] {
|
||||
assert!(ids.contains(expected), "missing {expected}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_transition_requires_multiple_occurrences() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let one = detector.analyze("Moreover, one transition is ordinary.");
|
||||
let repeated =
|
||||
detector.analyze("Moreover, the first claim.\nAdditionally, the second claim.");
|
||||
|
||||
assert!(
|
||||
!one.findings
|
||||
.iter()
|
||||
.any(|finding| finding.rule_id.as_str() == "repeated-transition")
|
||||
);
|
||||
assert_eq!(
|
||||
repeated
|
||||
.findings
|
||||
.iter()
|
||||
.filter(|finding| finding.rule_id.as_str() == "repeated-transition")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -111,7 +111,7 @@ configuration is explainable, and all precedence/validation tests pass.
|
|||
validated heuristic config. Compile reusable matchers once.
|
||||
- [x] Implement exact phrase and regex rules with precise byte spans, match text,
|
||||
category, weight, message, help, and stable rule ID.
|
||||
- [ ] Add conservative first-party rules for copied assistant disclaimers,
|
||||
- [x] Add conservative first-party rules for copied assistant disclaimers,
|
||||
knowledge-cutoff language, prompt/response leakage, templated filler, repetitive
|
||||
transitions/conclusions, suspicious Unicode, and code-specific placeholder or
|
||||
tutorial residue.
|
||||
|
|
|
|||
Loading…
Reference in New Issue