feat: prioritize prompt leakage evidence

This commit is contained in:
Dandaedre 2026-09-29 20:57:15 +02:00
parent 30445d7983
commit d6163076d9
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
3 changed files with 52 additions and 4 deletions

View File

@ -1,8 +1,8 @@
[last_run_clean]
run_status = "Success"
lines_parsed = 337
coverage = 92.16
lines_parsed = 338
coverage = 92.19
[last_run_clean.run_duration]
secs = 29
nanos = 214508516
nanos = 930030912

View File

@ -209,7 +209,8 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
.map(|index| 0.5_f64.powi(i32::try_from(index).unwrap_or(i32::MAX)))
.sum::<f64>();
let family_index = family_rule_counts.entry(rule.category).or_default();
let family_factor = 0.75_f64.powi(i32::try_from(*family_index).unwrap_or(i32::MAX));
let family_factor =
family_decay(rule.category).powi(i32::try_from(*family_index).unwrap_or(i32::MAX));
evidence += rule.weight * occurrence_factor * family_factor;
*family_index += 1;
}
@ -222,6 +223,13 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
Confidence::from_score(normalized.round() as u8)
}
const fn family_decay(category: Category) -> f64 {
match category {
Category::PromptLeakage => 0.90,
Category::Typography | Category::Style | Category::SourceCode => 0.75,
}
}
fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand {
if score >= thresholds.high {
ConfidenceBand::High
@ -739,6 +747,26 @@ mod tests {
assert_eq!(correlated.findings.len(), independent.findings.len());
}
#[test]
fn prompt_leakage_uses_lighter_family_damping_than_other_categories() {
assert_eq!(family_decay(Category::PromptLeakage), 0.90);
assert_eq!(family_decay(Category::Style), 0.75);
assert_eq!(family_decay(Category::Typography), 0.75);
assert_eq!(family_decay(Category::SourceCode), 0.75);
let source = concat!(
"User: request\n",
"Assistant: Certainly! Here's a response.\n",
"As an AI language model, I cannot browse the internet. ",
"My knowledge cutoff applies."
);
let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source);
assert_eq!(analysis.findings.len(), 6);
assert_eq!(analysis.confidence.score(), 67);
assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium));
}
#[test]
fn configured_thresholds_define_named_band_boundaries() {
let thresholds = crate::ConfidenceThresholds {

View File

@ -269,6 +269,26 @@ Acceptance: a default detector invocation renders graphical Unicode diagnostics
with ANSI severity colors; explicit machine modes remain stable and uncolored,
and builds without `fancy-diagnostics` retain the plain human fallback.
## Milestone 10 — Prompt-leakage scoring priority
- [x] Give distinct prompt-leakage rules lighter correlation damping (`0.90ⁿ`)
while preserving `0.75ⁿ` damping for style, typography, and source-code
families and preserving per-rule occurrence damping/caps.
- [ ] Re-run the reviewed corpus, renew the existing default-policy sign-off,
and document the changed formula and showcase score without changing the
sign-off mechanism itself.
Acceptance: multiple independent prompt-leakage artifacts contribute more than
equally weighted correlated style signals; the diagnostic showcase score is
pinned, the evaluation corpus remains free of observed regressions, and all
quality/coverage gates pass.
Optional follow-up, intentionally not required for this milestone:
- [ ] After observing real usage, choose one direction for the default-policy
sign-off feature: harden its change boundary beyond the catalog hash, or
remove it if its maintenance cost exceeds its demonstrated value.
## Deferred by design
- Statistical or model-based authorship classification.