feat: prioritize prompt leakage evidence
This commit is contained in:
parent
30445d7983
commit
d6163076d9
|
|
@ -1,8 +1,8 @@
|
|||
[last_run_clean]
|
||||
run_status = "Success"
|
||||
lines_parsed = 337
|
||||
coverage = 92.16
|
||||
lines_parsed = 338
|
||||
coverage = 92.19
|
||||
|
||||
[last_run_clean.run_duration]
|
||||
secs = 29
|
||||
nanos = 214508516
|
||||
nanos = 930030912
|
||||
|
|
|
|||
|
|
@ -209,7 +209,8 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
|
|||
.map(|index| 0.5_f64.powi(i32::try_from(index).unwrap_or(i32::MAX)))
|
||||
.sum::<f64>();
|
||||
let family_index = family_rule_counts.entry(rule.category).or_default();
|
||||
let family_factor = 0.75_f64.powi(i32::try_from(*family_index).unwrap_or(i32::MAX));
|
||||
let family_factor =
|
||||
family_decay(rule.category).powi(i32::try_from(*family_index).unwrap_or(i32::MAX));
|
||||
evidence += rule.weight * occurrence_factor * family_factor;
|
||||
*family_index += 1;
|
||||
}
|
||||
|
|
@ -222,6 +223,13 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
|
|||
Confidence::from_score(normalized.round() as u8)
|
||||
}
|
||||
|
||||
const fn family_decay(category: Category) -> f64 {
|
||||
match category {
|
||||
Category::PromptLeakage => 0.90,
|
||||
Category::Typography | Category::Style | Category::SourceCode => 0.75,
|
||||
}
|
||||
}
|
||||
|
||||
fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand {
|
||||
if score >= thresholds.high {
|
||||
ConfidenceBand::High
|
||||
|
|
@ -739,6 +747,26 @@ mod tests {
|
|||
assert_eq!(correlated.findings.len(), independent.findings.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_leakage_uses_lighter_family_damping_than_other_categories() {
|
||||
assert_eq!(family_decay(Category::PromptLeakage), 0.90);
|
||||
assert_eq!(family_decay(Category::Style), 0.75);
|
||||
assert_eq!(family_decay(Category::Typography), 0.75);
|
||||
assert_eq!(family_decay(Category::SourceCode), 0.75);
|
||||
|
||||
let source = concat!(
|
||||
"User: request\n",
|
||||
"Assistant: Certainly! Here's a response.\n",
|
||||
"As an AI language model, I cannot browse the internet. ",
|
||||
"My knowledge cutoff applies."
|
||||
);
|
||||
let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source);
|
||||
|
||||
assert_eq!(analysis.findings.len(), 6);
|
||||
assert_eq!(analysis.confidence.score(), 67);
|
||||
assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_thresholds_define_named_band_boundaries() {
|
||||
let thresholds = crate::ConfidenceThresholds {
|
||||
|
|
|
|||
20
tasks.md
20
tasks.md
|
|
@ -269,6 +269,26 @@ Acceptance: a default detector invocation renders graphical Unicode diagnostics
|
|||
with ANSI severity colors; explicit machine modes remain stable and uncolored,
|
||||
and builds without `fancy-diagnostics` retain the plain human fallback.
|
||||
|
||||
## Milestone 10 — Prompt-leakage scoring priority
|
||||
|
||||
- [x] Give distinct prompt-leakage rules lighter correlation damping (`0.90ⁿ`)
|
||||
while preserving `0.75ⁿ` damping for style, typography, and source-code
|
||||
families and preserving per-rule occurrence damping/caps.
|
||||
- [ ] Re-run the reviewed corpus, renew the existing default-policy sign-off,
|
||||
and document the changed formula and showcase score without changing the
|
||||
sign-off mechanism itself.
|
||||
|
||||
Acceptance: multiple independent prompt-leakage artifacts contribute more than
|
||||
equally weighted correlated style signals; the diagnostic showcase score is
|
||||
pinned, the evaluation corpus remains free of observed regressions, and all
|
||||
quality/coverage gates pass.
|
||||
|
||||
Optional follow-up, intentionally not required for this milestone:
|
||||
|
||||
- [ ] After observing real usage, choose one direction for the default-policy
|
||||
sign-off feature: harden its change boundary beyond the catalog hash, or
|
||||
remove it if its maintenance cost exceeds its demonstrated value.
|
||||
|
||||
## Deferred by design
|
||||
|
||||
- Statistical or model-based authorship classification.
|
||||
|
|
|
|||
Loading…
Reference in New Issue