diff --git a/.run_status.toml b/.run_status.toml index bf16634..c679ead 100644 --- a/.run_status.toml +++ b/.run_status.toml @@ -1,8 +1,8 @@ [last_run_clean] run_status = "Success" -lines_parsed = 337 -coverage = 92.16 +lines_parsed = 338 +coverage = 92.19 [last_run_clean.run_duration] secs = 29 -nanos = 214508516 +nanos = 930030912 diff --git a/src/detector.rs b/src/detector.rs index de8fe72..5c0fd1f 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -209,7 +209,8 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence { .map(|index| 0.5_f64.powi(i32::try_from(index).unwrap_or(i32::MAX))) .sum::(); let family_index = family_rule_counts.entry(rule.category).or_default(); - let family_factor = 0.75_f64.powi(i32::try_from(*family_index).unwrap_or(i32::MAX)); + let family_factor = + family_decay(rule.category).powi(i32::try_from(*family_index).unwrap_or(i32::MAX)); evidence += rule.weight * occurrence_factor * family_factor; *family_index += 1; } @@ -222,6 +223,13 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence { Confidence::from_score(normalized.round() as u8) } +const fn family_decay(category: Category) -> f64 { + match category { + Category::PromptLeakage => 0.90, + Category::Typography | Category::Style | Category::SourceCode => 0.75, + } +} + fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand { if score >= thresholds.high { ConfidenceBand::High @@ -739,6 +747,26 @@ mod tests { assert_eq!(correlated.findings.len(), independent.findings.len()); } + #[test] + fn prompt_leakage_uses_lighter_family_damping_than_other_categories() { + assert_eq!(family_decay(Category::PromptLeakage), 0.90); + assert_eq!(family_decay(Category::Style), 0.75); + assert_eq!(family_decay(Category::Typography), 0.75); + assert_eq!(family_decay(Category::SourceCode), 0.75); + + let source = concat!( + "User: request\n", + "Assistant: Certainly! Here's a response.\n", + "As an AI language model, I cannot browse the internet. ", + "My knowledge cutoff applies." + ); + let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source); + + assert_eq!(analysis.findings.len(), 6); + assert_eq!(analysis.confidence.score(), 67); + assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium)); + } + #[test] fn configured_thresholds_define_named_band_boundaries() { let thresholds = crate::ConfidenceThresholds { diff --git a/tasks.md b/tasks.md index 85e89e7..64917ad 100644 --- a/tasks.md +++ b/tasks.md @@ -269,6 +269,26 @@ Acceptance: a default detector invocation renders graphical Unicode diagnostics with ANSI severity colors; explicit machine modes remain stable and uncolored, and builds without `fancy-diagnostics` retain the plain human fallback. +## Milestone 10 — Prompt-leakage scoring priority + +- [x] Give distinct prompt-leakage rules lighter correlation damping (`0.90ⁿ`) + while preserving `0.75ⁿ` damping for style, typography, and source-code + families and preserving per-rule occurrence damping/caps. +- [ ] Re-run the reviewed corpus, renew the existing default-policy sign-off, + and document the changed formula and showcase score without changing the + sign-off mechanism itself. + +Acceptance: multiple independent prompt-leakage artifacts contribute more than +equally weighted correlated style signals; the diagnostic showcase score is +pinned, the evaluation corpus remains free of observed regressions, and all +quality/coverage gates pass. + +Optional follow-up, intentionally not required for this milestone: + +- [ ] After observing real usage, choose one direction for the default-policy + sign-off feature: harden its change boundary beyond the catalog hash, or + remove it if its maintenance cost exceeds its demonstrated value. + ## Deferred by design - Statistical or model-based authorship classification.