From 6fe5f90c17978a5ea68bca0f098f53e127627f6a Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 29 Sep 2026 15:27:56 +0200 Subject: [PATCH] feat: score bounded heuristic evidence --- docs/README.md | 2 + docs/heuristics.md | 31 ++++++++++++ src/detector.rs | 120 ++++++++++++++++++++++++++++++++++++++++++--- src/lib.rs | 4 +- src/model.rs | 17 ++++++- tasks.md | 2 +- 6 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 docs/heuristics.md diff --git a/docs/README.md b/docs/README.md index aa9fe74..a80c0d0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,8 @@ the details needed to use, embed, and develop John Slop: values, input safety, and the reusable library facade. - [Configuration](configuration.md) documents schema version 1, source precedence, stable-ID merges, validation, and inspection commands. +- [Heuristics and confidence](heuristics.md) documents rule execution, + applicability, evidence damping, normalization, and score interpretation. - [Development](development.md) covers the supported Rust version, feature matrix, snapshots, coverage, badges, and CI quality gates. - [Background](background.md) collects external context on observable signs of diff --git a/docs/heuristics.md b/docs/heuristics.md new file mode 100644 index 0000000..e0dae7d --- /dev/null +++ b/docs/heuristics.md @@ -0,0 +1,31 @@ +# Heuristics and confidence + +John Slop reports observable low-care signals. It does not classify authorship. +Every contribution to the score is represented by a finding with an exact +source span, stable rule ID, category, message, help, and configured weight. + +The detector compiles the resolved rule catalog once. Exact phrases, regular +expressions, repeated-pattern rules, and configured Unicode replacements share +the same finding model. Content kind, extension, path globs, and conventional +generated/vendor/minified paths decide whether each rule applies. + +## Confidence formula + +For each rule in deterministic catalog order, only the configured +`occurrence_cap` contributes to confidence. Its first occurrence receives the +full weight, then later occurrences receive one half, one quarter, and so on. +Findings beyond the cap remain visible but do not increase the score. + +Rules are grouped by category as correlated evidence families. The first +matching rule in a category receives its full occurrence-adjusted contribution; +later distinct rules receive successive factors of `0.75`. This prevents a +cluster of closely related phrases from being treated as independent proof. + +If the resulting evidence sum is `E`, the displayed score is +`round(100 × E / (E + 50))`, bounded to `0..=100`. The configured `low`, +`medium`, and `high` thresholds name the resulting band. Evidence below `low` +is explicitly reported as below threshold. No findings produce score zero and +no confidence verdict. + +Weights and thresholds are review policy, not calibrated probabilities. A +score of 75 does not mean a 75% chance that AI wrote the input. diff --git a/src/detector.rs b/src/detector.rs index 3fa78a2..112a837 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -1,13 +1,16 @@ //! Compiled, reusable registry of configured detection rules. -use std::path::{Path, PathBuf}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; use globset::{Glob, GlobSet, GlobSetBuilder}; use regex::Regex; use crate::{ - Analysis, Category, Confidence, Config, ConfigValidationError, ContentKind, Finding, - HeuristicRule, MatcherType, RuleId, SourceId, replacement, + Analysis, Category, Confidence, ConfidenceBand, Config, ConfigValidationError, ContentKind, + Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement, }; /// A validated heuristic definition paired with its reusable matcher. @@ -163,15 +166,61 @@ impl Detector { .filter(|rule| rule.is_enabled() && rule.applies(context)) .flat_map(|rule| rule.findings(&self.config, source)) .collect::>(); - let score = u8::try_from(findings.len().saturating_mul(10)).unwrap_or(u8::MAX); + let confidence = aggregate_confidence(&self.config, &findings); + let confidence_band = (!findings.is_empty()) + .then(|| band_for_score(self.config.thresholds, confidence.score())); Analysis { findings, - confidence: Confidence::from_score(score), + confidence, + confidence_band, } } } +fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence { + let mut family_rule_counts = BTreeMap::::new(); + let mut evidence = 0.0; + + for rule in &config.rules { + let occurrences = findings + .iter() + .filter(|finding| finding.rule_id.as_str() == rule.id) + .count() + .min(rule.occurrence_cap); + if occurrences == 0 { + continue; + } + + let occurrence_factor = (0..occurrences) + .map(|index| 0.5_f64.powi(i32::try_from(index).unwrap_or(i32::MAX))) + .sum::(); + let family_index = family_rule_counts.entry(rule.category).or_default(); + let family_factor = 0.75_f64.powi(i32::try_from(*family_index).unwrap_or(i32::MAX)); + evidence += rule.weight * occurrence_factor * family_factor; + *family_index += 1; + } + + let normalized = if evidence == 0.0 { + 0.0 + } else { + 100.0 * evidence / (evidence + 50.0) + }; + Confidence::from_score(normalized.round() as u8) +} + +fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand { + if score >= thresholds.high { + ConfidenceBand::High + } else if score >= thresholds.medium { + ConfidenceBand::Medium + } else if score >= thresholds.low { + ConfidenceBand::Low + } else { + ConfidenceBand::BelowThreshold + } +} + impl Rule { fn compile(definition: HeuristicRule) -> Result { let matcher = match definition.matcher { @@ -452,7 +501,11 @@ mod tests { ); assert_eq!(analysis.findings[0].matched_text, "—"); assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-"); - assert_eq!(analysis.confidence.score(), 10); + assert_eq!(analysis.confidence.score(), 17); + assert_eq!( + analysis.confidence_band, + Some(ConfidenceBand::BelowThreshold) + ); } #[test] @@ -643,4 +696,59 @@ mod tests { assert_eq!(context.path(), Some(Path::new("src/lib.rs"))); assert_eq!(context.content_kind(), ContentKind::SourceCode); } + + #[test] + fn confidence_caps_occurrences_and_applies_diminishing_returns() { + let detector = Detector::new(&Config::embedded()).unwrap(); + let five = detector.analyze(&"—".repeat(5)); + let twenty = detector.analyze(&"—".repeat(20)); + + assert_eq!(five.confidence, twenty.confidence); + assert_eq!(five.confidence.score(), 28); + assert_eq!(twenty.findings.len(), 20); + } + + #[test] + fn confidence_damps_distinct_rules_in_the_same_family() { + let source = "As an AI language model, I cannot browse the internet."; + let correlated = Detector::new(&Config::embedded()).unwrap().analyze(source); + let mut independent_config = Config::embedded(); + independent_config + .rules + .iter_mut() + .find(|rule| rule.id == "assistant-capability-disclaimer") + .unwrap() + .category = Category::Style; + let independent = Detector::new(&independent_config).unwrap().analyze(source); + + assert!(correlated.confidence < independent.confidence); + assert_eq!(correlated.findings.len(), independent.findings.len()); + } + + #[test] + fn configured_thresholds_define_named_band_boundaries() { + let thresholds = crate::ConfidenceThresholds { + low: 20, + medium: 50, + high: 75, + }; + + assert_eq!( + band_for_score(thresholds, 19), + ConfidenceBand::BelowThreshold + ); + assert_eq!(band_for_score(thresholds, 20), ConfidenceBand::Low); + assert_eq!(band_for_score(thresholds, 50), ConfidenceBand::Medium); + assert_eq!(band_for_score(thresholds, 75), ConfidenceBand::High); + } + + #[test] + fn one_weak_repeated_signal_cannot_reach_high_confidence() { + let detector = Detector::new(&Config::embedded()).unwrap(); + let source = "Moreover, first.\nAdditionally, second.\nFurthermore, third."; + let analysis = detector.analyze(source); + + assert_eq!(analysis.findings.len(), 3); + assert_ne!(analysis.confidence_band, Some(ConfidenceBand::High)); + } } diff --git a/src/lib.rs b/src/lib.rs index a5778e0..0345b98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,9 @@ pub use config::{ pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule}; pub use error::{InputError, InputResult}; pub use input::{Document, Documents}; -pub use model::{Analysis, Category, Confidence, Finding, RuleId, SourceId, TextEdit}; +pub use model::{ + Analysis, Category, Confidence, ConfidenceBand, Finding, RuleId, SourceId, TextEdit, +}; /// Load embedded defaults plus optional platform user configuration. pub fn load_config() -> Result { diff --git a/src/model.rs b/src/model.rs index 0b9f5d5..2750300 100644 --- a/src/model.rs +++ b/src/model.rs @@ -32,7 +32,7 @@ impl fmt::Display for RuleId { } /// Broad family of evidence reported by a rule. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum Category { /// Suspicious typography or character usage. @@ -63,6 +63,19 @@ impl Confidence { } } +/// Named interpretation of a non-empty analysis score. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum ConfidenceBand { + /// Evidence exists but remains below the configured low threshold. + BelowThreshold, + /// Score meets the configured low threshold. + Low, + /// Score meets the configured medium threshold. + Medium, + /// Score meets the configured high threshold. + High, +} + /// Identity of the exact source snapshot being analyzed. #[derive(Debug, Clone, PartialEq, Eq)] pub enum SourceId { @@ -128,6 +141,8 @@ pub struct Analysis { pub findings: Vec, /// Aggregate confidence that configured signals are present. pub confidence: Confidence, + /// Named score band, or no verdict when no evidence was found. + pub confidence_band: Option, } #[cfg(test)] diff --git a/tasks.md b/tasks.md index e9253ab..eaab5fb 100644 --- a/tasks.md +++ b/tasks.md @@ -117,7 +117,7 @@ configuration is explainable, and all precedence/validation tests pass. tutorial residue. - [x] Add applicability filters for prose versus source code, extension/language, include/exclude globs, and generated/vendor/minified paths. -- [ ] Design the confidence formula: bounded weighted evidence, per-rule caps, +- [x] Design the confidence formula: bounded weighted evidence, per-rule caps, correlated-family damping, normalization to `0..=100`, and configurable low/medium/high thresholds. - [ ] Return zero confidence and no verdict for empty/no-evidence input. Keep