diff --git a/assets/heuristics.toml b/assets/heuristics.toml index f2fb73e..948a62b 100644 --- a/assets/heuristics.toml +++ b/assets/heuristics.toml @@ -10,6 +10,7 @@ high = 75 [[rules]] id = "assistant-self-identification" category = "prompt-leakage" +severity = "warning" matcher = "exact-phrase" pattern = "As an AI language model" weight = 30.0 @@ -25,6 +26,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "knowledge-cutoff-disclaimer" category = "prompt-leakage" +severity = "warning" matcher = "exact-phrase" pattern = "knowledge cutoff" weight = 20.0 @@ -40,6 +42,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "assistant-capability-disclaimer" category = "prompt-leakage" +severity = "warning" matcher = "exact-phrase" pattern = "I cannot browse the internet" weight = 20.0 @@ -55,6 +58,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "assistant-answer-preface" category = "prompt-leakage" +severity = "warning" matcher = "exact-phrase" pattern = "Certainly! Here's" weight = 15.0 @@ -70,6 +74,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "prompt-response-scaffolding" category = "prompt-leakage" +severity = "warning" matcher = "regex" pattern = "(?m)^(User|Assistant):[ \\t]" weight = 25.0 @@ -85,6 +90,7 @@ extensions = [] [[rules]] id = "templated-conclusion" category = "style" +severity = "advice" matcher = "regex" pattern = "(?i)\\bin conclusion,? (it is|we can)" weight = 5.0 @@ -100,6 +106,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "templated-filler" category = "style" +severity = "advice" matcher = "regex" pattern = "(?i)\\bit is (?:important|worth) (?:to note|noting) that\\b" weight = 4.0 @@ -115,6 +122,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "repeated-transition" category = "style" +severity = "advice" matcher = "repeated-regex" pattern = "(?im)^(?:Moreover|Furthermore|Additionally),[ \\t]" weight = 4.0 @@ -130,6 +138,7 @@ extensions = ["md", "txt", "rst", "adoc"] [[rules]] id = "source-placeholder-todo" category = "source-code" +severity = "warning" matcher = "regex" pattern = "(?i)\\bTODO:[ \\t]+(implement|add implementation)\\b" weight = 10.0 @@ -145,6 +154,7 @@ extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"] [[rules]] id = "source-tutorial-narration" category = "source-code" +severity = "advice" matcher = "regex" pattern = "(?im)^[ \\t]*//[ \\t]*(?:first|next|finally),?[ \\t]+we\\b" weight = 8.0 @@ -160,6 +170,7 @@ extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"] [[rules]] id = "typography-non-ascii-hyphen" category = "typography" +severity = "advice" matcher = "replacement" pattern = "configured-replacements" weight = 10.0 diff --git a/docs/configuration.md b/docs/configuration.md index d260372..7e5df37 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,6 +56,10 @@ layer, invalid regular expressions or globs, unsafe matchers, non-ASCII replacement targets, and invalid thresholds are fatal configuration diagnostics. +Each heuristic also configures `severity` as `advice`, `warning`, or `error`. +Severity controls diagnostic presentation and summary counts; confidence still +comes from the independently configured weight and evidence formula. + ## Inspection Print the fully layered and validated model as deterministic TOML: diff --git a/src/config.rs b/src/config.rs index ba8c78a..a48e9ca 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,7 +11,7 @@ use figment::{ use miette::{Diagnostic, NamedSource, SourceSpan}; use serde::{Deserialize, Serialize}; -use crate::Category; +use crate::{Category, FindingSeverity}; const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml"); const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml"); @@ -487,6 +487,7 @@ struct ReplacementPatch { struct RulePatch { id: String, category: Option, + severity: Option, matcher: Option, pattern: Option, weight: Option, @@ -589,6 +590,9 @@ fn merge_rules( if let Some(value) = patch.category { rule.category = value; } + if let Some(value) = patch.severity { + rule.severity = value; + } if let Some(value) = patch.matcher { rule.matcher = value; } @@ -617,6 +621,7 @@ fn merge_rules( let id = patch.id; target.push(HeuristicRule { category: required(patch.category, "rule", &id, "category", path)?, + severity: required(patch.severity, "rule", &id, "severity", path)?, matcher: required(patch.matcher, "rule", &id, "matcher", path)?, pattern: required(patch.pattern, "rule", &id, "pattern", path)?, weight: required(patch.weight, "rule", &id, "weight", path)?, @@ -737,6 +742,8 @@ pub struct HeuristicRule { pub id: String, /// Broad evidence family. pub category: Category, + /// Presentation severity for each finding produced by this rule. + pub severity: FindingSeverity, /// Matching implementation selected for this rule. pub matcher: MatcherType, /// Exact phrase, regular expression, or named replacement source. diff --git a/src/detector.rs b/src/detector.rs index 9e3d241..de8fe72 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -345,6 +345,7 @@ impl Rule { Finding { rule_id: RuleId::new(self.id()), category: self.category(), + severity: self.definition.severity, message: self.definition.message.clone(), help: Some(self.definition.help.clone()), matched_text: source[byte_span.clone()].to_owned(), diff --git a/src/diagnostic.rs b/src/diagnostic.rs new file mode 100644 index 0000000..f7fea62 --- /dev/null +++ b/src/diagnostic.rs @@ -0,0 +1,127 @@ +//! Rich source diagnostics derived from presentation-neutral findings. + +use std::{error::Error, fmt}; + +use miette::{Diagnostic, LabeledSpan, NamedSource, Severity, SourceCode, SourceSpan}; + +use crate::{Finding, FindingSeverity, RuleId, SourceId}; + +/// Owned `miette` diagnostic for one finding against an exact source snapshot. +#[derive(Debug)] +pub struct FindingDiagnostic { + rule_id: RuleId, + severity: FindingSeverity, + message: String, + help: Option, + matched_text: String, + source: NamedSource, + span: SourceSpan, +} + +impl FindingDiagnostic { + /// Create a diagnostic by binding a finding to its named source snapshot. + pub fn new(source_id: &SourceId, source: &str, finding: &Finding) -> Self { + Self { + rule_id: finding.rule_id.clone(), + severity: finding.severity, + message: finding.message.clone(), + help: finding.help.clone(), + matched_text: finding.matched_text.clone(), + source: NamedSource::new(source_id.to_string(), source.to_owned()), + span: finding.byte_span.clone().into(), + } + } + + /// Return the stable rule identifier used as the diagnostic code. + pub fn rule_id(&self) -> &RuleId { + &self.rule_id + } + + /// Return the configured finding severity. + pub const fn finding_severity(&self) -> FindingSeverity { + self.severity + } + + /// Return the exact source span labeled by this diagnostic. + pub const fn span(&self) -> SourceSpan { + self.span + } + + /// Return the named source snapshot used for rendering. + pub fn named_source(&self) -> &NamedSource { + &self.source + } +} + +impl fmt::Display for FindingDiagnostic { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.message.fmt(formatter) + } +} + +impl Error for FindingDiagnostic {} + +impl Diagnostic for FindingDiagnostic { + fn code(&self) -> Option> { + Some(Box::new(&self.rule_id)) + } + + fn severity(&self) -> Option { + Some(match self.severity { + FindingSeverity::Advice => Severity::Advice, + FindingSeverity::Warning => Severity::Warning, + FindingSeverity::Error => Severity::Error, + }) + } + + fn help(&self) -> Option> { + self.help + .as_deref() + .map(|help| Box::new(help) as Box) + } + + fn source_code(&self) -> Option<&dyn SourceCode> { + Some(&self.source) + } + + fn labels(&self) -> Option + '_>> { + Some(Box::new(std::iter::once( + LabeledSpan::new_primary_with_span(Some(self.matched_text.clone()), self.span), + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AnalysisContext, Config, Detector}; + + #[test] + fn finding_diagnostic_preserves_code_severity_source_span_and_help() { + let source = "préface: As an AI language model, I can help."; + let finding = Detector::new(&Config::embedded()) + .unwrap() + .analyze_with(source, &AnalysisContext::for_path("article.md")) + .findings + .into_iter() + .find(|finding| finding.rule_id.as_str() == "assistant-self-identification") + .unwrap(); + let diagnostic = + FindingDiagnostic::new(&SourceId::File("article.md".into()), source, &finding); + + assert_eq!( + diagnostic.code().unwrap().to_string(), + finding.rule_id.as_str() + ); + assert_eq!(diagnostic.severity(), Some(Severity::Warning)); + assert!(diagnostic.help().unwrap().to_string().contains("copied")); + assert_eq!(diagnostic.named_source().name(), "article.md"); + let label = diagnostic.labels().unwrap().next().unwrap(); + assert_eq!(label.offset(), finding.byte_span.start); + assert_eq!(label.len(), finding.byte_span.len()); + assert_eq!( + &source[label.offset()..label.offset() + label.len()], + finding.matched_text + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 0345b98..15e07ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use std::path::Path; mod config; mod detector; +mod diagnostic; mod error; mod input; mod model; @@ -19,10 +20,12 @@ pub use config::{ ThresholdOverrides, }; pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule}; +pub use diagnostic::FindingDiagnostic; pub use error::{InputError, InputResult}; pub use input::{Document, Documents}; pub use model::{ - Analysis, Category, Confidence, ConfidenceBand, Finding, RuleId, SourceId, TextEdit, + Analysis, Category, Confidence, ConfidenceBand, Finding, FindingSeverity, RuleId, SourceId, + TextEdit, }; /// Load embedded defaults plus optional platform user configuration. diff --git a/src/model.rs b/src/model.rs index 2750300..78bd172 100644 --- a/src/model.rs +++ b/src/model.rs @@ -45,6 +45,18 @@ pub enum Category { SourceCode, } +/// Review severity attached to an individual configured finding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FindingSeverity { + /// Informational signal worth considering during review. + Advice, + /// Stronger signal that should be reviewed before acceptance. + Warning, + /// Definite policy violation under the selected configuration. + Error, +} + /// Confidence that configured low-care signals are present. /// /// This is deliberately not an AI-authorship probability. @@ -120,6 +132,8 @@ pub struct Finding { pub rule_id: RuleId, /// Evidence family. pub category: Category, + /// Configured review severity. + pub severity: FindingSeverity, /// Concise description of the observed signal. pub message: String, /// Optional remediation or review guidance. diff --git a/tasks.md b/tasks.md index 1b03cc5..a6dfe26 100644 --- a/tasks.md +++ b/tasks.md @@ -138,7 +138,7 @@ formula are pinned by tests. ## Milestone 4 — Diagnostics and detector CLI -- [ ] Convert each `Finding` into a typed `miette::Diagnostic` with named source, +- [x] Convert each `Finding` into a typed `miette::Diagnostic` with named source, exact `SourceSpan`, severity, rule ID as diagnostic code, message, and help. - [ ] Add a summary that reports confidence band/score and counts by severity, while retaining every contributing diagnostic.