feat: expose typed finding diagnostics

This commit is contained in:
Dandaedre 2026-09-29 15:48:58 +02:00
parent d9e4e378ea
commit 0428cd8a9a
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
8 changed files with 170 additions and 3 deletions

View File

@ -10,6 +10,7 @@ high = 75
[[rules]]
id = "assistant-self-identification"
category = "prompt-leakage"
severity = "warning"
matcher = "exact-phrase"
pattern = "As an AI language model"
weight = 30.0
@ -25,6 +26,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "knowledge-cutoff-disclaimer"
category = "prompt-leakage"
severity = "warning"
matcher = "exact-phrase"
pattern = "knowledge cutoff"
weight = 20.0
@ -40,6 +42,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "assistant-capability-disclaimer"
category = "prompt-leakage"
severity = "warning"
matcher = "exact-phrase"
pattern = "I cannot browse the internet"
weight = 20.0
@ -55,6 +58,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "assistant-answer-preface"
category = "prompt-leakage"
severity = "warning"
matcher = "exact-phrase"
pattern = "Certainly! Here's"
weight = 15.0
@ -70,6 +74,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "prompt-response-scaffolding"
category = "prompt-leakage"
severity = "warning"
matcher = "regex"
pattern = "(?m)^(User|Assistant):[ \\t]"
weight = 25.0
@ -85,6 +90,7 @@ extensions = []
[[rules]]
id = "templated-conclusion"
category = "style"
severity = "advice"
matcher = "regex"
pattern = "(?i)\\bin conclusion,? (it is|we can)"
weight = 5.0
@ -100,6 +106,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "templated-filler"
category = "style"
severity = "advice"
matcher = "regex"
pattern = "(?i)\\bit is (?:important|worth) (?:to note|noting) that\\b"
weight = 4.0
@ -115,6 +122,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "repeated-transition"
category = "style"
severity = "advice"
matcher = "repeated-regex"
pattern = "(?im)^(?:Moreover|Furthermore|Additionally),[ \\t]"
weight = 4.0
@ -130,6 +138,7 @@ extensions = ["md", "txt", "rst", "adoc"]
[[rules]]
id = "source-placeholder-todo"
category = "source-code"
severity = "warning"
matcher = "regex"
pattern = "(?i)\\bTODO:[ \\t]+(implement|add implementation)\\b"
weight = 10.0
@ -145,6 +154,7 @@ extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
[[rules]]
id = "source-tutorial-narration"
category = "source-code"
severity = "advice"
matcher = "regex"
pattern = "(?im)^[ \\t]*//[ \\t]*(?:first|next|finally),?[ \\t]+we\\b"
weight = 8.0
@ -160,6 +170,7 @@ extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
[[rules]]
id = "typography-non-ascii-hyphen"
category = "typography"
severity = "advice"
matcher = "replacement"
pattern = "configured-replacements"
weight = 10.0

View File

@ -56,6 +56,10 @@ layer, invalid regular expressions or globs, unsafe matchers, non-ASCII
replacement targets, and invalid thresholds are fatal configuration
diagnostics.
Each heuristic also configures `severity` as `advice`, `warning`, or `error`.
Severity controls diagnostic presentation and summary counts; confidence still
comes from the independently configured weight and evidence formula.
## Inspection
Print the fully layered and validated model as deterministic TOML:

View File

@ -11,7 +11,7 @@ use figment::{
use miette::{Diagnostic, NamedSource, SourceSpan};
use serde::{Deserialize, Serialize};
use crate::Category;
use crate::{Category, FindingSeverity};
const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml");
const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml");
@ -487,6 +487,7 @@ struct ReplacementPatch {
struct RulePatch {
id: String,
category: Option<Category>,
severity: Option<FindingSeverity>,
matcher: Option<MatcherType>,
pattern: Option<String>,
weight: Option<f64>,
@ -589,6 +590,9 @@ fn merge_rules(
if let Some(value) = patch.category {
rule.category = value;
}
if let Some(value) = patch.severity {
rule.severity = value;
}
if let Some(value) = patch.matcher {
rule.matcher = value;
}
@ -617,6 +621,7 @@ fn merge_rules(
let id = patch.id;
target.push(HeuristicRule {
category: required(patch.category, "rule", &id, "category", path)?,
severity: required(patch.severity, "rule", &id, "severity", path)?,
matcher: required(patch.matcher, "rule", &id, "matcher", path)?,
pattern: required(patch.pattern, "rule", &id, "pattern", path)?,
weight: required(patch.weight, "rule", &id, "weight", path)?,
@ -737,6 +742,8 @@ pub struct HeuristicRule {
pub id: String,
/// Broad evidence family.
pub category: Category,
/// Presentation severity for each finding produced by this rule.
pub severity: FindingSeverity,
/// Matching implementation selected for this rule.
pub matcher: MatcherType,
/// Exact phrase, regular expression, or named replacement source.

View File

@ -345,6 +345,7 @@ impl Rule {
Finding {
rule_id: RuleId::new(self.id()),
category: self.category(),
severity: self.definition.severity,
message: self.definition.message.clone(),
help: Some(self.definition.help.clone()),
matched_text: source[byte_span.clone()].to_owned(),

127
src/diagnostic.rs Normal file
View File

@ -0,0 +1,127 @@
//! Rich source diagnostics derived from presentation-neutral findings.
use std::{error::Error, fmt};
use miette::{Diagnostic, LabeledSpan, NamedSource, Severity, SourceCode, SourceSpan};
use crate::{Finding, FindingSeverity, RuleId, SourceId};
/// Owned `miette` diagnostic for one finding against an exact source snapshot.
#[derive(Debug)]
pub struct FindingDiagnostic {
rule_id: RuleId,
severity: FindingSeverity,
message: String,
help: Option<String>,
matched_text: String,
source: NamedSource<String>,
span: SourceSpan,
}
impl FindingDiagnostic {
/// Create a diagnostic by binding a finding to its named source snapshot.
pub fn new(source_id: &SourceId, source: &str, finding: &Finding) -> Self {
Self {
rule_id: finding.rule_id.clone(),
severity: finding.severity,
message: finding.message.clone(),
help: finding.help.clone(),
matched_text: finding.matched_text.clone(),
source: NamedSource::new(source_id.to_string(), source.to_owned()),
span: finding.byte_span.clone().into(),
}
}
/// Return the stable rule identifier used as the diagnostic code.
pub fn rule_id(&self) -> &RuleId {
&self.rule_id
}
/// Return the configured finding severity.
pub const fn finding_severity(&self) -> FindingSeverity {
self.severity
}
/// Return the exact source span labeled by this diagnostic.
pub const fn span(&self) -> SourceSpan {
self.span
}
/// Return the named source snapshot used for rendering.
pub fn named_source(&self) -> &NamedSource<String> {
&self.source
}
}
impl fmt::Display for FindingDiagnostic {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
self.message.fmt(formatter)
}
}
impl Error for FindingDiagnostic {}
impl Diagnostic for FindingDiagnostic {
fn code(&self) -> Option<Box<dyn fmt::Display + '_>> {
Some(Box::new(&self.rule_id))
}
fn severity(&self) -> Option<Severity> {
Some(match self.severity {
FindingSeverity::Advice => Severity::Advice,
FindingSeverity::Warning => Severity::Warning,
FindingSeverity::Error => Severity::Error,
})
}
fn help(&self) -> Option<Box<dyn fmt::Display + '_>> {
self.help
.as_deref()
.map(|help| Box::new(help) as Box<dyn fmt::Display>)
}
fn source_code(&self) -> Option<&dyn SourceCode> {
Some(&self.source)
}
fn labels(&self) -> Option<Box<dyn Iterator<Item = LabeledSpan> + '_>> {
Some(Box::new(std::iter::once(
LabeledSpan::new_primary_with_span(Some(self.matched_text.clone()), self.span),
)))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{AnalysisContext, Config, Detector};
#[test]
fn finding_diagnostic_preserves_code_severity_source_span_and_help() {
let source = "préface: As an AI language model, I can help.";
let finding = Detector::new(&Config::embedded())
.unwrap()
.analyze_with(source, &AnalysisContext::for_path("article.md"))
.findings
.into_iter()
.find(|finding| finding.rule_id.as_str() == "assistant-self-identification")
.unwrap();
let diagnostic =
FindingDiagnostic::new(&SourceId::File("article.md".into()), source, &finding);
assert_eq!(
diagnostic.code().unwrap().to_string(),
finding.rule_id.as_str()
);
assert_eq!(diagnostic.severity(), Some(Severity::Warning));
assert!(diagnostic.help().unwrap().to_string().contains("copied"));
assert_eq!(diagnostic.named_source().name(), "article.md");
let label = diagnostic.labels().unwrap().next().unwrap();
assert_eq!(label.offset(), finding.byte_span.start);
assert_eq!(label.len(), finding.byte_span.len());
assert_eq!(
&source[label.offset()..label.offset() + label.len()],
finding.matched_text
);
}
}

View File

@ -7,6 +7,7 @@ use std::path::Path;
mod config;
mod detector;
mod diagnostic;
mod error;
mod input;
mod model;
@ -19,10 +20,12 @@ pub use config::{
ThresholdOverrides,
};
pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule};
pub use diagnostic::FindingDiagnostic;
pub use error::{InputError, InputResult};
pub use input::{Document, Documents};
pub use model::{
Analysis, Category, Confidence, ConfidenceBand, Finding, RuleId, SourceId, TextEdit,
Analysis, Category, Confidence, ConfidenceBand, Finding, FindingSeverity, RuleId, SourceId,
TextEdit,
};
/// Load embedded defaults plus optional platform user configuration.

View File

@ -45,6 +45,18 @@ pub enum Category {
SourceCode,
}
/// Review severity attached to an individual configured finding.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum FindingSeverity {
/// Informational signal worth considering during review.
Advice,
/// Stronger signal that should be reviewed before acceptance.
Warning,
/// Definite policy violation under the selected configuration.
Error,
}
/// Confidence that configured low-care signals are present.
///
/// This is deliberately not an AI-authorship probability.
@ -120,6 +132,8 @@ pub struct Finding {
pub rule_id: RuleId,
/// Evidence family.
pub category: Category,
/// Configured review severity.
pub severity: FindingSeverity,
/// Concise description of the observed signal.
pub message: String,
/// Optional remediation or review guidance.

View File

@ -138,7 +138,7 @@ formula are pinned by tests.
## Milestone 4 — Diagnostics and detector CLI
- [ ] Convert each `Finding` into a typed `miette::Diagnostic` with named source,
- [x] Convert each `Finding` into a typed `miette::Diagnostic` with named source,
exact `SourceSpan`, severity, rule ID as diagnostic code, message, and help.
- [ ] Add a summary that reports confidence band/score and counts by severity,
while retaining every contributing diagnostic.