feat: expose typed finding diagnostics
This commit is contained in:
parent
d9e4e378ea
commit
0428cd8a9a
|
|
@ -10,6 +10,7 @@ high = 75
|
|||
[[rules]]
|
||||
id = "assistant-self-identification"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "As an AI language model"
|
||||
weight = 30.0
|
||||
|
|
@ -25,6 +26,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "knowledge-cutoff-disclaimer"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "knowledge cutoff"
|
||||
weight = 20.0
|
||||
|
|
@ -40,6 +42,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "assistant-capability-disclaimer"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "I cannot browse the internet"
|
||||
weight = 20.0
|
||||
|
|
@ -55,6 +58,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "assistant-answer-preface"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "exact-phrase"
|
||||
pattern = "Certainly! Here's"
|
||||
weight = 15.0
|
||||
|
|
@ -70,6 +74,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "prompt-response-scaffolding"
|
||||
category = "prompt-leakage"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?m)^(User|Assistant):[ \\t]"
|
||||
weight = 25.0
|
||||
|
|
@ -85,6 +90,7 @@ extensions = []
|
|||
[[rules]]
|
||||
id = "templated-conclusion"
|
||||
category = "style"
|
||||
severity = "advice"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\bin conclusion,? (it is|we can)"
|
||||
weight = 5.0
|
||||
|
|
@ -100,6 +106,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "templated-filler"
|
||||
category = "style"
|
||||
severity = "advice"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\bit is (?:important|worth) (?:to note|noting) that\\b"
|
||||
weight = 4.0
|
||||
|
|
@ -115,6 +122,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "repeated-transition"
|
||||
category = "style"
|
||||
severity = "advice"
|
||||
matcher = "repeated-regex"
|
||||
pattern = "(?im)^(?:Moreover|Furthermore|Additionally),[ \\t]"
|
||||
weight = 4.0
|
||||
|
|
@ -130,6 +138,7 @@ extensions = ["md", "txt", "rst", "adoc"]
|
|||
[[rules]]
|
||||
id = "source-placeholder-todo"
|
||||
category = "source-code"
|
||||
severity = "warning"
|
||||
matcher = "regex"
|
||||
pattern = "(?i)\\bTODO:[ \\t]+(implement|add implementation)\\b"
|
||||
weight = 10.0
|
||||
|
|
@ -145,6 +154,7 @@ extensions = ["c", "cpp", "go", "java", "js", "py", "rs", "ts"]
|
|||
[[rules]]
|
||||
id = "source-tutorial-narration"
|
||||
category = "source-code"
|
||||
severity = "advice"
|
||||
matcher = "regex"
|
||||
pattern = "(?im)^[ \\t]*//[ \\t]*(?:first|next|finally),?[ \\t]+we\\b"
|
||||
weight = 8.0
|
||||
|
|
@ -160,6 +170,7 @@ extensions = ["c", "cpp", "go", "java", "js", "rs", "ts"]
|
|||
[[rules]]
|
||||
id = "typography-non-ascii-hyphen"
|
||||
category = "typography"
|
||||
severity = "advice"
|
||||
matcher = "replacement"
|
||||
pattern = "configured-replacements"
|
||||
weight = 10.0
|
||||
|
|
|
|||
|
|
@ -56,6 +56,10 @@ layer, invalid regular expressions or globs, unsafe matchers, non-ASCII
|
|||
replacement targets, and invalid thresholds are fatal configuration
|
||||
diagnostics.
|
||||
|
||||
Each heuristic also configures `severity` as `advice`, `warning`, or `error`.
|
||||
Severity controls diagnostic presentation and summary counts; confidence still
|
||||
comes from the independently configured weight and evidence formula.
|
||||
|
||||
## Inspection
|
||||
|
||||
Print the fully layered and validated model as deterministic TOML:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use figment::{
|
|||
use miette::{Diagnostic, NamedSource, SourceSpan};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::Category;
|
||||
use crate::{Category, FindingSeverity};
|
||||
|
||||
const EMBEDDED_CONFIG: &str = include_str!("../assets/config.toml");
|
||||
const EMBEDDED_HEURISTICS: &str = include_str!("../assets/heuristics.toml");
|
||||
|
|
@ -487,6 +487,7 @@ struct ReplacementPatch {
|
|||
struct RulePatch {
|
||||
id: String,
|
||||
category: Option<Category>,
|
||||
severity: Option<FindingSeverity>,
|
||||
matcher: Option<MatcherType>,
|
||||
pattern: Option<String>,
|
||||
weight: Option<f64>,
|
||||
|
|
@ -589,6 +590,9 @@ fn merge_rules(
|
|||
if let Some(value) = patch.category {
|
||||
rule.category = value;
|
||||
}
|
||||
if let Some(value) = patch.severity {
|
||||
rule.severity = value;
|
||||
}
|
||||
if let Some(value) = patch.matcher {
|
||||
rule.matcher = value;
|
||||
}
|
||||
|
|
@ -617,6 +621,7 @@ fn merge_rules(
|
|||
let id = patch.id;
|
||||
target.push(HeuristicRule {
|
||||
category: required(patch.category, "rule", &id, "category", path)?,
|
||||
severity: required(patch.severity, "rule", &id, "severity", path)?,
|
||||
matcher: required(patch.matcher, "rule", &id, "matcher", path)?,
|
||||
pattern: required(patch.pattern, "rule", &id, "pattern", path)?,
|
||||
weight: required(patch.weight, "rule", &id, "weight", path)?,
|
||||
|
|
@ -737,6 +742,8 @@ pub struct HeuristicRule {
|
|||
pub id: String,
|
||||
/// Broad evidence family.
|
||||
pub category: Category,
|
||||
/// Presentation severity for each finding produced by this rule.
|
||||
pub severity: FindingSeverity,
|
||||
/// Matching implementation selected for this rule.
|
||||
pub matcher: MatcherType,
|
||||
/// Exact phrase, regular expression, or named replacement source.
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ impl Rule {
|
|||
Finding {
|
||||
rule_id: RuleId::new(self.id()),
|
||||
category: self.category(),
|
||||
severity: self.definition.severity,
|
||||
message: self.definition.message.clone(),
|
||||
help: Some(self.definition.help.clone()),
|
||||
matched_text: source[byte_span.clone()].to_owned(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
//! Rich source diagnostics derived from presentation-neutral findings.
|
||||
|
||||
use std::{error::Error, fmt};
|
||||
|
||||
use miette::{Diagnostic, LabeledSpan, NamedSource, Severity, SourceCode, SourceSpan};
|
||||
|
||||
use crate::{Finding, FindingSeverity, RuleId, SourceId};
|
||||
|
||||
/// Owned `miette` diagnostic for one finding against an exact source snapshot.
|
||||
#[derive(Debug)]
|
||||
pub struct FindingDiagnostic {
|
||||
rule_id: RuleId,
|
||||
severity: FindingSeverity,
|
||||
message: String,
|
||||
help: Option<String>,
|
||||
matched_text: String,
|
||||
source: NamedSource<String>,
|
||||
span: SourceSpan,
|
||||
}
|
||||
|
||||
impl FindingDiagnostic {
|
||||
/// Create a diagnostic by binding a finding to its named source snapshot.
|
||||
pub fn new(source_id: &SourceId, source: &str, finding: &Finding) -> Self {
|
||||
Self {
|
||||
rule_id: finding.rule_id.clone(),
|
||||
severity: finding.severity,
|
||||
message: finding.message.clone(),
|
||||
help: finding.help.clone(),
|
||||
matched_text: finding.matched_text.clone(),
|
||||
source: NamedSource::new(source_id.to_string(), source.to_owned()),
|
||||
span: finding.byte_span.clone().into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the stable rule identifier used as the diagnostic code.
|
||||
pub fn rule_id(&self) -> &RuleId {
|
||||
&self.rule_id
|
||||
}
|
||||
|
||||
/// Return the configured finding severity.
|
||||
pub const fn finding_severity(&self) -> FindingSeverity {
|
||||
self.severity
|
||||
}
|
||||
|
||||
/// Return the exact source span labeled by this diagnostic.
|
||||
pub const fn span(&self) -> SourceSpan {
|
||||
self.span
|
||||
}
|
||||
|
||||
/// Return the named source snapshot used for rendering.
|
||||
pub fn named_source(&self) -> &NamedSource<String> {
|
||||
&self.source
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FindingDiagnostic {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
self.message.fmt(formatter)
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for FindingDiagnostic {}
|
||||
|
||||
impl Diagnostic for FindingDiagnostic {
|
||||
fn code(&self) -> Option<Box<dyn fmt::Display + '_>> {
|
||||
Some(Box::new(&self.rule_id))
|
||||
}
|
||||
|
||||
fn severity(&self) -> Option<Severity> {
|
||||
Some(match self.severity {
|
||||
FindingSeverity::Advice => Severity::Advice,
|
||||
FindingSeverity::Warning => Severity::Warning,
|
||||
FindingSeverity::Error => Severity::Error,
|
||||
})
|
||||
}
|
||||
|
||||
fn help(&self) -> Option<Box<dyn fmt::Display + '_>> {
|
||||
self.help
|
||||
.as_deref()
|
||||
.map(|help| Box::new(help) as Box<dyn fmt::Display>)
|
||||
}
|
||||
|
||||
fn source_code(&self) -> Option<&dyn SourceCode> {
|
||||
Some(&self.source)
|
||||
}
|
||||
|
||||
fn labels(&self) -> Option<Box<dyn Iterator<Item = LabeledSpan> + '_>> {
|
||||
Some(Box::new(std::iter::once(
|
||||
LabeledSpan::new_primary_with_span(Some(self.matched_text.clone()), self.span),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{AnalysisContext, Config, Detector};
|
||||
|
||||
#[test]
|
||||
fn finding_diagnostic_preserves_code_severity_source_span_and_help() {
|
||||
let source = "préface: As an AI language model, I can help.";
|
||||
let finding = Detector::new(&Config::embedded())
|
||||
.unwrap()
|
||||
.analyze_with(source, &AnalysisContext::for_path("article.md"))
|
||||
.findings
|
||||
.into_iter()
|
||||
.find(|finding| finding.rule_id.as_str() == "assistant-self-identification")
|
||||
.unwrap();
|
||||
let diagnostic =
|
||||
FindingDiagnostic::new(&SourceId::File("article.md".into()), source, &finding);
|
||||
|
||||
assert_eq!(
|
||||
diagnostic.code().unwrap().to_string(),
|
||||
finding.rule_id.as_str()
|
||||
);
|
||||
assert_eq!(diagnostic.severity(), Some(Severity::Warning));
|
||||
assert!(diagnostic.help().unwrap().to_string().contains("copied"));
|
||||
assert_eq!(diagnostic.named_source().name(), "article.md");
|
||||
let label = diagnostic.labels().unwrap().next().unwrap();
|
||||
assert_eq!(label.offset(), finding.byte_span.start);
|
||||
assert_eq!(label.len(), finding.byte_span.len());
|
||||
assert_eq!(
|
||||
&source[label.offset()..label.offset() + label.len()],
|
||||
finding.matched_text
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ use std::path::Path;
|
|||
|
||||
mod config;
|
||||
mod detector;
|
||||
mod diagnostic;
|
||||
mod error;
|
||||
mod input;
|
||||
mod model;
|
||||
|
|
@ -19,10 +20,12 @@ pub use config::{
|
|||
ThresholdOverrides,
|
||||
};
|
||||
pub use detector::{AnalysisContext, Detector, DetectorBuildError, Rule};
|
||||
pub use diagnostic::FindingDiagnostic;
|
||||
pub use error::{InputError, InputResult};
|
||||
pub use input::{Document, Documents};
|
||||
pub use model::{
|
||||
Analysis, Category, Confidence, ConfidenceBand, Finding, RuleId, SourceId, TextEdit,
|
||||
Analysis, Category, Confidence, ConfidenceBand, Finding, FindingSeverity, RuleId, SourceId,
|
||||
TextEdit,
|
||||
};
|
||||
|
||||
/// Load embedded defaults plus optional platform user configuration.
|
||||
|
|
|
|||
14
src/model.rs
14
src/model.rs
|
|
@ -45,6 +45,18 @@ pub enum Category {
|
|||
SourceCode,
|
||||
}
|
||||
|
||||
/// Review severity attached to an individual configured finding.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum FindingSeverity {
|
||||
/// Informational signal worth considering during review.
|
||||
Advice,
|
||||
/// Stronger signal that should be reviewed before acceptance.
|
||||
Warning,
|
||||
/// Definite policy violation under the selected configuration.
|
||||
Error,
|
||||
}
|
||||
|
||||
/// Confidence that configured low-care signals are present.
|
||||
///
|
||||
/// This is deliberately not an AI-authorship probability.
|
||||
|
|
@ -120,6 +132,8 @@ pub struct Finding {
|
|||
pub rule_id: RuleId,
|
||||
/// Evidence family.
|
||||
pub category: Category,
|
||||
/// Configured review severity.
|
||||
pub severity: FindingSeverity,
|
||||
/// Concise description of the observed signal.
|
||||
pub message: String,
|
||||
/// Optional remediation or review guidance.
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -138,7 +138,7 @@ formula are pinned by tests.
|
|||
|
||||
## Milestone 4 — Diagnostics and detector CLI
|
||||
|
||||
- [ ] Convert each `Finding` into a typed `miette::Diagnostic` with named source,
|
||||
- [x] Convert each `Finding` into a typed `miette::Diagnostic` with named source,
|
||||
exact `SourceSpan`, severity, rule ID as diagnostic code, message, and help.
|
||||
- [ ] Add a summary that reports confidence band/score and counts by severity,
|
||||
while retaining every contributing diagnostic.
|
||||
|
|
|
|||
Loading…
Reference in New Issue