diff --git a/examples/analyze_path.rs b/examples/analyze_path.rs index 01486d6..0ccc9de 100644 --- a/examples/analyze_path.rs +++ b/examples/analyze_path.rs @@ -17,9 +17,9 @@ fn main() -> Result<(), Box> { let context = AnalysisContext::for_source(&document.source); let analysis = detector.analyze_with(&document.content, &context); println!( - "{}: confidence={}, findings={}", + "{}: evidence_score={}, findings={}", document.source, - analysis.confidence.score(), + analysis.evidence_score.score(), analysis.findings.len() ); } diff --git a/examples/analyze_text.rs b/examples/analyze_text.rs index a20ccb1..6c02cc1 100644 --- a/examples/analyze_text.rs +++ b/examples/analyze_text.rs @@ -11,11 +11,11 @@ fn main() -> Result<(), Box> { let summary = analysis.summary(); println!( - "confidence={} band={} findings={}", - summary.confidence_score, + "evidence_score={} band={} findings={}", + summary.evidence_score, summary - .confidence_band - .map_or("none", john_slop::ConfidenceBand::as_str), + .evidence_band + .map_or("none", john_slop::EvidenceBand::as_str), summary.findings.total() ); diff --git a/examples/config/config.toml b/examples/config/config.toml index 36c23da..32e88cb 100644 --- a/examples/config/config.toml +++ b/examples/config/config.toml @@ -7,7 +7,7 @@ schema_version = 1 max_document_bytes = 4194304 [thresholds] -# Confidence bands affect labels and `--fail-level`; they are not probabilities. +# Evidence bands affect labels and `--fail-level`; they are not probabilities. low = 20 medium = 50 high = 75 @@ -25,6 +25,6 @@ enabled = false [[rules]] id = "source-placeholder-todo" -# Severity changes presentation; weight changes confidence contribution. +# Severity changes presentation; weight changes evidence-score contribution. severity = "error" weight = 15.0 diff --git a/src/bin/john-slop.rs b/src/bin/john-slop.rs index c5b419e..634cd81 100644 --- a/src/bin/john-slop.rs +++ b/src/bin/john-slop.rs @@ -83,17 +83,18 @@ struct Cli { )] output: OutputFormat, - /// Return exit code 1 when a document reaches this evidence-confidence score. + /// Return exit code 1 when a document reaches this low-care evidence score. #[arg( long, + visible_alias = "fail-confidence", default_value_t = 100, - value_parser = parse_confidence, + value_parser = parse_evidence_score, conflicts_with = "fail_level" )] - fail_confidence: u8, + fail_score: u8, /// Return exit code 1 at the configured low, medium, or high threshold. - #[arg(long, value_enum, conflicts_with = "fail_confidence")] + #[arg(long, value_enum, conflicts_with = "fail_score")] fail_level: Option, /// Files, directories, or supported ZIP-based containers to inspect. @@ -125,7 +126,7 @@ enum ConfigCommand { Paths, } -/// Configured confidence band that should make analysis fail. +/// Configured evidence band that should make analysis fail. #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] enum FailLevel { Low, @@ -200,7 +201,8 @@ fn run(cli: &Cli) -> Result { render_analysis(cli.output, &document.source, &document.content, &analysis) .into_diagnostic()? ); - outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold; + outcome.threshold_reached |= + analysis.evidence_score.score() >= failure_threshold; } } Err(error) => { @@ -241,7 +243,7 @@ fn run(cli: &Cli) -> Result { render_analysis(cli.output, &document.source, &document.content, &analysis) .into_diagnostic()? ); - outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold; + outcome.threshold_reached |= analysis.evidence_score.score() >= failure_threshold; } } @@ -263,7 +265,7 @@ const fn fail_threshold(cli: &Cli, config: &Config) -> u8 { Some(FailLevel::Low) => config.thresholds.low, Some(FailLevel::Medium) => config.thresholds.medium, Some(FailLevel::High) => config.thresholds.high, - None => cli.fail_confidence, + None => cli.fail_score, } } @@ -317,13 +319,13 @@ fn print_config_paths(loader: &ConfigLoader) { println!("command_line_overrides = \"\""); } -/// Parse and validate an inclusive confidence threshold. -fn parse_confidence(value: &str) -> Result { +/// Parse and validate an inclusive evidence-score threshold. +fn parse_evidence_score(value: &str) -> Result { let score = value .parse::() - .map_err(|_| "confidence must be an integer from 0 through 100".to_owned())?; + .map_err(|_| "evidence score must be an integer from 0 through 100".to_owned())?; if score > 100 { - return Err("confidence must be an integer from 0 through 100".to_owned()); + return Err("evidence score must be an integer from 0 through 100".to_owned()); } Ok(score) } @@ -334,11 +336,10 @@ mod tests { #[test] fn parses_multiple_paths_and_threshold() { - let cli = - Cli::try_parse_from(["john-slop", "--fail-confidence", "42", "one.txt", "two.txt"]) - .expect("valid CLI"); + let cli = Cli::try_parse_from(["john-slop", "--fail-score", "42", "one.txt", "two.txt"]) + .expect("valid CLI"); - assert_eq!(cli.fail_confidence, 42); + assert_eq!(cli.fail_score, 42); assert_eq!(cli.fail_level, None); assert_eq!(cli.output, OutputFormat::Miette); assert_eq!(cli.config_name, None); @@ -351,7 +352,7 @@ mod tests { #[test] fn rejects_invalid_threshold() { - let error = Cli::try_parse_from(["john-slop", "--fail-confidence", "101", "file.txt"]) + let error = Cli::try_parse_from(["john-slop", "--fail-score", "101", "file.txt"]) .expect_err("threshold above 100 should fail"); assert_eq!(error.exit_code(), 2); @@ -426,7 +427,7 @@ mod tests { "john-slop", "--fail-level", "low", - "--fail-confidence", + "--fail-score", "20", "article.md", ]) @@ -447,10 +448,13 @@ mod tests { Cli::try_parse_from(["john-slop", "--fail-level", level, "article.md"]).unwrap(); assert_eq!(fail_threshold(&cli, &config), expected); } - let cli = - Cli::try_parse_from(["john-slop", "--fail-confidence", "17", "article.md"]).unwrap(); + let cli = Cli::try_parse_from(["john-slop", "--fail-score", "17", "article.md"]).unwrap(); assert_eq!(fail_threshold(&cli, &config), 17); + let compatibility = + Cli::try_parse_from(["john-slop", "--fail-confidence", "18", "article.md"]).unwrap(); + assert_eq!(fail_threshold(&compatibility, &config), 18); + assert_eq!(source_path(&SourceId::Stdin), None); assert_eq!( source_path(&SourceId::File("article.md".into())), diff --git a/src/bin/john-slop/output.rs b/src/bin/john-slop/output.rs index f60caaf..c9d74f3 100644 --- a/src/bin/john-slop/output.rs +++ b/src/bin/john-slop/output.rs @@ -51,7 +51,7 @@ pub(super) fn render_input_error( } OutputFormat::Concise => Ok(format!("{source}: input-error: {message}\n")), OutputFormat::Json => serde_json::to_string(&JsonInputError { - schema_version: 1, + schema_version: 2, kind: "input-error", source, message, @@ -102,13 +102,13 @@ fn append_summary(rendered: &mut String, source_id: &SourceId, analysis: &Analys let summary = analysis.summary(); let band = summary - .confidence_band - .map_or("none", john_slop::ConfidenceBand::as_str); + .evidence_band + .map_or("none", john_slop::EvidenceBand::as_str); writeln!( rendered, - "{}: confidence {} ({band}); {} finding(s): {} advice, {} warning, {} error", + "{}: low-care evidence score: {} ({band}); {} finding(s): {} advice, {} warning, {} error", source_id, - summary.confidence_score, + summary.evidence_score, summary.findings.total(), summary.findings.advice, summary.findings.warnings, @@ -136,13 +136,13 @@ fn render_concise(source_id: &SourceId, analysis: &Analysis) -> String { } let summary = analysis.summary(); let band = summary - .confidence_band - .map_or("none", john_slop::ConfidenceBand::as_str); + .evidence_band + .map_or("none", john_slop::EvidenceBand::as_str); writeln!( rendered, - "{}: summary: confidence={} band={band} advice={} warnings={} errors={} total={}", + "{}: summary: evidence_score={} band={band} advice={} warnings={} errors={} total={}", source_id, - summary.confidence_score, + summary.evidence_score, summary.findings.advice, summary.findings.warnings, summary.findings.errors, @@ -165,12 +165,12 @@ impl<'a> JsonAnalysis<'a> { fn new(source_id: &SourceId, analysis: &'a Analysis) -> Self { let summary = analysis.summary(); Self { - schema_version: 1, + schema_version: 2, kind: "analysis", source: source_id.to_string(), summary: JsonSummary { - confidence_score: summary.confidence_score, - confidence_band: summary.confidence_band, + evidence_score: summary.evidence_score, + evidence_band: summary.evidence_band, advice: summary.findings.advice, warnings: summary.findings.warnings, errors: summary.findings.errors, @@ -183,8 +183,8 @@ impl<'a> JsonAnalysis<'a> { #[derive(Serialize)] struct JsonSummary { - confidence_score: u8, - confidence_band: Option, + evidence_score: u8, + evidence_band: Option, advice: usize, warnings: usize, errors: usize, @@ -277,7 +277,7 @@ mod tests { assert!(output.contains("article.md")); assert!(output.contains("préface—suffix")); assert!(output.contains("diagnostic help:")); - assert!(output.contains("confidence")); + assert!(output.contains("low-care evidence score")); assert!(!output.contains('\u{1b}')); } @@ -314,7 +314,7 @@ mod tests { assert!(output.contains('╭')); assert!(output.contains('╰')); assert!(output.contains("help:")); - assert!(output.contains("confidence")); + assert!(output.contains("low-care evidence score")); assert!(output.contains('\u{1b}')); } @@ -330,7 +330,9 @@ mod tests { .unwrap(); let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap(); - assert_eq!(value["schema_version"], 1); + assert_eq!(value["schema_version"], 2); + assert_eq!(value["summary"]["evidence_score"], 17); + assert!(value["summary"].get("confidence_score").is_none()); assert_eq!(value["kind"], "analysis"); assert_eq!(value["source"], "article.md"); assert_eq!(value["findings"][0]["byte_span"]["start"], 8); @@ -343,7 +345,7 @@ mod tests { let output = render_input_error(OutputFormat::Json, "broken.txt", "invalid UTF-8").unwrap(); let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap(); - assert_eq!(value["schema_version"], 1); + assert_eq!(value["schema_version"], 2); assert_eq!(value["kind"], "input-error"); assert_eq!(value["source"], "broken.txt"); assert!(!output.contains('\u{1b}')); @@ -357,7 +359,7 @@ mod tests { render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap(); assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]")); - assert!(output.contains("article.md: summary: confidence=")); + assert!(output.contains("article.md: summary: evidence_score=")); assert!(output.contains("advice=1")); assert_eq!( render_input_error(OutputFormat::Concise, "bad.txt", "binary").unwrap(), @@ -385,7 +387,7 @@ mod tests { assert_eq!( output, - ": summary: confidence=0 band=none advice=0 warnings=0 errors=0 total=0\n" + ": summary: evidence_score=0 band=none advice=0 warnings=0 errors=0 total=0\n" ); } } diff --git a/src/config.rs b/src/config.rs index f750898..3affadf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -80,7 +80,7 @@ pub struct ConfigOverrides { /// Input-limit overrides. #[serde(skip_serializing_if = "InputOverrides::is_empty")] pub input: InputOverrides, - /// Confidence-threshold overrides. + /// Evidence-score threshold overrides. #[serde(skip_serializing_if = "ThresholdOverrides::is_empty")] pub thresholds: ThresholdOverrides, } @@ -119,16 +119,16 @@ impl InputOverrides { } } -/// Command-line overrides for confidence band boundaries. +/// Command-line overrides for evidence band boundaries. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] pub struct ThresholdOverrides { - /// Override the first low-confidence score. + /// Override the first low evidence score. #[serde(skip_serializing_if = "Option::is_none")] pub low: Option, - /// Override the first medium-confidence score. + /// Override the first medium evidence score. #[serde(skip_serializing_if = "Option::is_none")] pub medium: Option, - /// Override the first high-confidence score. + /// Override the first high evidence score. #[serde(skip_serializing_if = "Option::is_none")] pub high: Option, } @@ -395,8 +395,8 @@ pub enum ConfigValidationError { #[source] source: globset::Error, }, - /// Confidence thresholds are not strictly increasing. - #[error("confidence thresholds must satisfy low < medium < high <= 100")] + /// Evidence thresholds are not strictly increasing. + #[error("evidence thresholds must satisfy low < medium < high <= 100")] InvalidThresholds, /// An input resource limit is zero. #[error("input limit `{field}` must be greater than zero")] @@ -574,7 +574,7 @@ fn apply_input_layer(target: &mut InputLimits, layer: InputLayer) { } } -fn apply_threshold_layer(target: &mut ConfidenceThresholds, layer: ThresholdLayer) { +fn apply_threshold_layer(target: &mut EvidenceThresholds, layer: ThresholdLayer) { if let Some(value) = layer.low { target.low = value; } @@ -769,8 +769,8 @@ pub struct Config { pub replacements: Vec, /// Configured detection rules in deterministic declaration order. pub rules: Vec, - /// Named confidence-band boundaries. - pub thresholds: ConfidenceThresholds, + /// Named evidence-band boundaries. + pub thresholds: EvidenceThresholds, /// Resource and traversal limits applied while reading inputs. pub input: InputLimits, } @@ -856,15 +856,15 @@ pub enum ContentKind { SourceCode, } -/// Confidence boundaries used to name aggregate score bands. +/// Boundaries used to name aggregate evidence-score bands. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ConfidenceThresholds { - /// First score considered low confidence. +pub struct EvidenceThresholds { + /// First score considered low evidence. pub low: u8, - /// First score considered medium confidence. + /// First score considered medium evidence. pub medium: u8, - /// First score considered high confidence. + /// First score considered high evidence. pub high: u8, } @@ -925,7 +925,7 @@ impl Config { } } -fn validate_thresholds(thresholds: ConfidenceThresholds) -> Result<(), ConfigValidationError> { +fn validate_thresholds(thresholds: EvidenceThresholds) -> Result<(), ConfigValidationError> { if !(thresholds.low < thresholds.medium && thresholds.medium < thresholds.high && thresholds.high <= 100) @@ -1503,7 +1503,7 @@ weight = 9.5 assert_eq!(config.input.max_walk_depth, 66); assert_eq!( config.thresholds, - ConfidenceThresholds { + EvidenceThresholds { low: 10, medium: 40, high: 90 @@ -1575,7 +1575,7 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"], assert_eq!(config.input.max_walk_depth, 106); assert_eq!( config.thresholds, - ConfidenceThresholds { + EvidenceThresholds { low: 10, medium: 40, high: 90 diff --git a/src/detector.rs b/src/detector.rs index 5c0fd1f..e87cafd 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -9,7 +9,7 @@ use globset::{Glob, GlobSet, GlobSetBuilder}; use regex::Regex; use crate::{ - Analysis, Category, Confidence, ConfidenceBand, Config, ConfigValidationError, ContentKind, + Analysis, Category, Config, ConfigValidationError, ContentKind, EvidenceBand, EvidenceScore, Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement, }; @@ -167,14 +167,14 @@ impl Detector { .flat_map(|rule| rule.findings(&self.config, source)) .collect::>(); normalize_findings(&mut findings); - let confidence = aggregate_confidence(&self.config, &findings); - let confidence_band = (!findings.is_empty()) - .then(|| band_for_score(self.config.thresholds, confidence.score())); + let evidence_score = aggregate_evidence_score(&self.config, &findings); + let evidence_band = (!findings.is_empty()) + .then(|| band_for_score(self.config.thresholds, evidence_score.score())); Analysis { findings, - confidence, - confidence_band, + evidence_score, + evidence_band, } } } @@ -191,7 +191,7 @@ fn normalize_findings(findings: &mut Vec) { .dedup_by(|right, left| right.rule_id == left.rule_id && right.byte_span == left.byte_span); } -fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence { +fn aggregate_evidence_score(config: &Config, findings: &[Finding]) -> EvidenceScore { let mut family_rule_counts = BTreeMap::::new(); let mut evidence = 0.0; @@ -220,7 +220,7 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence { } else { 100.0 * evidence / (evidence + 50.0) }; - Confidence::from_score(normalized.round() as u8) + EvidenceScore::from_score(normalized.round() as u8) } const fn family_decay(category: Category) -> f64 { @@ -230,15 +230,15 @@ const fn family_decay(category: Category) -> f64 { } } -fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand { +fn band_for_score(thresholds: crate::EvidenceThresholds, score: u8) -> EvidenceBand { if score >= thresholds.high { - ConfidenceBand::High + EvidenceBand::High } else if score >= thresholds.medium { - ConfidenceBand::Medium + EvidenceBand::Medium } else if score >= thresholds.low { - ConfidenceBand::Low + EvidenceBand::Low } else { - ConfidenceBand::BelowThreshold + EvidenceBand::BelowThreshold } } @@ -523,11 +523,8 @@ mod tests { ); assert_eq!(analysis.findings[0].matched_text, "—"); assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-"); - assert_eq!(analysis.confidence.score(), 17); - assert_eq!( - analysis.confidence_band, - Some(ConfidenceBand::BelowThreshold) - ); + assert_eq!(analysis.evidence_score.score(), 17); + assert_eq!(analysis.evidence_band, Some(EvidenceBand::BelowThreshold)); } #[test] @@ -720,18 +717,18 @@ mod tests { } #[test] - fn confidence_caps_occurrences_and_applies_diminishing_returns() { + fn evidence_score_caps_occurrences_and_applies_diminishing_returns() { let detector = Detector::new(&Config::embedded()).unwrap(); let five = detector.analyze(&"—".repeat(5)); let twenty = detector.analyze(&"—".repeat(20)); - assert_eq!(five.confidence, twenty.confidence); - assert_eq!(five.confidence.score(), 28); + assert_eq!(five.evidence_score, twenty.evidence_score); + assert_eq!(five.evidence_score.score(), 28); assert_eq!(twenty.findings.len(), 20); } #[test] - fn confidence_damps_distinct_rules_in_the_same_family() { + fn evidence_score_damps_distinct_rules_in_the_same_family() { let source = "As an AI language model, I cannot browse the internet."; let correlated = Detector::new(&Config::embedded()).unwrap().analyze(source); let mut independent_config = Config::embedded(); @@ -743,7 +740,7 @@ mod tests { .category = Category::Style; let independent = Detector::new(&independent_config).unwrap().analyze(source); - assert!(correlated.confidence < independent.confidence); + assert!(correlated.evidence_score < independent.evidence_score); assert_eq!(correlated.findings.len(), independent.findings.len()); } @@ -763,46 +760,43 @@ mod tests { let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source); assert_eq!(analysis.findings.len(), 6); - assert_eq!(analysis.confidence.score(), 67); - assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium)); + assert_eq!(analysis.evidence_score.score(), 67); + assert_eq!(analysis.evidence_band, Some(EvidenceBand::Medium)); } #[test] fn configured_thresholds_define_named_band_boundaries() { - let thresholds = crate::ConfidenceThresholds { + let thresholds = crate::EvidenceThresholds { low: 20, medium: 50, high: 75, }; - assert_eq!( - band_for_score(thresholds, 19), - ConfidenceBand::BelowThreshold - ); - assert_eq!(band_for_score(thresholds, 20), ConfidenceBand::Low); - assert_eq!(band_for_score(thresholds, 50), ConfidenceBand::Medium); - assert_eq!(band_for_score(thresholds, 75), ConfidenceBand::High); + assert_eq!(band_for_score(thresholds, 19), EvidenceBand::BelowThreshold); + assert_eq!(band_for_score(thresholds, 20), EvidenceBand::Low); + assert_eq!(band_for_score(thresholds, 50), EvidenceBand::Medium); + assert_eq!(band_for_score(thresholds, 75), EvidenceBand::High); } #[test] - fn one_weak_repeated_signal_cannot_reach_high_confidence() { + fn one_weak_repeated_signal_cannot_reach_high_evidence() { let detector = Detector::new(&Config::embedded()).unwrap(); let source = "Moreover, first.\nAdditionally, second.\nFurthermore, third."; let analysis = detector.analyze(source); assert_eq!(analysis.findings.len(), 3); - assert_ne!(analysis.confidence_band, Some(ConfidenceBand::High)); + assert_ne!(analysis.evidence_band, Some(EvidenceBand::High)); } #[test] - fn empty_and_clean_text_have_zero_confidence_and_no_verdict() { + fn empty_and_clean_text_have_zero_evidence_and_no_verdict() { let detector = Detector::new(&Config::embedded()).unwrap(); for source in ["", "A specific, carefully sourced statement."] { let analysis = detector.analyze(source); assert!(analysis.findings.is_empty()); - assert_eq!(analysis.confidence.score(), 0); - assert_eq!(analysis.confidence_band, None); + assert_eq!(analysis.evidence_score.score(), 0); + assert_eq!(analysis.evidence_band, None); } } diff --git a/src/lib.rs b/src/lib.rs index 220caf8..acf3281 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,8 +15,8 @@ mod model; mod replacement; pub use config::{ - ConfidenceThresholds, Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, - ConfigNameError, ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, + Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError, + ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, EvidenceThresholds, HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule, RuleScope, ThresholdOverrides, }; @@ -25,9 +25,21 @@ pub use diagnostic::FindingDiagnostic; pub use error::{InputError, InputResult}; pub use input::{Document, Documents, read_stdin}; pub use model::{ - Analysis, AnalysisSummary, Category, Confidence, ConfidenceBand, Finding, FindingSeverity, + Analysis, AnalysisSummary, Category, EvidenceBand, EvidenceScore, Finding, FindingSeverity, RuleId, SeverityCounts, SourceId, TextEdit, }; + +/// Deprecated name for [`EvidenceScore`]. +#[deprecated(since = "0.1.0", note = "use EvidenceScore")] +pub type Confidence = EvidenceScore; + +/// Deprecated name for [`EvidenceBand`]. +#[deprecated(since = "0.1.0", note = "use EvidenceBand")] +pub type ConfidenceBand = EvidenceBand; + +/// Deprecated name for [`EvidenceThresholds`]. +#[deprecated(since = "0.1.0", note = "use EvidenceThresholds")] +pub type ConfidenceThresholds = EvidenceThresholds; pub use replacement::ReplacementError; /// Load embedded defaults plus optional platform user configuration. diff --git a/src/model.rs b/src/model.rs index e2adcc3..fae7be7 100644 --- a/src/model.rs +++ b/src/model.rs @@ -92,14 +92,14 @@ impl fmt::Display for FindingSeverity { } } -/// Confidence that configured low-care signals are present. +/// Bounded strength of the configured low-care evidence found in a document. /// /// This is deliberately not an AI-authorship probability. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] -pub struct Confidence(u8); +pub struct EvidenceScore(u8); -impl Confidence { - /// Construct a confidence value, clamping values above 100. +impl EvidenceScore { + /// Construct an evidence score, clamping values above 100. pub const fn from_score(score: u8) -> Self { Self(if score > 100 { 100 } else { score }) } @@ -113,7 +113,7 @@ impl Confidence { /// Named interpretation of a non-empty analysis score. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] -pub enum ConfidenceBand { +pub enum EvidenceBand { /// Evidence exists but remains below the configured low threshold. BelowThreshold, /// Score meets the configured low threshold. @@ -124,7 +124,7 @@ pub enum ConfidenceBand { High, } -impl ConfidenceBand { +impl EvidenceBand { /// Return the stable kebab-case presentation name. pub const fn as_str(self) -> &'static str { match self { @@ -136,7 +136,7 @@ impl ConfidenceBand { } } -impl fmt::Display for ConfidenceBand { +impl fmt::Display for EvidenceBand { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(self.as_str()) } @@ -207,18 +207,18 @@ pub struct Finding { pub struct Analysis { /// Findings sorted by source offset and rule identifier. pub findings: Vec, - /// Aggregate confidence that configured signals are present. - pub confidence: Confidence, + /// Aggregate strength of the configured evidence found. + pub evidence_score: EvidenceScore, /// Named score band, or no verdict when no evidence was found. - pub confidence_band: Option, + pub evidence_band: Option, } impl Analysis { /// Summarize score, named band, and severity counts without consuming findings. pub fn summary(&self) -> AnalysisSummary { AnalysisSummary { - confidence_score: self.confidence.score(), - confidence_band: self.confidence_band, + evidence_score: self.evidence_score.score(), + evidence_band: self.evidence_band, findings: SeverityCounts::from_findings(&self.findings), } } @@ -228,9 +228,9 @@ impl Analysis { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AnalysisSummary { /// Normalized evidence score in `0..=100`. - pub confidence_score: u8, + pub evidence_score: u8, /// Named band, or no verdict when the source has no evidence. - pub confidence_band: Option, + pub evidence_band: Option, /// Finding totals grouped by configured severity. pub findings: SeverityCounts, } @@ -270,10 +270,10 @@ mod tests { use super::*; #[test] - fn confidence_is_bounded() { - assert_eq!(Confidence::from_score(0).score(), 0); - assert_eq!(Confidence::from_score(73).score(), 73); - assert_eq!(Confidence::from_score(255).score(), 100); + fn evidence_score_is_bounded() { + assert_eq!(EvidenceScore::from_score(0).score(), 0); + assert_eq!(EvidenceScore::from_score(73).score(), 73); + assert_eq!(EvidenceScore::from_score(255).score(), 100); } #[test] @@ -330,15 +330,15 @@ mod tests { fix: None, }, ], - confidence: Confidence::from_score(32), - confidence_band: Some(ConfidenceBand::Low), + evidence_score: EvidenceScore::from_score(32), + evidence_band: Some(EvidenceBand::Low), }; let summary = analysis.summary(); assert_eq!(analysis.findings.len(), 2); - assert_eq!(summary.confidence_score, 32); - assert_eq!(summary.confidence_band, Some(ConfidenceBand::Low)); + assert_eq!(summary.evidence_score, 32); + assert_eq!(summary.evidence_band, Some(EvidenceBand::Low)); assert_eq!(summary.findings.advice, 1); assert_eq!(summary.findings.warnings, 1); assert_eq!(summary.findings.errors, 0); @@ -365,10 +365,10 @@ mod tests { assert_eq!(severity.to_string(), expected); } for (band, expected) in [ - (ConfidenceBand::BelowThreshold, "below-threshold"), - (ConfidenceBand::Low, "low"), - (ConfidenceBand::Medium, "medium"), - (ConfidenceBand::High, "high"), + (EvidenceBand::BelowThreshold, "below-threshold"), + (EvidenceBand::Low, "low"), + (EvidenceBand::Medium, "medium"), + (EvidenceBand::High, "high"), ] { assert_eq!(band.as_str(), expected); assert_eq!(band.to_string(), expected); diff --git a/tasks.md b/tasks.md index 3a66961..db051c4 100644 --- a/tasks.md +++ b/tasks.md @@ -293,6 +293,23 @@ Optional follow-up, intentionally not required for this milestone: sign-off feature: harden its change boundary beyond the catalog hash, or remove it if its maintenance cost exceeds its demonstrated value. +## Milestone 11 — Evidence-score terminology + +- [x] Rename the public domain model from ambiguous confidence terminology to + `EvidenceScore`, `EvidenceBand`, and `EvidenceThresholds`; retain deprecated + Rust type aliases and the legacy `--fail-confidence` CLI alias for migration. +- [ ] Publish the low-care evidence-score wording across terminal output, + examples, and documentation; rename `--fail-confidence` to `--fail-score` and + version the JSON field rename as schema version 2. +- [ ] Run the complete quality and coverage gates, refresh generated badges, and + record the compatibility boundary and unchanged scoring formula. + +Acceptance: every active user-facing contract describes the metric as a bounded +low-care evidence score rather than a probability or calibrated confidence; +legacy type and CLI names remain usable with clear migration guidance; JSON +consumers can identify the field rename by schema version; scoring results are +unchanged; and all quality/coverage gates pass. + ## Deferred by design - Statistical or model-based authorship classification. diff --git a/tests/cli.rs b/tests/cli.rs index a1675c0..2bf007c 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -48,9 +48,9 @@ fn help_is_successful_and_names_the_command() { } #[test] -fn invalid_confidence_is_a_usage_error() { +fn invalid_evidence_score_is_a_usage_error() { let output = detector() - .args(["--fail-confidence", "101", "file.txt"]) + .args(["--fail-score", "101", "file.txt"]) .output() .expect("run detector"); @@ -84,12 +84,12 @@ fn findings_control_the_threshold_exit_code() { fs::write(&suspicious, "suspicious—dash").expect("write suspicious fixture"); let clean_output = detector() - .args(["--fail-confidence", "10"]) + .args(["--fail-score", "10"]) .arg(&clean) .output() .expect("run clean detector"); let suspicious_output = detector() - .args(["--fail-confidence", "10"]) + .args(["--fail-score", "10"]) .arg(&suspicious) .output() .expect("run suspicious detector"); @@ -182,7 +182,9 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() { let records = json_records(&output); assert_eq!(records.len(), 1); let record = &records[0]; - assert_eq!(record["schema_version"], 1); + assert_eq!(record["schema_version"], 2); + assert!(record["summary"]["evidence_score"].is_number()); + assert!(record["summary"].get("confidence_score").is_none()); assert_eq!(record["kind"], "analysis"); assert_eq!(record["source"], ""); assert_eq!(record["summary"]["total"], 1); diff --git a/tests/evaluation.rs b/tests/evaluation.rs index 9838870..bd5b86f 100644 --- a/tests/evaluation.rs +++ b/tests/evaluation.rs @@ -51,9 +51,9 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() { }; let analysis = detector.analyze_with(&text, &context); assert_eq!( - analysis.confidence.score(), + analysis.evidence_score.score(), case.expected_score, - "confidence score for {}", + "evidence score for {}", case.path ); let actual = analysis diff --git a/tests/example_fixtures.rs b/tests/example_fixtures.rs index 95ee886..469d1a5 100644 --- a/tests/example_fixtures.rs +++ b/tests/example_fixtures.rs @@ -159,7 +159,7 @@ fn diagnostic_showcase_is_an_expected_threshold_failure() { assert!(stdout.contains("assistant-self-identification")); assert!(stdout.contains("prompt-response-scaffolding")); assert!(stdout.contains("typography-non-ascii-hyphen")); - assert!(stdout.contains("confidence 71 (medium)")); + assert!(stdout.contains("low-care evidence score: 71 (medium)")); assert!(stdout.contains("advice")); assert!(stdout.contains("warning")); assert!(stdout.contains("error")); diff --git a/tests/heuristics.rs b/tests/heuristics.rs index 5b32a65..73105b4 100644 --- a/tests/heuristics.rs +++ b/tests/heuristics.rs @@ -154,6 +154,6 @@ fn ordinary_multilingual_text_is_not_evidence() { let analysis = detector.analyze_with(text, &context(FixtureKind::Prose)); assert!(analysis.findings.is_empty()); - assert_eq!(analysis.confidence.score(), 0); - assert!(analysis.confidence_band.is_none()); + assert_eq!(analysis.evidence_score.score(), 0); + assert!(analysis.evidence_band.is_none()); } diff --git a/tests/snapshots/cli__help_is_successful_and_names_the_command.snap b/tests/snapshots/cli__help_is_successful_and_names_the_command.snap index 88a00bb..40067d4 100644 --- a/tests/snapshots/cli__help_is_successful_and_names_the_command.snap +++ b/tests/snapshots/cli__help_is_successful_and_names_the_command.snap @@ -39,8 +39,8 @@ Options: Skip source paths matching a glob; may be supplied more than once --output Select colored graphical Miette, plain human, concise, or versioned JSON output [default: miette] - --fail-confidence - Return exit code 1 when a document reaches this evidence-confidence score [default: 100] + --fail-score + Return exit code 1 when a document reaches this low-care evidence score [default: 100] [alias: --fail-confidence] --fail-level Return exit code 1 at the configured low, medium, or high threshold [possible values: low, medium, high] -h, --help diff --git a/tests/snapshots/cli__invalid_confidence_is_a_usage_error.snap b/tests/snapshots/cli__invalid_evidence_score_is_a_usage_error.snap similarity index 53% rename from tests/snapshots/cli__invalid_confidence_is_a_usage_error.snap rename to tests/snapshots/cli__invalid_evidence_score_is_a_usage_error.snap index 535b6cc..a912828 100644 --- a/tests/snapshots/cli__invalid_confidence_is_a_usage_error.snap +++ b/tests/snapshots/cli__invalid_evidence_score_is_a_usage_error.snap @@ -2,6 +2,6 @@ source: tests/cli.rs expression: "String::from_utf8(output.stderr).expect(\"UTF-8 stderr\")" --- -error: invalid value '101' for '--fail-confidence ': confidence must be an integer from 0 through 100 +error: invalid value '101' for '--fail-score ': evidence score must be an integer from 0 through 100 For more information, try '--help'.