refactor: rename confidence to evidence score

This commit is contained in:
Dandaedre 2026-09-29 21:06:05 +02:00
parent 7d365af5df
commit a4dd4e2c5f
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
16 changed files with 177 additions and 146 deletions

View File

@ -17,9 +17,9 @@ fn main() -> Result<(), Box<dyn Error>> {
let context = AnalysisContext::for_source(&document.source);
let analysis = detector.analyze_with(&document.content, &context);
println!(
"{}: confidence={}, findings={}",
"{}: evidence_score={}, findings={}",
document.source,
analysis.confidence.score(),
analysis.evidence_score.score(),
analysis.findings.len()
);
}

View File

@ -11,11 +11,11 @@ fn main() -> Result<(), Box<dyn Error>> {
let summary = analysis.summary();
println!(
"confidence={} band={} findings={}",
summary.confidence_score,
"evidence_score={} band={} findings={}",
summary.evidence_score,
summary
.confidence_band
.map_or("none", john_slop::ConfidenceBand::as_str),
.evidence_band
.map_or("none", john_slop::EvidenceBand::as_str),
summary.findings.total()
);

View File

@ -7,7 +7,7 @@ schema_version = 1
max_document_bytes = 4194304
[thresholds]
# Confidence bands affect labels and `--fail-level`; they are not probabilities.
# Evidence bands affect labels and `--fail-level`; they are not probabilities.
low = 20
medium = 50
high = 75
@ -25,6 +25,6 @@ enabled = false
[[rules]]
id = "source-placeholder-todo"
# Severity changes presentation; weight changes confidence contribution.
# Severity changes presentation; weight changes evidence-score contribution.
severity = "error"
weight = 15.0

View File

@ -83,17 +83,18 @@ struct Cli {
)]
output: OutputFormat,
/// Return exit code 1 when a document reaches this evidence-confidence score.
/// Return exit code 1 when a document reaches this low-care evidence score.
#[arg(
long,
visible_alias = "fail-confidence",
default_value_t = 100,
value_parser = parse_confidence,
value_parser = parse_evidence_score,
conflicts_with = "fail_level"
)]
fail_confidence: u8,
fail_score: u8,
/// Return exit code 1 at the configured low, medium, or high threshold.
#[arg(long, value_enum, conflicts_with = "fail_confidence")]
#[arg(long, value_enum, conflicts_with = "fail_score")]
fail_level: Option<FailLevel>,
/// Files, directories, or supported ZIP-based containers to inspect.
@ -125,7 +126,7 @@ enum ConfigCommand {
Paths,
}
/// Configured confidence band that should make analysis fail.
/// Configured evidence band that should make analysis fail.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum FailLevel {
Low,
@ -200,7 +201,8 @@ fn run(cli: &Cli) -> Result<RunOutcome> {
render_analysis(cli.output, &document.source, &document.content, &analysis)
.into_diagnostic()?
);
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
outcome.threshold_reached |=
analysis.evidence_score.score() >= failure_threshold;
}
}
Err(error) => {
@ -241,7 +243,7 @@ fn run(cli: &Cli) -> Result<RunOutcome> {
render_analysis(cli.output, &document.source, &document.content, &analysis)
.into_diagnostic()?
);
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
outcome.threshold_reached |= analysis.evidence_score.score() >= failure_threshold;
}
}
@ -263,7 +265,7 @@ const fn fail_threshold(cli: &Cli, config: &Config) -> u8 {
Some(FailLevel::Low) => config.thresholds.low,
Some(FailLevel::Medium) => config.thresholds.medium,
Some(FailLevel::High) => config.thresholds.high,
None => cli.fail_confidence,
None => cli.fail_score,
}
}
@ -317,13 +319,13 @@ fn print_config_paths(loader: &ConfigLoader) {
println!("command_line_overrides = \"<current invocation>\"");
}
/// Parse and validate an inclusive confidence threshold.
fn parse_confidence(value: &str) -> Result<u8, String> {
/// Parse and validate an inclusive evidence-score threshold.
fn parse_evidence_score(value: &str) -> Result<u8, String> {
let score = value
.parse::<u8>()
.map_err(|_| "confidence must be an integer from 0 through 100".to_owned())?;
.map_err(|_| "evidence score must be an integer from 0 through 100".to_owned())?;
if score > 100 {
return Err("confidence must be an integer from 0 through 100".to_owned());
return Err("evidence score must be an integer from 0 through 100".to_owned());
}
Ok(score)
}
@ -334,11 +336,10 @@ mod tests {
#[test]
fn parses_multiple_paths_and_threshold() {
let cli =
Cli::try_parse_from(["john-slop", "--fail-confidence", "42", "one.txt", "two.txt"])
.expect("valid CLI");
let cli = Cli::try_parse_from(["john-slop", "--fail-score", "42", "one.txt", "two.txt"])
.expect("valid CLI");
assert_eq!(cli.fail_confidence, 42);
assert_eq!(cli.fail_score, 42);
assert_eq!(cli.fail_level, None);
assert_eq!(cli.output, OutputFormat::Miette);
assert_eq!(cli.config_name, None);
@ -351,7 +352,7 @@ mod tests {
#[test]
fn rejects_invalid_threshold() {
let error = Cli::try_parse_from(["john-slop", "--fail-confidence", "101", "file.txt"])
let error = Cli::try_parse_from(["john-slop", "--fail-score", "101", "file.txt"])
.expect_err("threshold above 100 should fail");
assert_eq!(error.exit_code(), 2);
@ -426,7 +427,7 @@ mod tests {
"john-slop",
"--fail-level",
"low",
"--fail-confidence",
"--fail-score",
"20",
"article.md",
])
@ -447,10 +448,13 @@ mod tests {
Cli::try_parse_from(["john-slop", "--fail-level", level, "article.md"]).unwrap();
assert_eq!(fail_threshold(&cli, &config), expected);
}
let cli =
Cli::try_parse_from(["john-slop", "--fail-confidence", "17", "article.md"]).unwrap();
let cli = Cli::try_parse_from(["john-slop", "--fail-score", "17", "article.md"]).unwrap();
assert_eq!(fail_threshold(&cli, &config), 17);
let compatibility =
Cli::try_parse_from(["john-slop", "--fail-confidence", "18", "article.md"]).unwrap();
assert_eq!(fail_threshold(&compatibility, &config), 18);
assert_eq!(source_path(&SourceId::Stdin), None);
assert_eq!(
source_path(&SourceId::File("article.md".into())),

View File

@ -51,7 +51,7 @@ pub(super) fn render_input_error(
}
OutputFormat::Concise => Ok(format!("{source}: input-error: {message}\n")),
OutputFormat::Json => serde_json::to_string(&JsonInputError {
schema_version: 1,
schema_version: 2,
kind: "input-error",
source,
message,
@ -102,13 +102,13 @@ fn append_summary(rendered: &mut String, source_id: &SourceId, analysis: &Analys
let summary = analysis.summary();
let band = summary
.confidence_band
.map_or("none", john_slop::ConfidenceBand::as_str);
.evidence_band
.map_or("none", john_slop::EvidenceBand::as_str);
writeln!(
rendered,
"{}: confidence {} ({band}); {} finding(s): {} advice, {} warning, {} error",
"{}: low-care evidence score: {} ({band}); {} finding(s): {} advice, {} warning, {} error",
source_id,
summary.confidence_score,
summary.evidence_score,
summary.findings.total(),
summary.findings.advice,
summary.findings.warnings,
@ -136,13 +136,13 @@ fn render_concise(source_id: &SourceId, analysis: &Analysis) -> String {
}
let summary = analysis.summary();
let band = summary
.confidence_band
.map_or("none", john_slop::ConfidenceBand::as_str);
.evidence_band
.map_or("none", john_slop::EvidenceBand::as_str);
writeln!(
rendered,
"{}: summary: confidence={} band={band} advice={} warnings={} errors={} total={}",
"{}: summary: evidence_score={} band={band} advice={} warnings={} errors={} total={}",
source_id,
summary.confidence_score,
summary.evidence_score,
summary.findings.advice,
summary.findings.warnings,
summary.findings.errors,
@ -165,12 +165,12 @@ impl<'a> JsonAnalysis<'a> {
fn new(source_id: &SourceId, analysis: &'a Analysis) -> Self {
let summary = analysis.summary();
Self {
schema_version: 1,
schema_version: 2,
kind: "analysis",
source: source_id.to_string(),
summary: JsonSummary {
confidence_score: summary.confidence_score,
confidence_band: summary.confidence_band,
evidence_score: summary.evidence_score,
evidence_band: summary.evidence_band,
advice: summary.findings.advice,
warnings: summary.findings.warnings,
errors: summary.findings.errors,
@ -183,8 +183,8 @@ impl<'a> JsonAnalysis<'a> {
#[derive(Serialize)]
struct JsonSummary {
confidence_score: u8,
confidence_band: Option<john_slop::ConfidenceBand>,
evidence_score: u8,
evidence_band: Option<john_slop::EvidenceBand>,
advice: usize,
warnings: usize,
errors: usize,
@ -277,7 +277,7 @@ mod tests {
assert!(output.contains("article.md"));
assert!(output.contains("préface—suffix"));
assert!(output.contains("diagnostic help:"));
assert!(output.contains("confidence"));
assert!(output.contains("low-care evidence score"));
assert!(!output.contains('\u{1b}'));
}
@ -314,7 +314,7 @@ mod tests {
assert!(output.contains('╭'));
assert!(output.contains('╰'));
assert!(output.contains("help:"));
assert!(output.contains("confidence"));
assert!(output.contains("low-care evidence score"));
assert!(output.contains('\u{1b}'));
}
@ -330,7 +330,9 @@ mod tests {
.unwrap();
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
assert_eq!(value["schema_version"], 1);
assert_eq!(value["schema_version"], 2);
assert_eq!(value["summary"]["evidence_score"], 17);
assert!(value["summary"].get("confidence_score").is_none());
assert_eq!(value["kind"], "analysis");
assert_eq!(value["source"], "article.md");
assert_eq!(value["findings"][0]["byte_span"]["start"], 8);
@ -343,7 +345,7 @@ mod tests {
let output = render_input_error(OutputFormat::Json, "broken.txt", "invalid UTF-8").unwrap();
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
assert_eq!(value["schema_version"], 1);
assert_eq!(value["schema_version"], 2);
assert_eq!(value["kind"], "input-error");
assert_eq!(value["source"], "broken.txt");
assert!(!output.contains('\u{1b}'));
@ -357,7 +359,7 @@ mod tests {
render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap();
assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]"));
assert!(output.contains("article.md: summary: confidence="));
assert!(output.contains("article.md: summary: evidence_score="));
assert!(output.contains("advice=1"));
assert_eq!(
render_input_error(OutputFormat::Concise, "bad.txt", "binary").unwrap(),
@ -385,7 +387,7 @@ mod tests {
assert_eq!(
output,
"<stdin>: summary: confidence=0 band=none advice=0 warnings=0 errors=0 total=0\n"
"<stdin>: summary: evidence_score=0 band=none advice=0 warnings=0 errors=0 total=0\n"
);
}
}

View File

@ -80,7 +80,7 @@ pub struct ConfigOverrides {
/// Input-limit overrides.
#[serde(skip_serializing_if = "InputOverrides::is_empty")]
pub input: InputOverrides,
/// Confidence-threshold overrides.
/// Evidence-score threshold overrides.
#[serde(skip_serializing_if = "ThresholdOverrides::is_empty")]
pub thresholds: ThresholdOverrides,
}
@ -119,16 +119,16 @@ impl InputOverrides {
}
}
/// Command-line overrides for confidence band boundaries.
/// Command-line overrides for evidence band boundaries.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct ThresholdOverrides {
/// Override the first low-confidence score.
/// Override the first low evidence score.
#[serde(skip_serializing_if = "Option::is_none")]
pub low: Option<u8>,
/// Override the first medium-confidence score.
/// Override the first medium evidence score.
#[serde(skip_serializing_if = "Option::is_none")]
pub medium: Option<u8>,
/// Override the first high-confidence score.
/// Override the first high evidence score.
#[serde(skip_serializing_if = "Option::is_none")]
pub high: Option<u8>,
}
@ -395,8 +395,8 @@ pub enum ConfigValidationError {
#[source]
source: globset::Error,
},
/// Confidence thresholds are not strictly increasing.
#[error("confidence thresholds must satisfy low < medium < high <= 100")]
/// Evidence thresholds are not strictly increasing.
#[error("evidence thresholds must satisfy low < medium < high <= 100")]
InvalidThresholds,
/// An input resource limit is zero.
#[error("input limit `{field}` must be greater than zero")]
@ -574,7 +574,7 @@ fn apply_input_layer(target: &mut InputLimits, layer: InputLayer) {
}
}
fn apply_threshold_layer(target: &mut ConfidenceThresholds, layer: ThresholdLayer) {
fn apply_threshold_layer(target: &mut EvidenceThresholds, layer: ThresholdLayer) {
if let Some(value) = layer.low {
target.low = value;
}
@ -769,8 +769,8 @@ pub struct Config {
pub replacements: Vec<ReplacementRule>,
/// Configured detection rules in deterministic declaration order.
pub rules: Vec<HeuristicRule>,
/// Named confidence-band boundaries.
pub thresholds: ConfidenceThresholds,
/// Named evidence-band boundaries.
pub thresholds: EvidenceThresholds,
/// Resource and traversal limits applied while reading inputs.
pub input: InputLimits,
}
@ -856,15 +856,15 @@ pub enum ContentKind {
SourceCode,
}
/// Confidence boundaries used to name aggregate score bands.
/// Boundaries used to name aggregate evidence-score bands.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ConfidenceThresholds {
/// First score considered low confidence.
pub struct EvidenceThresholds {
/// First score considered low evidence.
pub low: u8,
/// First score considered medium confidence.
/// First score considered medium evidence.
pub medium: u8,
/// First score considered high confidence.
/// First score considered high evidence.
pub high: u8,
}
@ -925,7 +925,7 @@ impl Config {
}
}
fn validate_thresholds(thresholds: ConfidenceThresholds) -> Result<(), ConfigValidationError> {
fn validate_thresholds(thresholds: EvidenceThresholds) -> Result<(), ConfigValidationError> {
if !(thresholds.low < thresholds.medium
&& thresholds.medium < thresholds.high
&& thresholds.high <= 100)
@ -1503,7 +1503,7 @@ weight = 9.5
assert_eq!(config.input.max_walk_depth, 66);
assert_eq!(
config.thresholds,
ConfidenceThresholds {
EvidenceThresholds {
low: 10,
medium: 40,
high: 90
@ -1575,7 +1575,7 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
assert_eq!(config.input.max_walk_depth, 106);
assert_eq!(
config.thresholds,
ConfidenceThresholds {
EvidenceThresholds {
low: 10,
medium: 40,
high: 90

View File

@ -9,7 +9,7 @@ use globset::{Glob, GlobSet, GlobSetBuilder};
use regex::Regex;
use crate::{
Analysis, Category, Confidence, ConfidenceBand, Config, ConfigValidationError, ContentKind,
Analysis, Category, Config, ConfigValidationError, ContentKind, EvidenceBand, EvidenceScore,
Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement,
};
@ -167,14 +167,14 @@ impl Detector {
.flat_map(|rule| rule.findings(&self.config, source))
.collect::<Vec<_>>();
normalize_findings(&mut findings);
let confidence = aggregate_confidence(&self.config, &findings);
let confidence_band = (!findings.is_empty())
.then(|| band_for_score(self.config.thresholds, confidence.score()));
let evidence_score = aggregate_evidence_score(&self.config, &findings);
let evidence_band = (!findings.is_empty())
.then(|| band_for_score(self.config.thresholds, evidence_score.score()));
Analysis {
findings,
confidence,
confidence_band,
evidence_score,
evidence_band,
}
}
}
@ -191,7 +191,7 @@ fn normalize_findings(findings: &mut Vec<Finding>) {
.dedup_by(|right, left| right.rule_id == left.rule_id && right.byte_span == left.byte_span);
}
fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
fn aggregate_evidence_score(config: &Config, findings: &[Finding]) -> EvidenceScore {
let mut family_rule_counts = BTreeMap::<Category, usize>::new();
let mut evidence = 0.0;
@ -220,7 +220,7 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
} else {
100.0 * evidence / (evidence + 50.0)
};
Confidence::from_score(normalized.round() as u8)
EvidenceScore::from_score(normalized.round() as u8)
}
const fn family_decay(category: Category) -> f64 {
@ -230,15 +230,15 @@ const fn family_decay(category: Category) -> f64 {
}
}
fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand {
fn band_for_score(thresholds: crate::EvidenceThresholds, score: u8) -> EvidenceBand {
if score >= thresholds.high {
ConfidenceBand::High
EvidenceBand::High
} else if score >= thresholds.medium {
ConfidenceBand::Medium
EvidenceBand::Medium
} else if score >= thresholds.low {
ConfidenceBand::Low
EvidenceBand::Low
} else {
ConfidenceBand::BelowThreshold
EvidenceBand::BelowThreshold
}
}
@ -523,11 +523,8 @@ mod tests {
);
assert_eq!(analysis.findings[0].matched_text, "—");
assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-");
assert_eq!(analysis.confidence.score(), 17);
assert_eq!(
analysis.confidence_band,
Some(ConfidenceBand::BelowThreshold)
);
assert_eq!(analysis.evidence_score.score(), 17);
assert_eq!(analysis.evidence_band, Some(EvidenceBand::BelowThreshold));
}
#[test]
@ -720,18 +717,18 @@ mod tests {
}
#[test]
fn confidence_caps_occurrences_and_applies_diminishing_returns() {
fn evidence_score_caps_occurrences_and_applies_diminishing_returns() {
let detector = Detector::new(&Config::embedded()).unwrap();
let five = detector.analyze(&"—".repeat(5));
let twenty = detector.analyze(&"—".repeat(20));
assert_eq!(five.confidence, twenty.confidence);
assert_eq!(five.confidence.score(), 28);
assert_eq!(five.evidence_score, twenty.evidence_score);
assert_eq!(five.evidence_score.score(), 28);
assert_eq!(twenty.findings.len(), 20);
}
#[test]
fn confidence_damps_distinct_rules_in_the_same_family() {
fn evidence_score_damps_distinct_rules_in_the_same_family() {
let source = "As an AI language model, I cannot browse the internet.";
let correlated = Detector::new(&Config::embedded()).unwrap().analyze(source);
let mut independent_config = Config::embedded();
@ -743,7 +740,7 @@ mod tests {
.category = Category::Style;
let independent = Detector::new(&independent_config).unwrap().analyze(source);
assert!(correlated.confidence < independent.confidence);
assert!(correlated.evidence_score < independent.evidence_score);
assert_eq!(correlated.findings.len(), independent.findings.len());
}
@ -763,46 +760,43 @@ mod tests {
let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source);
assert_eq!(analysis.findings.len(), 6);
assert_eq!(analysis.confidence.score(), 67);
assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium));
assert_eq!(analysis.evidence_score.score(), 67);
assert_eq!(analysis.evidence_band, Some(EvidenceBand::Medium));
}
#[test]
fn configured_thresholds_define_named_band_boundaries() {
let thresholds = crate::ConfidenceThresholds {
let thresholds = crate::EvidenceThresholds {
low: 20,
medium: 50,
high: 75,
};
assert_eq!(
band_for_score(thresholds, 19),
ConfidenceBand::BelowThreshold
);
assert_eq!(band_for_score(thresholds, 20), ConfidenceBand::Low);
assert_eq!(band_for_score(thresholds, 50), ConfidenceBand::Medium);
assert_eq!(band_for_score(thresholds, 75), ConfidenceBand::High);
assert_eq!(band_for_score(thresholds, 19), EvidenceBand::BelowThreshold);
assert_eq!(band_for_score(thresholds, 20), EvidenceBand::Low);
assert_eq!(band_for_score(thresholds, 50), EvidenceBand::Medium);
assert_eq!(band_for_score(thresholds, 75), EvidenceBand::High);
}
#[test]
fn one_weak_repeated_signal_cannot_reach_high_confidence() {
fn one_weak_repeated_signal_cannot_reach_high_evidence() {
let detector = Detector::new(&Config::embedded()).unwrap();
let source = "Moreover, first.\nAdditionally, second.\nFurthermore, third.";
let analysis = detector.analyze(source);
assert_eq!(analysis.findings.len(), 3);
assert_ne!(analysis.confidence_band, Some(ConfidenceBand::High));
assert_ne!(analysis.evidence_band, Some(EvidenceBand::High));
}
#[test]
fn empty_and_clean_text_have_zero_confidence_and_no_verdict() {
fn empty_and_clean_text_have_zero_evidence_and_no_verdict() {
let detector = Detector::new(&Config::embedded()).unwrap();
for source in ["", "A specific, carefully sourced statement."] {
let analysis = detector.analyze(source);
assert!(analysis.findings.is_empty());
assert_eq!(analysis.confidence.score(), 0);
assert_eq!(analysis.confidence_band, None);
assert_eq!(analysis.evidence_score.score(), 0);
assert_eq!(analysis.evidence_band, None);
}
}

View File

@ -15,8 +15,8 @@ mod model;
mod replacement;
pub use config::{
ConfidenceThresholds, Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName,
ConfigNameError, ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind,
Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError,
ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, EvidenceThresholds,
HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule, RuleScope,
ThresholdOverrides,
};
@ -25,9 +25,21 @@ pub use diagnostic::FindingDiagnostic;
pub use error::{InputError, InputResult};
pub use input::{Document, Documents, read_stdin};
pub use model::{
Analysis, AnalysisSummary, Category, Confidence, ConfidenceBand, Finding, FindingSeverity,
Analysis, AnalysisSummary, Category, EvidenceBand, EvidenceScore, Finding, FindingSeverity,
RuleId, SeverityCounts, SourceId, TextEdit,
};
/// Deprecated name for [`EvidenceScore`].
#[deprecated(since = "0.1.0", note = "use EvidenceScore")]
pub type Confidence = EvidenceScore;
/// Deprecated name for [`EvidenceBand`].
#[deprecated(since = "0.1.0", note = "use EvidenceBand")]
pub type ConfidenceBand = EvidenceBand;
/// Deprecated name for [`EvidenceThresholds`].
#[deprecated(since = "0.1.0", note = "use EvidenceThresholds")]
pub type ConfidenceThresholds = EvidenceThresholds;
pub use replacement::ReplacementError;
/// Load embedded defaults plus optional platform user configuration.

View File

@ -92,14 +92,14 @@ impl fmt::Display for FindingSeverity {
}
}
/// Confidence that configured low-care signals are present.
/// Bounded strength of the configured low-care evidence found in a document.
///
/// This is deliberately not an AI-authorship probability.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)]
pub struct Confidence(u8);
pub struct EvidenceScore(u8);
impl Confidence {
/// Construct a confidence value, clamping values above 100.
impl EvidenceScore {
/// Construct an evidence score, clamping values above 100.
pub const fn from_score(score: u8) -> Self {
Self(if score > 100 { 100 } else { score })
}
@ -113,7 +113,7 @@ impl Confidence {
/// Named interpretation of a non-empty analysis score.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ConfidenceBand {
pub enum EvidenceBand {
/// Evidence exists but remains below the configured low threshold.
BelowThreshold,
/// Score meets the configured low threshold.
@ -124,7 +124,7 @@ pub enum ConfidenceBand {
High,
}
impl ConfidenceBand {
impl EvidenceBand {
/// Return the stable kebab-case presentation name.
pub const fn as_str(self) -> &'static str {
match self {
@ -136,7 +136,7 @@ impl ConfidenceBand {
}
}
impl fmt::Display for ConfidenceBand {
impl fmt::Display for EvidenceBand {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
@ -207,18 +207,18 @@ pub struct Finding {
pub struct Analysis {
/// Findings sorted by source offset and rule identifier.
pub findings: Vec<Finding>,
/// Aggregate confidence that configured signals are present.
pub confidence: Confidence,
/// Aggregate strength of the configured evidence found.
pub evidence_score: EvidenceScore,
/// Named score band, or no verdict when no evidence was found.
pub confidence_band: Option<ConfidenceBand>,
pub evidence_band: Option<EvidenceBand>,
}
impl Analysis {
/// Summarize score, named band, and severity counts without consuming findings.
pub fn summary(&self) -> AnalysisSummary {
AnalysisSummary {
confidence_score: self.confidence.score(),
confidence_band: self.confidence_band,
evidence_score: self.evidence_score.score(),
evidence_band: self.evidence_band,
findings: SeverityCounts::from_findings(&self.findings),
}
}
@ -228,9 +228,9 @@ impl Analysis {
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct AnalysisSummary {
/// Normalized evidence score in `0..=100`.
pub confidence_score: u8,
pub evidence_score: u8,
/// Named band, or no verdict when the source has no evidence.
pub confidence_band: Option<ConfidenceBand>,
pub evidence_band: Option<EvidenceBand>,
/// Finding totals grouped by configured severity.
pub findings: SeverityCounts,
}
@ -270,10 +270,10 @@ mod tests {
use super::*;
#[test]
fn confidence_is_bounded() {
assert_eq!(Confidence::from_score(0).score(), 0);
assert_eq!(Confidence::from_score(73).score(), 73);
assert_eq!(Confidence::from_score(255).score(), 100);
fn evidence_score_is_bounded() {
assert_eq!(EvidenceScore::from_score(0).score(), 0);
assert_eq!(EvidenceScore::from_score(73).score(), 73);
assert_eq!(EvidenceScore::from_score(255).score(), 100);
}
#[test]
@ -330,15 +330,15 @@ mod tests {
fix: None,
},
],
confidence: Confidence::from_score(32),
confidence_band: Some(ConfidenceBand::Low),
evidence_score: EvidenceScore::from_score(32),
evidence_band: Some(EvidenceBand::Low),
};
let summary = analysis.summary();
assert_eq!(analysis.findings.len(), 2);
assert_eq!(summary.confidence_score, 32);
assert_eq!(summary.confidence_band, Some(ConfidenceBand::Low));
assert_eq!(summary.evidence_score, 32);
assert_eq!(summary.evidence_band, Some(EvidenceBand::Low));
assert_eq!(summary.findings.advice, 1);
assert_eq!(summary.findings.warnings, 1);
assert_eq!(summary.findings.errors, 0);
@ -365,10 +365,10 @@ mod tests {
assert_eq!(severity.to_string(), expected);
}
for (band, expected) in [
(ConfidenceBand::BelowThreshold, "below-threshold"),
(ConfidenceBand::Low, "low"),
(ConfidenceBand::Medium, "medium"),
(ConfidenceBand::High, "high"),
(EvidenceBand::BelowThreshold, "below-threshold"),
(EvidenceBand::Low, "low"),
(EvidenceBand::Medium, "medium"),
(EvidenceBand::High, "high"),
] {
assert_eq!(band.as_str(), expected);
assert_eq!(band.to_string(), expected);

View File

@ -293,6 +293,23 @@ Optional follow-up, intentionally not required for this milestone:
sign-off feature: harden its change boundary beyond the catalog hash, or
remove it if its maintenance cost exceeds its demonstrated value.
## Milestone 11 — Evidence-score terminology
- [x] Rename the public domain model from ambiguous confidence terminology to
`EvidenceScore`, `EvidenceBand`, and `EvidenceThresholds`; retain deprecated
Rust type aliases and the legacy `--fail-confidence` CLI alias for migration.
- [ ] Publish the low-care evidence-score wording across terminal output,
examples, and documentation; rename `--fail-confidence` to `--fail-score` and
version the JSON field rename as schema version 2.
- [ ] Run the complete quality and coverage gates, refresh generated badges, and
record the compatibility boundary and unchanged scoring formula.
Acceptance: every active user-facing contract describes the metric as a bounded
low-care evidence score rather than a probability or calibrated confidence;
legacy type and CLI names remain usable with clear migration guidance; JSON
consumers can identify the field rename by schema version; scoring results are
unchanged; and all quality/coverage gates pass.
## Deferred by design
- Statistical or model-based authorship classification.

View File

@ -48,9 +48,9 @@ fn help_is_successful_and_names_the_command() {
}
#[test]
fn invalid_confidence_is_a_usage_error() {
fn invalid_evidence_score_is_a_usage_error() {
let output = detector()
.args(["--fail-confidence", "101", "file.txt"])
.args(["--fail-score", "101", "file.txt"])
.output()
.expect("run detector");
@ -84,12 +84,12 @@ fn findings_control_the_threshold_exit_code() {
fs::write(&suspicious, "suspicious—dash").expect("write suspicious fixture");
let clean_output = detector()
.args(["--fail-confidence", "10"])
.args(["--fail-score", "10"])
.arg(&clean)
.output()
.expect("run clean detector");
let suspicious_output = detector()
.args(["--fail-confidence", "10"])
.args(["--fail-score", "10"])
.arg(&suspicious)
.output()
.expect("run suspicious detector");
@ -182,7 +182,9 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
let records = json_records(&output);
assert_eq!(records.len(), 1);
let record = &records[0];
assert_eq!(record["schema_version"], 1);
assert_eq!(record["schema_version"], 2);
assert!(record["summary"]["evidence_score"].is_number());
assert!(record["summary"].get("confidence_score").is_none());
assert_eq!(record["kind"], "analysis");
assert_eq!(record["source"], "<stdin>");
assert_eq!(record["summary"]["total"], 1);

View File

@ -51,9 +51,9 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
};
let analysis = detector.analyze_with(&text, &context);
assert_eq!(
analysis.confidence.score(),
analysis.evidence_score.score(),
case.expected_score,
"confidence score for {}",
"evidence score for {}",
case.path
);
let actual = analysis

View File

@ -159,7 +159,7 @@ fn diagnostic_showcase_is_an_expected_threshold_failure() {
assert!(stdout.contains("assistant-self-identification"));
assert!(stdout.contains("prompt-response-scaffolding"));
assert!(stdout.contains("typography-non-ascii-hyphen"));
assert!(stdout.contains("confidence 71 (medium)"));
assert!(stdout.contains("low-care evidence score: 71 (medium)"));
assert!(stdout.contains("advice"));
assert!(stdout.contains("warning"));
assert!(stdout.contains("error"));

View File

@ -154,6 +154,6 @@ fn ordinary_multilingual_text_is_not_evidence() {
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
assert!(analysis.findings.is_empty());
assert_eq!(analysis.confidence.score(), 0);
assert!(analysis.confidence_band.is_none());
assert_eq!(analysis.evidence_score.score(), 0);
assert!(analysis.evidence_band.is_none());
}

View File

@ -39,8 +39,8 @@ Options:
Skip source paths matching a glob; may be supplied more than once
--output <human|miette|concise|json>
Select colored graphical Miette, plain human, concise, or versioned JSON output [default: miette]
--fail-confidence <FAIL_CONFIDENCE>
Return exit code 1 when a document reaches this evidence-confidence score [default: 100]
--fail-score <FAIL_SCORE>
Return exit code 1 when a document reaches this low-care evidence score [default: 100] [alias: --fail-confidence]
--fail-level <FAIL_LEVEL>
Return exit code 1 at the configured low, medium, or high threshold [possible values: low, medium, high]
-h, --help

View File

@ -2,6 +2,6 @@
source: tests/cli.rs
expression: "String::from_utf8(output.stderr).expect(\"UTF-8 stderr\")"
---
error: invalid value '101' for '--fail-confidence <FAIL_CONFIDENCE>': confidence must be an integer from 0 through 100
error: invalid value '101' for '--fail-score <FAIL_SCORE>': evidence score must be an integer from 0 through 100
For more information, try '--help'.