refactor: rename confidence to evidence score
This commit is contained in:
parent
7d365af5df
commit
a4dd4e2c5f
|
|
@ -17,9 +17,9 @@ fn main() -> Result<(), Box<dyn Error>> {
|
|||
let context = AnalysisContext::for_source(&document.source);
|
||||
let analysis = detector.analyze_with(&document.content, &context);
|
||||
println!(
|
||||
"{}: confidence={}, findings={}",
|
||||
"{}: evidence_score={}, findings={}",
|
||||
document.source,
|
||||
analysis.confidence.score(),
|
||||
analysis.evidence_score.score(),
|
||||
analysis.findings.len()
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ fn main() -> Result<(), Box<dyn Error>> {
|
|||
let summary = analysis.summary();
|
||||
|
||||
println!(
|
||||
"confidence={} band={} findings={}",
|
||||
summary.confidence_score,
|
||||
"evidence_score={} band={} findings={}",
|
||||
summary.evidence_score,
|
||||
summary
|
||||
.confidence_band
|
||||
.map_or("none", john_slop::ConfidenceBand::as_str),
|
||||
.evidence_band
|
||||
.map_or("none", john_slop::EvidenceBand::as_str),
|
||||
summary.findings.total()
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ schema_version = 1
|
|||
max_document_bytes = 4194304
|
||||
|
||||
[thresholds]
|
||||
# Confidence bands affect labels and `--fail-level`; they are not probabilities.
|
||||
# Evidence bands affect labels and `--fail-level`; they are not probabilities.
|
||||
low = 20
|
||||
medium = 50
|
||||
high = 75
|
||||
|
|
@ -25,6 +25,6 @@ enabled = false
|
|||
|
||||
[[rules]]
|
||||
id = "source-placeholder-todo"
|
||||
# Severity changes presentation; weight changes confidence contribution.
|
||||
# Severity changes presentation; weight changes evidence-score contribution.
|
||||
severity = "error"
|
||||
weight = 15.0
|
||||
|
|
|
|||
|
|
@ -83,17 +83,18 @@ struct Cli {
|
|||
)]
|
||||
output: OutputFormat,
|
||||
|
||||
/// Return exit code 1 when a document reaches this evidence-confidence score.
|
||||
/// Return exit code 1 when a document reaches this low-care evidence score.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "fail-confidence",
|
||||
default_value_t = 100,
|
||||
value_parser = parse_confidence,
|
||||
value_parser = parse_evidence_score,
|
||||
conflicts_with = "fail_level"
|
||||
)]
|
||||
fail_confidence: u8,
|
||||
fail_score: u8,
|
||||
|
||||
/// Return exit code 1 at the configured low, medium, or high threshold.
|
||||
#[arg(long, value_enum, conflicts_with = "fail_confidence")]
|
||||
#[arg(long, value_enum, conflicts_with = "fail_score")]
|
||||
fail_level: Option<FailLevel>,
|
||||
|
||||
/// Files, directories, or supported ZIP-based containers to inspect.
|
||||
|
|
@ -125,7 +126,7 @@ enum ConfigCommand {
|
|||
Paths,
|
||||
}
|
||||
|
||||
/// Configured confidence band that should make analysis fail.
|
||||
/// Configured evidence band that should make analysis fail.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
enum FailLevel {
|
||||
Low,
|
||||
|
|
@ -200,7 +201,8 @@ fn run(cli: &Cli) -> Result<RunOutcome> {
|
|||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
outcome.threshold_reached |=
|
||||
analysis.evidence_score.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
|
|
@ -241,7 +243,7 @@ fn run(cli: &Cli) -> Result<RunOutcome> {
|
|||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
outcome.threshold_reached |= analysis.evidence_score.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -263,7 +265,7 @@ const fn fail_threshold(cli: &Cli, config: &Config) -> u8 {
|
|||
Some(FailLevel::Low) => config.thresholds.low,
|
||||
Some(FailLevel::Medium) => config.thresholds.medium,
|
||||
Some(FailLevel::High) => config.thresholds.high,
|
||||
None => cli.fail_confidence,
|
||||
None => cli.fail_score,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -317,13 +319,13 @@ fn print_config_paths(loader: &ConfigLoader) {
|
|||
println!("command_line_overrides = \"<current invocation>\"");
|
||||
}
|
||||
|
||||
/// Parse and validate an inclusive confidence threshold.
|
||||
fn parse_confidence(value: &str) -> Result<u8, String> {
|
||||
/// Parse and validate an inclusive evidence-score threshold.
|
||||
fn parse_evidence_score(value: &str) -> Result<u8, String> {
|
||||
let score = value
|
||||
.parse::<u8>()
|
||||
.map_err(|_| "confidence must be an integer from 0 through 100".to_owned())?;
|
||||
.map_err(|_| "evidence score must be an integer from 0 through 100".to_owned())?;
|
||||
if score > 100 {
|
||||
return Err("confidence must be an integer from 0 through 100".to_owned());
|
||||
return Err("evidence score must be an integer from 0 through 100".to_owned());
|
||||
}
|
||||
Ok(score)
|
||||
}
|
||||
|
|
@ -334,11 +336,10 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn parses_multiple_paths_and_threshold() {
|
||||
let cli =
|
||||
Cli::try_parse_from(["john-slop", "--fail-confidence", "42", "one.txt", "two.txt"])
|
||||
.expect("valid CLI");
|
||||
let cli = Cli::try_parse_from(["john-slop", "--fail-score", "42", "one.txt", "two.txt"])
|
||||
.expect("valid CLI");
|
||||
|
||||
assert_eq!(cli.fail_confidence, 42);
|
||||
assert_eq!(cli.fail_score, 42);
|
||||
assert_eq!(cli.fail_level, None);
|
||||
assert_eq!(cli.output, OutputFormat::Miette);
|
||||
assert_eq!(cli.config_name, None);
|
||||
|
|
@ -351,7 +352,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn rejects_invalid_threshold() {
|
||||
let error = Cli::try_parse_from(["john-slop", "--fail-confidence", "101", "file.txt"])
|
||||
let error = Cli::try_parse_from(["john-slop", "--fail-score", "101", "file.txt"])
|
||||
.expect_err("threshold above 100 should fail");
|
||||
|
||||
assert_eq!(error.exit_code(), 2);
|
||||
|
|
@ -426,7 +427,7 @@ mod tests {
|
|||
"john-slop",
|
||||
"--fail-level",
|
||||
"low",
|
||||
"--fail-confidence",
|
||||
"--fail-score",
|
||||
"20",
|
||||
"article.md",
|
||||
])
|
||||
|
|
@ -447,10 +448,13 @@ mod tests {
|
|||
Cli::try_parse_from(["john-slop", "--fail-level", level, "article.md"]).unwrap();
|
||||
assert_eq!(fail_threshold(&cli, &config), expected);
|
||||
}
|
||||
let cli =
|
||||
Cli::try_parse_from(["john-slop", "--fail-confidence", "17", "article.md"]).unwrap();
|
||||
let cli = Cli::try_parse_from(["john-slop", "--fail-score", "17", "article.md"]).unwrap();
|
||||
assert_eq!(fail_threshold(&cli, &config), 17);
|
||||
|
||||
let compatibility =
|
||||
Cli::try_parse_from(["john-slop", "--fail-confidence", "18", "article.md"]).unwrap();
|
||||
assert_eq!(fail_threshold(&compatibility, &config), 18);
|
||||
|
||||
assert_eq!(source_path(&SourceId::Stdin), None);
|
||||
assert_eq!(
|
||||
source_path(&SourceId::File("article.md".into())),
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ pub(super) fn render_input_error(
|
|||
}
|
||||
OutputFormat::Concise => Ok(format!("{source}: input-error: {message}\n")),
|
||||
OutputFormat::Json => serde_json::to_string(&JsonInputError {
|
||||
schema_version: 1,
|
||||
schema_version: 2,
|
||||
kind: "input-error",
|
||||
source,
|
||||
message,
|
||||
|
|
@ -102,13 +102,13 @@ fn append_summary(rendered: &mut String, source_id: &SourceId, analysis: &Analys
|
|||
|
||||
let summary = analysis.summary();
|
||||
let band = summary
|
||||
.confidence_band
|
||||
.map_or("none", john_slop::ConfidenceBand::as_str);
|
||||
.evidence_band
|
||||
.map_or("none", john_slop::EvidenceBand::as_str);
|
||||
writeln!(
|
||||
rendered,
|
||||
"{}: confidence {} ({band}); {} finding(s): {} advice, {} warning, {} error",
|
||||
"{}: low-care evidence score: {} ({band}); {} finding(s): {} advice, {} warning, {} error",
|
||||
source_id,
|
||||
summary.confidence_score,
|
||||
summary.evidence_score,
|
||||
summary.findings.total(),
|
||||
summary.findings.advice,
|
||||
summary.findings.warnings,
|
||||
|
|
@ -136,13 +136,13 @@ fn render_concise(source_id: &SourceId, analysis: &Analysis) -> String {
|
|||
}
|
||||
let summary = analysis.summary();
|
||||
let band = summary
|
||||
.confidence_band
|
||||
.map_or("none", john_slop::ConfidenceBand::as_str);
|
||||
.evidence_band
|
||||
.map_or("none", john_slop::EvidenceBand::as_str);
|
||||
writeln!(
|
||||
rendered,
|
||||
"{}: summary: confidence={} band={band} advice={} warnings={} errors={} total={}",
|
||||
"{}: summary: evidence_score={} band={band} advice={} warnings={} errors={} total={}",
|
||||
source_id,
|
||||
summary.confidence_score,
|
||||
summary.evidence_score,
|
||||
summary.findings.advice,
|
||||
summary.findings.warnings,
|
||||
summary.findings.errors,
|
||||
|
|
@ -165,12 +165,12 @@ impl<'a> JsonAnalysis<'a> {
|
|||
fn new(source_id: &SourceId, analysis: &'a Analysis) -> Self {
|
||||
let summary = analysis.summary();
|
||||
Self {
|
||||
schema_version: 1,
|
||||
schema_version: 2,
|
||||
kind: "analysis",
|
||||
source: source_id.to_string(),
|
||||
summary: JsonSummary {
|
||||
confidence_score: summary.confidence_score,
|
||||
confidence_band: summary.confidence_band,
|
||||
evidence_score: summary.evidence_score,
|
||||
evidence_band: summary.evidence_band,
|
||||
advice: summary.findings.advice,
|
||||
warnings: summary.findings.warnings,
|
||||
errors: summary.findings.errors,
|
||||
|
|
@ -183,8 +183,8 @@ impl<'a> JsonAnalysis<'a> {
|
|||
|
||||
#[derive(Serialize)]
|
||||
struct JsonSummary {
|
||||
confidence_score: u8,
|
||||
confidence_band: Option<john_slop::ConfidenceBand>,
|
||||
evidence_score: u8,
|
||||
evidence_band: Option<john_slop::EvidenceBand>,
|
||||
advice: usize,
|
||||
warnings: usize,
|
||||
errors: usize,
|
||||
|
|
@ -277,7 +277,7 @@ mod tests {
|
|||
assert!(output.contains("article.md"));
|
||||
assert!(output.contains("préface—suffix"));
|
||||
assert!(output.contains("diagnostic help:"));
|
||||
assert!(output.contains("confidence"));
|
||||
assert!(output.contains("low-care evidence score"));
|
||||
assert!(!output.contains('\u{1b}'));
|
||||
}
|
||||
|
||||
|
|
@ -314,7 +314,7 @@ mod tests {
|
|||
assert!(output.contains('╭'));
|
||||
assert!(output.contains('╰'));
|
||||
assert!(output.contains("help:"));
|
||||
assert!(output.contains("confidence"));
|
||||
assert!(output.contains("low-care evidence score"));
|
||||
assert!(output.contains('\u{1b}'));
|
||||
}
|
||||
|
||||
|
|
@ -330,7 +330,9 @@ mod tests {
|
|||
.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
|
||||
|
||||
assert_eq!(value["schema_version"], 1);
|
||||
assert_eq!(value["schema_version"], 2);
|
||||
assert_eq!(value["summary"]["evidence_score"], 17);
|
||||
assert!(value["summary"].get("confidence_score").is_none());
|
||||
assert_eq!(value["kind"], "analysis");
|
||||
assert_eq!(value["source"], "article.md");
|
||||
assert_eq!(value["findings"][0]["byte_span"]["start"], 8);
|
||||
|
|
@ -343,7 +345,7 @@ mod tests {
|
|||
let output = render_input_error(OutputFormat::Json, "broken.txt", "invalid UTF-8").unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
|
||||
|
||||
assert_eq!(value["schema_version"], 1);
|
||||
assert_eq!(value["schema_version"], 2);
|
||||
assert_eq!(value["kind"], "input-error");
|
||||
assert_eq!(value["source"], "broken.txt");
|
||||
assert!(!output.contains('\u{1b}'));
|
||||
|
|
@ -357,7 +359,7 @@ mod tests {
|
|||
render_analysis(OutputFormat::Concise, &source_id, &source, &analysis).unwrap();
|
||||
|
||||
assert!(output.contains("article.md:8:11: advice[typography-non-ascii-hyphen]"));
|
||||
assert!(output.contains("article.md: summary: confidence="));
|
||||
assert!(output.contains("article.md: summary: evidence_score="));
|
||||
assert!(output.contains("advice=1"));
|
||||
assert_eq!(
|
||||
render_input_error(OutputFormat::Concise, "bad.txt", "binary").unwrap(),
|
||||
|
|
@ -385,7 +387,7 @@ mod tests {
|
|||
|
||||
assert_eq!(
|
||||
output,
|
||||
"<stdin>: summary: confidence=0 band=none advice=0 warnings=0 errors=0 total=0\n"
|
||||
"<stdin>: summary: evidence_score=0 band=none advice=0 warnings=0 errors=0 total=0\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ pub struct ConfigOverrides {
|
|||
/// Input-limit overrides.
|
||||
#[serde(skip_serializing_if = "InputOverrides::is_empty")]
|
||||
pub input: InputOverrides,
|
||||
/// Confidence-threshold overrides.
|
||||
/// Evidence-score threshold overrides.
|
||||
#[serde(skip_serializing_if = "ThresholdOverrides::is_empty")]
|
||||
pub thresholds: ThresholdOverrides,
|
||||
}
|
||||
|
|
@ -119,16 +119,16 @@ impl InputOverrides {
|
|||
}
|
||||
}
|
||||
|
||||
/// Command-line overrides for confidence band boundaries.
|
||||
/// Command-line overrides for evidence band boundaries.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct ThresholdOverrides {
|
||||
/// Override the first low-confidence score.
|
||||
/// Override the first low evidence score.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub low: Option<u8>,
|
||||
/// Override the first medium-confidence score.
|
||||
/// Override the first medium evidence score.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub medium: Option<u8>,
|
||||
/// Override the first high-confidence score.
|
||||
/// Override the first high evidence score.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub high: Option<u8>,
|
||||
}
|
||||
|
|
@ -395,8 +395,8 @@ pub enum ConfigValidationError {
|
|||
#[source]
|
||||
source: globset::Error,
|
||||
},
|
||||
/// Confidence thresholds are not strictly increasing.
|
||||
#[error("confidence thresholds must satisfy low < medium < high <= 100")]
|
||||
/// Evidence thresholds are not strictly increasing.
|
||||
#[error("evidence thresholds must satisfy low < medium < high <= 100")]
|
||||
InvalidThresholds,
|
||||
/// An input resource limit is zero.
|
||||
#[error("input limit `{field}` must be greater than zero")]
|
||||
|
|
@ -574,7 +574,7 @@ fn apply_input_layer(target: &mut InputLimits, layer: InputLayer) {
|
|||
}
|
||||
}
|
||||
|
||||
fn apply_threshold_layer(target: &mut ConfidenceThresholds, layer: ThresholdLayer) {
|
||||
fn apply_threshold_layer(target: &mut EvidenceThresholds, layer: ThresholdLayer) {
|
||||
if let Some(value) = layer.low {
|
||||
target.low = value;
|
||||
}
|
||||
|
|
@ -769,8 +769,8 @@ pub struct Config {
|
|||
pub replacements: Vec<ReplacementRule>,
|
||||
/// Configured detection rules in deterministic declaration order.
|
||||
pub rules: Vec<HeuristicRule>,
|
||||
/// Named confidence-band boundaries.
|
||||
pub thresholds: ConfidenceThresholds,
|
||||
/// Named evidence-band boundaries.
|
||||
pub thresholds: EvidenceThresholds,
|
||||
/// Resource and traversal limits applied while reading inputs.
|
||||
pub input: InputLimits,
|
||||
}
|
||||
|
|
@ -856,15 +856,15 @@ pub enum ContentKind {
|
|||
SourceCode,
|
||||
}
|
||||
|
||||
/// Confidence boundaries used to name aggregate score bands.
|
||||
/// Boundaries used to name aggregate evidence-score bands.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ConfidenceThresholds {
|
||||
/// First score considered low confidence.
|
||||
pub struct EvidenceThresholds {
|
||||
/// First score considered low evidence.
|
||||
pub low: u8,
|
||||
/// First score considered medium confidence.
|
||||
/// First score considered medium evidence.
|
||||
pub medium: u8,
|
||||
/// First score considered high confidence.
|
||||
/// First score considered high evidence.
|
||||
pub high: u8,
|
||||
}
|
||||
|
||||
|
|
@ -925,7 +925,7 @@ impl Config {
|
|||
}
|
||||
}
|
||||
|
||||
fn validate_thresholds(thresholds: ConfidenceThresholds) -> Result<(), ConfigValidationError> {
|
||||
fn validate_thresholds(thresholds: EvidenceThresholds) -> Result<(), ConfigValidationError> {
|
||||
if !(thresholds.low < thresholds.medium
|
||||
&& thresholds.medium < thresholds.high
|
||||
&& thresholds.high <= 100)
|
||||
|
|
@ -1503,7 +1503,7 @@ weight = 9.5
|
|||
assert_eq!(config.input.max_walk_depth, 66);
|
||||
assert_eq!(
|
||||
config.thresholds,
|
||||
ConfidenceThresholds {
|
||||
EvidenceThresholds {
|
||||
low: 10,
|
||||
medium: 40,
|
||||
high: 90
|
||||
|
|
@ -1575,7 +1575,7 @@ scope = { content = ["source-code"], extensions = ["rs"], include = ["src/**"],
|
|||
assert_eq!(config.input.max_walk_depth, 106);
|
||||
assert_eq!(
|
||||
config.thresholds,
|
||||
ConfidenceThresholds {
|
||||
EvidenceThresholds {
|
||||
low: 10,
|
||||
medium: 40,
|
||||
high: 90
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use globset::{Glob, GlobSet, GlobSetBuilder};
|
|||
use regex::Regex;
|
||||
|
||||
use crate::{
|
||||
Analysis, Category, Confidence, ConfidenceBand, Config, ConfigValidationError, ContentKind,
|
||||
Analysis, Category, Config, ConfigValidationError, ContentKind, EvidenceBand, EvidenceScore,
|
||||
Finding, HeuristicRule, MatcherType, RuleId, SourceId, replacement,
|
||||
};
|
||||
|
||||
|
|
@ -167,14 +167,14 @@ impl Detector {
|
|||
.flat_map(|rule| rule.findings(&self.config, source))
|
||||
.collect::<Vec<_>>();
|
||||
normalize_findings(&mut findings);
|
||||
let confidence = aggregate_confidence(&self.config, &findings);
|
||||
let confidence_band = (!findings.is_empty())
|
||||
.then(|| band_for_score(self.config.thresholds, confidence.score()));
|
||||
let evidence_score = aggregate_evidence_score(&self.config, &findings);
|
||||
let evidence_band = (!findings.is_empty())
|
||||
.then(|| band_for_score(self.config.thresholds, evidence_score.score()));
|
||||
|
||||
Analysis {
|
||||
findings,
|
||||
confidence,
|
||||
confidence_band,
|
||||
evidence_score,
|
||||
evidence_band,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -191,7 +191,7 @@ fn normalize_findings(findings: &mut Vec<Finding>) {
|
|||
.dedup_by(|right, left| right.rule_id == left.rule_id && right.byte_span == left.byte_span);
|
||||
}
|
||||
|
||||
fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
|
||||
fn aggregate_evidence_score(config: &Config, findings: &[Finding]) -> EvidenceScore {
|
||||
let mut family_rule_counts = BTreeMap::<Category, usize>::new();
|
||||
let mut evidence = 0.0;
|
||||
|
||||
|
|
@ -220,7 +220,7 @@ fn aggregate_confidence(config: &Config, findings: &[Finding]) -> Confidence {
|
|||
} else {
|
||||
100.0 * evidence / (evidence + 50.0)
|
||||
};
|
||||
Confidence::from_score(normalized.round() as u8)
|
||||
EvidenceScore::from_score(normalized.round() as u8)
|
||||
}
|
||||
|
||||
const fn family_decay(category: Category) -> f64 {
|
||||
|
|
@ -230,15 +230,15 @@ const fn family_decay(category: Category) -> f64 {
|
|||
}
|
||||
}
|
||||
|
||||
fn band_for_score(thresholds: crate::ConfidenceThresholds, score: u8) -> ConfidenceBand {
|
||||
fn band_for_score(thresholds: crate::EvidenceThresholds, score: u8) -> EvidenceBand {
|
||||
if score >= thresholds.high {
|
||||
ConfidenceBand::High
|
||||
EvidenceBand::High
|
||||
} else if score >= thresholds.medium {
|
||||
ConfidenceBand::Medium
|
||||
EvidenceBand::Medium
|
||||
} else if score >= thresholds.low {
|
||||
ConfidenceBand::Low
|
||||
EvidenceBand::Low
|
||||
} else {
|
||||
ConfidenceBand::BelowThreshold
|
||||
EvidenceBand::BelowThreshold
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -523,11 +523,8 @@ mod tests {
|
|||
);
|
||||
assert_eq!(analysis.findings[0].matched_text, "—");
|
||||
assert_eq!(analysis.findings[0].fix.as_ref().unwrap().replacement, "-");
|
||||
assert_eq!(analysis.confidence.score(), 17);
|
||||
assert_eq!(
|
||||
analysis.confidence_band,
|
||||
Some(ConfidenceBand::BelowThreshold)
|
||||
);
|
||||
assert_eq!(analysis.evidence_score.score(), 17);
|
||||
assert_eq!(analysis.evidence_band, Some(EvidenceBand::BelowThreshold));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -720,18 +717,18 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn confidence_caps_occurrences_and_applies_diminishing_returns() {
|
||||
fn evidence_score_caps_occurrences_and_applies_diminishing_returns() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let five = detector.analyze(&"—".repeat(5));
|
||||
let twenty = detector.analyze(&"—".repeat(20));
|
||||
|
||||
assert_eq!(five.confidence, twenty.confidence);
|
||||
assert_eq!(five.confidence.score(), 28);
|
||||
assert_eq!(five.evidence_score, twenty.evidence_score);
|
||||
assert_eq!(five.evidence_score.score(), 28);
|
||||
assert_eq!(twenty.findings.len(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confidence_damps_distinct_rules_in_the_same_family() {
|
||||
fn evidence_score_damps_distinct_rules_in_the_same_family() {
|
||||
let source = "As an AI language model, I cannot browse the internet.";
|
||||
let correlated = Detector::new(&Config::embedded()).unwrap().analyze(source);
|
||||
let mut independent_config = Config::embedded();
|
||||
|
|
@ -743,7 +740,7 @@ mod tests {
|
|||
.category = Category::Style;
|
||||
let independent = Detector::new(&independent_config).unwrap().analyze(source);
|
||||
|
||||
assert!(correlated.confidence < independent.confidence);
|
||||
assert!(correlated.evidence_score < independent.evidence_score);
|
||||
assert_eq!(correlated.findings.len(), independent.findings.len());
|
||||
}
|
||||
|
||||
|
|
@ -763,46 +760,43 @@ mod tests {
|
|||
let analysis = Detector::new(&Config::embedded()).unwrap().analyze(source);
|
||||
|
||||
assert_eq!(analysis.findings.len(), 6);
|
||||
assert_eq!(analysis.confidence.score(), 67);
|
||||
assert_eq!(analysis.confidence_band, Some(ConfidenceBand::Medium));
|
||||
assert_eq!(analysis.evidence_score.score(), 67);
|
||||
assert_eq!(analysis.evidence_band, Some(EvidenceBand::Medium));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_thresholds_define_named_band_boundaries() {
|
||||
let thresholds = crate::ConfidenceThresholds {
|
||||
let thresholds = crate::EvidenceThresholds {
|
||||
low: 20,
|
||||
medium: 50,
|
||||
high: 75,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
band_for_score(thresholds, 19),
|
||||
ConfidenceBand::BelowThreshold
|
||||
);
|
||||
assert_eq!(band_for_score(thresholds, 20), ConfidenceBand::Low);
|
||||
assert_eq!(band_for_score(thresholds, 50), ConfidenceBand::Medium);
|
||||
assert_eq!(band_for_score(thresholds, 75), ConfidenceBand::High);
|
||||
assert_eq!(band_for_score(thresholds, 19), EvidenceBand::BelowThreshold);
|
||||
assert_eq!(band_for_score(thresholds, 20), EvidenceBand::Low);
|
||||
assert_eq!(band_for_score(thresholds, 50), EvidenceBand::Medium);
|
||||
assert_eq!(band_for_score(thresholds, 75), EvidenceBand::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_weak_repeated_signal_cannot_reach_high_confidence() {
|
||||
fn one_weak_repeated_signal_cannot_reach_high_evidence() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
let source = "Moreover, first.\nAdditionally, second.\nFurthermore, third.";
|
||||
let analysis = detector.analyze(source);
|
||||
|
||||
assert_eq!(analysis.findings.len(), 3);
|
||||
assert_ne!(analysis.confidence_band, Some(ConfidenceBand::High));
|
||||
assert_ne!(analysis.evidence_band, Some(EvidenceBand::High));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_clean_text_have_zero_confidence_and_no_verdict() {
|
||||
fn empty_and_clean_text_have_zero_evidence_and_no_verdict() {
|
||||
let detector = Detector::new(&Config::embedded()).unwrap();
|
||||
|
||||
for source in ["", "A specific, carefully sourced statement."] {
|
||||
let analysis = detector.analyze(source);
|
||||
assert!(analysis.findings.is_empty());
|
||||
assert_eq!(analysis.confidence.score(), 0);
|
||||
assert_eq!(analysis.confidence_band, None);
|
||||
assert_eq!(analysis.evidence_score.score(), 0);
|
||||
assert_eq!(analysis.evidence_band, None);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
18
src/lib.rs
18
src/lib.rs
|
|
@ -15,8 +15,8 @@ mod model;
|
|||
mod replacement;
|
||||
|
||||
pub use config::{
|
||||
ConfidenceThresholds, Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName,
|
||||
ConfigNameError, ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind,
|
||||
Config, ConfigError, ConfigLayerError, ConfigLoader, ConfigName, ConfigNameError,
|
||||
ConfigOverrides, ConfigPaths, ConfigValidationError, ContentKind, EvidenceThresholds,
|
||||
HeuristicRule, InputLimits, InputOverrides, MatcherType, ReplacementRule, RuleScope,
|
||||
ThresholdOverrides,
|
||||
};
|
||||
|
|
@ -25,9 +25,21 @@ pub use diagnostic::FindingDiagnostic;
|
|||
pub use error::{InputError, InputResult};
|
||||
pub use input::{Document, Documents, read_stdin};
|
||||
pub use model::{
|
||||
Analysis, AnalysisSummary, Category, Confidence, ConfidenceBand, Finding, FindingSeverity,
|
||||
Analysis, AnalysisSummary, Category, EvidenceBand, EvidenceScore, Finding, FindingSeverity,
|
||||
RuleId, SeverityCounts, SourceId, TextEdit,
|
||||
};
|
||||
|
||||
/// Deprecated name for [`EvidenceScore`].
|
||||
#[deprecated(since = "0.1.0", note = "use EvidenceScore")]
|
||||
pub type Confidence = EvidenceScore;
|
||||
|
||||
/// Deprecated name for [`EvidenceBand`].
|
||||
#[deprecated(since = "0.1.0", note = "use EvidenceBand")]
|
||||
pub type ConfidenceBand = EvidenceBand;
|
||||
|
||||
/// Deprecated name for [`EvidenceThresholds`].
|
||||
#[deprecated(since = "0.1.0", note = "use EvidenceThresholds")]
|
||||
pub type ConfidenceThresholds = EvidenceThresholds;
|
||||
pub use replacement::ReplacementError;
|
||||
|
||||
/// Load embedded defaults plus optional platform user configuration.
|
||||
|
|
|
|||
52
src/model.rs
52
src/model.rs
|
|
@ -92,14 +92,14 @@ impl fmt::Display for FindingSeverity {
|
|||
}
|
||||
}
|
||||
|
||||
/// Confidence that configured low-care signals are present.
|
||||
/// Bounded strength of the configured low-care evidence found in a document.
|
||||
///
|
||||
/// This is deliberately not an AI-authorship probability.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Confidence(u8);
|
||||
pub struct EvidenceScore(u8);
|
||||
|
||||
impl Confidence {
|
||||
/// Construct a confidence value, clamping values above 100.
|
||||
impl EvidenceScore {
|
||||
/// Construct an evidence score, clamping values above 100.
|
||||
pub const fn from_score(score: u8) -> Self {
|
||||
Self(if score > 100 { 100 } else { score })
|
||||
}
|
||||
|
|
@ -113,7 +113,7 @@ impl Confidence {
|
|||
/// Named interpretation of a non-empty analysis score.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum ConfidenceBand {
|
||||
pub enum EvidenceBand {
|
||||
/// Evidence exists but remains below the configured low threshold.
|
||||
BelowThreshold,
|
||||
/// Score meets the configured low threshold.
|
||||
|
|
@ -124,7 +124,7 @@ pub enum ConfidenceBand {
|
|||
High,
|
||||
}
|
||||
|
||||
impl ConfidenceBand {
|
||||
impl EvidenceBand {
|
||||
/// Return the stable kebab-case presentation name.
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
|
|
@ -136,7 +136,7 @@ impl ConfidenceBand {
|
|||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ConfidenceBand {
|
||||
impl fmt::Display for EvidenceBand {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
|
|
@ -207,18 +207,18 @@ pub struct Finding {
|
|||
pub struct Analysis {
|
||||
/// Findings sorted by source offset and rule identifier.
|
||||
pub findings: Vec<Finding>,
|
||||
/// Aggregate confidence that configured signals are present.
|
||||
pub confidence: Confidence,
|
||||
/// Aggregate strength of the configured evidence found.
|
||||
pub evidence_score: EvidenceScore,
|
||||
/// Named score band, or no verdict when no evidence was found.
|
||||
pub confidence_band: Option<ConfidenceBand>,
|
||||
pub evidence_band: Option<EvidenceBand>,
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
/// Summarize score, named band, and severity counts without consuming findings.
|
||||
pub fn summary(&self) -> AnalysisSummary {
|
||||
AnalysisSummary {
|
||||
confidence_score: self.confidence.score(),
|
||||
confidence_band: self.confidence_band,
|
||||
evidence_score: self.evidence_score.score(),
|
||||
evidence_band: self.evidence_band,
|
||||
findings: SeverityCounts::from_findings(&self.findings),
|
||||
}
|
||||
}
|
||||
|
|
@ -228,9 +228,9 @@ impl Analysis {
|
|||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct AnalysisSummary {
|
||||
/// Normalized evidence score in `0..=100`.
|
||||
pub confidence_score: u8,
|
||||
pub evidence_score: u8,
|
||||
/// Named band, or no verdict when the source has no evidence.
|
||||
pub confidence_band: Option<ConfidenceBand>,
|
||||
pub evidence_band: Option<EvidenceBand>,
|
||||
/// Finding totals grouped by configured severity.
|
||||
pub findings: SeverityCounts,
|
||||
}
|
||||
|
|
@ -270,10 +270,10 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn confidence_is_bounded() {
|
||||
assert_eq!(Confidence::from_score(0).score(), 0);
|
||||
assert_eq!(Confidence::from_score(73).score(), 73);
|
||||
assert_eq!(Confidence::from_score(255).score(), 100);
|
||||
fn evidence_score_is_bounded() {
|
||||
assert_eq!(EvidenceScore::from_score(0).score(), 0);
|
||||
assert_eq!(EvidenceScore::from_score(73).score(), 73);
|
||||
assert_eq!(EvidenceScore::from_score(255).score(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -330,15 +330,15 @@ mod tests {
|
|||
fix: None,
|
||||
},
|
||||
],
|
||||
confidence: Confidence::from_score(32),
|
||||
confidence_band: Some(ConfidenceBand::Low),
|
||||
evidence_score: EvidenceScore::from_score(32),
|
||||
evidence_band: Some(EvidenceBand::Low),
|
||||
};
|
||||
|
||||
let summary = analysis.summary();
|
||||
|
||||
assert_eq!(analysis.findings.len(), 2);
|
||||
assert_eq!(summary.confidence_score, 32);
|
||||
assert_eq!(summary.confidence_band, Some(ConfidenceBand::Low));
|
||||
assert_eq!(summary.evidence_score, 32);
|
||||
assert_eq!(summary.evidence_band, Some(EvidenceBand::Low));
|
||||
assert_eq!(summary.findings.advice, 1);
|
||||
assert_eq!(summary.findings.warnings, 1);
|
||||
assert_eq!(summary.findings.errors, 0);
|
||||
|
|
@ -365,10 +365,10 @@ mod tests {
|
|||
assert_eq!(severity.to_string(), expected);
|
||||
}
|
||||
for (band, expected) in [
|
||||
(ConfidenceBand::BelowThreshold, "below-threshold"),
|
||||
(ConfidenceBand::Low, "low"),
|
||||
(ConfidenceBand::Medium, "medium"),
|
||||
(ConfidenceBand::High, "high"),
|
||||
(EvidenceBand::BelowThreshold, "below-threshold"),
|
||||
(EvidenceBand::Low, "low"),
|
||||
(EvidenceBand::Medium, "medium"),
|
||||
(EvidenceBand::High, "high"),
|
||||
] {
|
||||
assert_eq!(band.as_str(), expected);
|
||||
assert_eq!(band.to_string(), expected);
|
||||
|
|
|
|||
17
tasks.md
17
tasks.md
|
|
@ -293,6 +293,23 @@ Optional follow-up, intentionally not required for this milestone:
|
|||
sign-off feature: harden its change boundary beyond the catalog hash, or
|
||||
remove it if its maintenance cost exceeds its demonstrated value.
|
||||
|
||||
## Milestone 11 — Evidence-score terminology
|
||||
|
||||
- [x] Rename the public domain model from ambiguous confidence terminology to
|
||||
`EvidenceScore`, `EvidenceBand`, and `EvidenceThresholds`; retain deprecated
|
||||
Rust type aliases and the legacy `--fail-confidence` CLI alias for migration.
|
||||
- [ ] Publish the low-care evidence-score wording across terminal output,
|
||||
examples, and documentation; rename `--fail-confidence` to `--fail-score` and
|
||||
version the JSON field rename as schema version 2.
|
||||
- [ ] Run the complete quality and coverage gates, refresh generated badges, and
|
||||
record the compatibility boundary and unchanged scoring formula.
|
||||
|
||||
Acceptance: every active user-facing contract describes the metric as a bounded
|
||||
low-care evidence score rather than a probability or calibrated confidence;
|
||||
legacy type and CLI names remain usable with clear migration guidance; JSON
|
||||
consumers can identify the field rename by schema version; scoring results are
|
||||
unchanged; and all quality/coverage gates pass.
|
||||
|
||||
## Deferred by design
|
||||
|
||||
- Statistical or model-based authorship classification.
|
||||
|
|
|
|||
12
tests/cli.rs
12
tests/cli.rs
|
|
@ -48,9 +48,9 @@ fn help_is_successful_and_names_the_command() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_confidence_is_a_usage_error() {
|
||||
fn invalid_evidence_score_is_a_usage_error() {
|
||||
let output = detector()
|
||||
.args(["--fail-confidence", "101", "file.txt"])
|
||||
.args(["--fail-score", "101", "file.txt"])
|
||||
.output()
|
||||
.expect("run detector");
|
||||
|
||||
|
|
@ -84,12 +84,12 @@ fn findings_control_the_threshold_exit_code() {
|
|||
fs::write(&suspicious, "suspicious—dash").expect("write suspicious fixture");
|
||||
|
||||
let clean_output = detector()
|
||||
.args(["--fail-confidence", "10"])
|
||||
.args(["--fail-score", "10"])
|
||||
.arg(&clean)
|
||||
.output()
|
||||
.expect("run clean detector");
|
||||
let suspicious_output = detector()
|
||||
.args(["--fail-confidence", "10"])
|
||||
.args(["--fail-score", "10"])
|
||||
.arg(&suspicious)
|
||||
.output()
|
||||
.expect("run suspicious detector");
|
||||
|
|
@ -182,7 +182,9 @@ fn json_spans_are_utf8_bytes_and_schema_is_complete() {
|
|||
let records = json_records(&output);
|
||||
assert_eq!(records.len(), 1);
|
||||
let record = &records[0];
|
||||
assert_eq!(record["schema_version"], 1);
|
||||
assert_eq!(record["schema_version"], 2);
|
||||
assert!(record["summary"]["evidence_score"].is_number());
|
||||
assert!(record["summary"].get("confidence_score").is_none());
|
||||
assert_eq!(record["kind"], "analysis");
|
||||
assert_eq!(record["source"], "<stdin>");
|
||||
assert_eq!(record["summary"]["total"], 1);
|
||||
|
|
|
|||
|
|
@ -51,9 +51,9 @@ fn reviewed_corpus_reports_per_rule_metrics_and_false_positive_rate() {
|
|||
};
|
||||
let analysis = detector.analyze_with(&text, &context);
|
||||
assert_eq!(
|
||||
analysis.confidence.score(),
|
||||
analysis.evidence_score.score(),
|
||||
case.expected_score,
|
||||
"confidence score for {}",
|
||||
"evidence score for {}",
|
||||
case.path
|
||||
);
|
||||
let actual = analysis
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ fn diagnostic_showcase_is_an_expected_threshold_failure() {
|
|||
assert!(stdout.contains("assistant-self-identification"));
|
||||
assert!(stdout.contains("prompt-response-scaffolding"));
|
||||
assert!(stdout.contains("typography-non-ascii-hyphen"));
|
||||
assert!(stdout.contains("confidence 71 (medium)"));
|
||||
assert!(stdout.contains("low-care evidence score: 71 (medium)"));
|
||||
assert!(stdout.contains("advice"));
|
||||
assert!(stdout.contains("warning"));
|
||||
assert!(stdout.contains("error"));
|
||||
|
|
|
|||
|
|
@ -154,6 +154,6 @@ fn ordinary_multilingual_text_is_not_evidence() {
|
|||
let analysis = detector.analyze_with(text, &context(FixtureKind::Prose));
|
||||
|
||||
assert!(analysis.findings.is_empty());
|
||||
assert_eq!(analysis.confidence.score(), 0);
|
||||
assert!(analysis.confidence_band.is_none());
|
||||
assert_eq!(analysis.evidence_score.score(), 0);
|
||||
assert!(analysis.evidence_band.is_none());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ Options:
|
|||
Skip source paths matching a glob; may be supplied more than once
|
||||
--output <human|miette|concise|json>
|
||||
Select colored graphical Miette, plain human, concise, or versioned JSON output [default: miette]
|
||||
--fail-confidence <FAIL_CONFIDENCE>
|
||||
Return exit code 1 when a document reaches this evidence-confidence score [default: 100]
|
||||
--fail-score <FAIL_SCORE>
|
||||
Return exit code 1 when a document reaches this low-care evidence score [default: 100] [alias: --fail-confidence]
|
||||
--fail-level <FAIL_LEVEL>
|
||||
Return exit code 1 at the configured low, medium, or high threshold [possible values: low, medium, high]
|
||||
-h, --help
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@
|
|||
source: tests/cli.rs
|
||||
expression: "String::from_utf8(output.stderr).expect(\"UTF-8 stderr\")"
|
||||
---
|
||||
error: invalid value '101' for '--fail-confidence <FAIL_CONFIDENCE>': confidence must be an integer from 0 through 100
|
||||
error: invalid value '101' for '--fail-score <FAIL_SCORE>': evidence score must be an integer from 0 through 100
|
||||
|
||||
For more information, try '--help'.
|
||||
Loading…
Reference in New Issue