feat: add configured CLI failure levels
This commit is contained in:
parent
cf4f0be12d
commit
7d8a45433a
|
|
@ -40,6 +40,11 @@ Exit statuses are stable:
|
|||
- `1`: at least one document reaches the threshold;
|
||||
- `2`: arguments are invalid or an operational/input error prevents completion.
|
||||
|
||||
`--fail-confidence SCORE` selects an exact inclusive score threshold and
|
||||
defaults to 100. `--fail-level low|medium|high` instead uses the corresponding
|
||||
threshold from the fully resolved configuration. The two options are mutually
|
||||
exclusive.
|
||||
|
||||
Confidence is a summary of observable configured evidence. It is not an
|
||||
AI-authorship probability. See [Heuristics and confidence](heuristics.md) for
|
||||
the formula and [Limitations](limitations.md) before using thresholds in review
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ use std::{
|
|||
process::ExitCode,
|
||||
};
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
use globset::{Glob, GlobSet, GlobSetBuilder};
|
||||
use john_slop::{
|
||||
AnalysisContext, ConfigLoader, ConfigName, ConfigOverrides, Detector, InputOverrides, SourceId,
|
||||
read_inputs, read_stdin,
|
||||
AnalysisContext, Config, ConfigLoader, ConfigName, ConfigOverrides, Detector, InputOverrides,
|
||||
SourceId, read_inputs, read_stdin,
|
||||
};
|
||||
use miette::{IntoDiagnostic, Result, WrapErr};
|
||||
|
||||
|
|
@ -84,9 +84,18 @@ struct Cli {
|
|||
output: OutputFormat,
|
||||
|
||||
/// Return exit code 1 when a document reaches this evidence-confidence score.
|
||||
#[arg(long, default_value_t = 100, value_parser = parse_confidence)]
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = 100,
|
||||
value_parser = parse_confidence,
|
||||
conflicts_with = "fail_level"
|
||||
)]
|
||||
fail_confidence: u8,
|
||||
|
||||
/// Return exit code 1 at the configured low, medium, or high threshold.
|
||||
#[arg(long, value_enum, conflicts_with = "fail_confidence")]
|
||||
fail_level: Option<FailLevel>,
|
||||
|
||||
/// Files, directories, or supported ZIP-based containers to inspect.
|
||||
#[arg(required_unless_present = "stdin", value_name = "PATH")]
|
||||
paths: Vec<PathBuf>,
|
||||
|
|
@ -116,6 +125,14 @@ enum ConfigCommand {
|
|||
Paths,
|
||||
}
|
||||
|
||||
/// Configured confidence band that should make analysis fail.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
enum FailLevel {
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let cli = Cli::parse();
|
||||
match run(&cli) {
|
||||
|
|
@ -157,6 +174,7 @@ fn run(cli: &Cli) -> Result<bool> {
|
|||
let config = loader.load()?;
|
||||
let detector = Detector::new(&config).into_diagnostic()?;
|
||||
let ignore = compile_ignore_globs(&cli.ignore)?;
|
||||
let failure_threshold = fail_threshold(cli, &config);
|
||||
let mut threshold_reached = false;
|
||||
|
||||
if cli.stdin {
|
||||
|
|
@ -174,7 +192,7 @@ fn run(cli: &Cli) -> Result<bool> {
|
|||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
threshold_reached |= analysis.confidence.score() >= cli.fail_confidence;
|
||||
threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -196,13 +214,22 @@ fn run(cli: &Cli) -> Result<bool> {
|
|||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
threshold_reached |= analysis.confidence.score() >= cli.fail_confidence;
|
||||
threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(threshold_reached)
|
||||
}
|
||||
|
||||
const fn fail_threshold(cli: &Cli, config: &Config) -> u8 {
|
||||
match cli.fail_level {
|
||||
Some(FailLevel::Low) => config.thresholds.low,
|
||||
Some(FailLevel::Medium) => config.thresholds.medium,
|
||||
Some(FailLevel::High) => config.thresholds.high,
|
||||
None => cli.fail_confidence,
|
||||
}
|
||||
}
|
||||
|
||||
fn compile_ignore_globs(patterns: &[String]) -> Result<GlobSet> {
|
||||
let mut builder = GlobSetBuilder::new();
|
||||
for pattern in patterns {
|
||||
|
|
@ -275,6 +302,7 @@ mod tests {
|
|||
.expect("valid CLI");
|
||||
|
||||
assert_eq!(cli.fail_confidence, 42);
|
||||
assert_eq!(cli.fail_level, None);
|
||||
assert_eq!(cli.config_name, None);
|
||||
assert!(cli.command.is_none());
|
||||
assert_eq!(
|
||||
|
|
@ -343,4 +371,29 @@ mod tests {
|
|||
assert_eq!(cli.max_walk_depth, Some(4));
|
||||
assert!(cli.paths.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_level_uses_resolved_configured_threshold() {
|
||||
let cli =
|
||||
Cli::try_parse_from(["john-slop", "--fail-level", "medium", "article.md"]).unwrap();
|
||||
let mut config = Config::embedded();
|
||||
config.thresholds.medium = 47;
|
||||
|
||||
assert_eq!(fail_threshold(&cli, &config), 47);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_level_and_explicit_score_are_mutually_exclusive() {
|
||||
let error = Cli::try_parse_from([
|
||||
"john-slop",
|
||||
"--fail-level",
|
||||
"low",
|
||||
"--fail-confidence",
|
||||
"20",
|
||||
"article.md",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(error.exit_code(), 2);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -146,7 +146,7 @@ formula are pinned by tests.
|
|||
for language/path scoping); add ignore and maximum-size controls.
|
||||
- [x] Add human, concise, and versioned JSON output modes. Ensure JSON and stdout
|
||||
diagnostics never contain logs or ANSI escapes.
|
||||
- [ ] Implement documented exit codes: `0` below fail threshold, `1` at/above it,
|
||||
- [x] Implement documented exit codes: `0` below fail threshold, `1` at/above it,
|
||||
`2` for usage/configuration/I/O failure. Add `--fail-level`.
|
||||
- [ ] Report per-document input errors without hiding findings from other
|
||||
documents; define whether partial failure returns `2` and test it.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ Options:
|
|||
Select human, concise, or versioned JSON output [default: human]
|
||||
--fail-confidence <FAIL_CONFIDENCE>
|
||||
Return exit code 1 when a document reaches this evidence-confidence score [default: 100]
|
||||
--fail-level <FAIL_LEVEL>
|
||||
Return exit code 1 at the configured low, medium, or high threshold [possible values: low, medium, high]
|
||||
-h, --help
|
||||
Print help
|
||||
-V, --version
|
||||
|
|
|
|||
Loading…
Reference in New Issue