fix: preserve results across input failures
This commit is contained in:
parent
7d8a45433a
commit
593e91a4bc
|
|
@ -45,6 +45,13 @@ defaults to 100. `--fail-level low|medium|high` instead uses the corresponding
|
|||
threshold from the fully resolved configuration. The two options are mutually
|
||||
exclusive.
|
||||
|
||||
Input failures are isolated per document whenever iteration can continue. Valid
|
||||
documents still emit all findings, while any partial input failure makes the
|
||||
final status `2` (taking precedence over threshold status `1`). Human and
|
||||
concise input errors go to stderr. JSON mode emits versioned `input-error`
|
||||
records on stdout alongside `analysis` records and leaves stderr empty unless a
|
||||
fatal setup failure prevents the output contract from starting.
|
||||
|
||||
Confidence is a summary of observable configured evidence. It is not an
|
||||
AI-authorship probability. See [Heuristics and confidence](heuristics.md) for
|
||||
the formula and [Limitations](limitations.md) before using thresholds in review
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ use john_slop::{
|
|||
};
|
||||
use miette::{IntoDiagnostic, Result, WrapErr};
|
||||
|
||||
use output::{OutputFormat, render_analysis};
|
||||
use output::{OutputFormat, render_analysis, render_input_error};
|
||||
|
||||
/// Inspect files, directories, or ZIP-based documents for low-care signals.
|
||||
#[derive(Debug, Parser)]
|
||||
|
|
@ -136,8 +136,9 @@ enum FailLevel {
|
|||
fn main() -> ExitCode {
|
||||
let cli = Cli::parse();
|
||||
match run(&cli) {
|
||||
Ok(true) => ExitCode::from(1),
|
||||
Ok(false) => ExitCode::SUCCESS,
|
||||
Ok(outcome) if outcome.input_errors => ExitCode::from(2),
|
||||
Ok(outcome) if outcome.threshold_reached => ExitCode::from(1),
|
||||
Ok(_) => ExitCode::SUCCESS,
|
||||
Err(error) => {
|
||||
eprintln!("{error:?}");
|
||||
ExitCode::from(2)
|
||||
|
|
@ -145,8 +146,14 @@ fn main() -> ExitCode {
|
|||
}
|
||||
}
|
||||
|
||||
/// Execute one parsed detector invocation and report whether its threshold fired.
|
||||
fn run(cli: &Cli) -> Result<bool> {
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct RunOutcome {
|
||||
threshold_reached: bool,
|
||||
input_errors: bool,
|
||||
}
|
||||
|
||||
/// Execute one parsed detector invocation and retain threshold/error state.
|
||||
fn run(cli: &Cli) -> Result<RunOutcome> {
|
||||
let loader = ConfigLoader::discover(cli.config_name.as_ref(), cli.config.clone())?
|
||||
.with_overrides(ConfigOverrides {
|
||||
input: InputOverrides {
|
||||
|
|
@ -168,42 +175,59 @@ fn run(cli: &Cli) -> Result<bool> {
|
|||
}
|
||||
ConfigCommand::Paths => print_config_paths(&loader),
|
||||
}
|
||||
return Ok(false);
|
||||
return Ok(RunOutcome::default());
|
||||
}
|
||||
|
||||
let config = loader.load()?;
|
||||
let detector = Detector::new(&config).into_diagnostic()?;
|
||||
let ignore = compile_ignore_globs(&cli.ignore)?;
|
||||
let failure_threshold = fail_threshold(cli, &config);
|
||||
let mut threshold_reached = false;
|
||||
let mut outcome = RunOutcome::default();
|
||||
|
||||
if cli.stdin {
|
||||
let document = read_stdin(std::io::stdin().lock(), &config.input).into_diagnostic()?;
|
||||
let context = cli
|
||||
.stdin_filename
|
||||
.as_ref()
|
||||
.map_or_else(AnalysisContext::prose, |path| {
|
||||
AnalysisContext::for_path(path.clone())
|
||||
});
|
||||
if !is_ignored(&ignore, cli.stdin_filename.as_deref()) {
|
||||
let analysis = detector.analyze_with(&document.content, &context);
|
||||
print!(
|
||||
"{}",
|
||||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
match read_stdin(std::io::stdin().lock(), &config.input) {
|
||||
Ok(document) => {
|
||||
let context = cli
|
||||
.stdin_filename
|
||||
.as_ref()
|
||||
.map_or_else(AnalysisContext::prose, |path| {
|
||||
AnalysisContext::for_path(path.clone())
|
||||
});
|
||||
if !is_ignored(&ignore, cli.stdin_filename.as_deref()) {
|
||||
let analysis = detector.analyze_with(&document.content, &context);
|
||||
print!(
|
||||
"{}",
|
||||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
emit_input_error(cli.output, "<stdin>", &error.to_string())?;
|
||||
outcome.input_errors = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for path in &cli.paths {
|
||||
let documents = read_inputs(&config, path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to open requested input `{}`", path.display()))?;
|
||||
let documents = match read_inputs(&config, path) {
|
||||
Ok(documents) => documents,
|
||||
Err(error) => {
|
||||
emit_input_error(cli.output, &path.display().to_string(), &error.to_string())?;
|
||||
outcome.input_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for document in documents {
|
||||
let document = document
|
||||
.into_diagnostic()
|
||||
.wrap_err("failed to read an input document")?;
|
||||
let document = match document {
|
||||
Ok(document) => document,
|
||||
Err(error) => {
|
||||
emit_input_error(cli.output, &path.display().to_string(), &error.to_string())?;
|
||||
outcome.input_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if is_ignored(&ignore, source_path(&document.source)) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -214,11 +238,21 @@ fn run(cli: &Cli) -> Result<bool> {
|
|||
render_analysis(cli.output, &document.source, &document.content, &analysis)
|
||||
.into_diagnostic()?
|
||||
);
|
||||
threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(threshold_reached)
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
fn emit_input_error(format: OutputFormat, source: &str, message: &str) -> Result<()> {
|
||||
let rendered = render_input_error(format, source, message).into_diagnostic()?;
|
||||
if format == OutputFormat::Json {
|
||||
print!("{rendered}");
|
||||
} else {
|
||||
eprint!("{rendered}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const fn fail_threshold(cli: &Cli, config: &Config) -> u8 {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,33 @@ pub(super) fn render_analysis(
|
|||
match format {
|
||||
OutputFormat::Human => Ok(render_human(source_id, source, analysis)),
|
||||
OutputFormat::Concise => Ok(render_concise(source_id, analysis)),
|
||||
OutputFormat::Json => serde_json::to_string(&JsonAnalysis::new(source_id, analysis)),
|
||||
OutputFormat::Json => {
|
||||
serde_json::to_string(&JsonAnalysis::new(source_id, analysis)).map(|mut record| {
|
||||
record.push('\n');
|
||||
record
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn render_input_error(
|
||||
format: OutputFormat,
|
||||
source: &str,
|
||||
message: &str,
|
||||
) -> Result<String, serde_json::Error> {
|
||||
match format {
|
||||
OutputFormat::Human => Ok(format!("{source}: input error: {message}\n")),
|
||||
OutputFormat::Concise => Ok(format!("{source}: input-error: {message}\n")),
|
||||
OutputFormat::Json => serde_json::to_string(&JsonInputError {
|
||||
schema_version: 1,
|
||||
kind: "input-error",
|
||||
source,
|
||||
message,
|
||||
})
|
||||
.map(|mut record| {
|
||||
record.push('\n');
|
||||
record
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -183,6 +209,14 @@ struct JsonSpan {
|
|||
end: usize,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct JsonInputError<'a> {
|
||||
schema_version: u8,
|
||||
kind: &'static str,
|
||||
source: &'a str,
|
||||
message: &'a str,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use john_slop::{AnalysisContext, Config, Detector};
|
||||
|
|
@ -225,13 +259,24 @@ mod tests {
|
|||
&analysis,
|
||||
)
|
||||
.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(&output).unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
|
||||
|
||||
assert_eq!(value["schema_version"], 1);
|
||||
assert_eq!(value["kind"], "analysis");
|
||||
assert_eq!(value["source"], "article.md");
|
||||
assert_eq!(value["findings"][0]["byte_span"]["start"], 8);
|
||||
assert!(!output.contains('\u{1b}'));
|
||||
assert!(!output.contains('\n'));
|
||||
assert_eq!(output.lines().count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_input_errors_are_versioned_line_records() {
|
||||
let output = render_input_error(OutputFormat::Json, "broken.txt", "invalid UTF-8").unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
|
||||
|
||||
assert_eq!(value["schema_version"], 1);
|
||||
assert_eq!(value["kind"], "input-error");
|
||||
assert_eq!(value["source"], "broken.txt");
|
||||
assert!(!output.contains('\u{1b}'));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -148,7 +148,7 @@ formula are pinned by tests.
|
|||
diagnostics never contain logs or ANSI escapes.
|
||||
- [x] Implement documented exit codes: `0` below fail threshold, `1` at/above it,
|
||||
`2` for usage/configuration/I/O failure. Add `--fail-level`.
|
||||
- [ ] Report per-document input errors without hiding findings from other
|
||||
- [x] Report per-document input errors without hiding findings from other
|
||||
documents; define whether partial failure returns `2` and test it.
|
||||
- [ ] Add CLI integration tests for spans containing multibyte UTF-8, multiple
|
||||
files, stdin, missing paths, malformed configs, stable ordering, JSON schema,
|
||||
|
|
|
|||
24
tests/cli.rs
24
tests/cli.rs
|
|
@ -93,3 +93,27 @@ fn config_paths_lists_embedded_and_platform_sources() {
|
|||
assert!(stdout.contains("config.toml"));
|
||||
assert!(stdout.contains("<not selected>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_document_errors_preserve_other_json_results_and_force_exit_two() {
|
||||
let directory = tempdir().expect("temp dir");
|
||||
fs::write(directory.path().join("a-invalid.txt"), [0xff]).unwrap();
|
||||
fs::write(directory.path().join("b-finding.txt"), "review—this").unwrap();
|
||||
|
||||
let output = detector()
|
||||
.args(["--output", "json"])
|
||||
.arg(directory.path())
|
||||
.output()
|
||||
.expect("run detector");
|
||||
|
||||
assert_eq!(output.status.code(), Some(2));
|
||||
assert!(output.stderr.is_empty());
|
||||
let records = String::from_utf8(output.stdout)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(records.len(), 2);
|
||||
assert!(records.iter().any(|record| record["kind"] == "input-error"));
|
||||
assert!(records.iter().any(|record| record["kind"] == "analysis"));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue