fix: preserve results across input failures

This commit is contained in:
Dandaedre 2026-09-29 16:00:27 +02:00
parent 7d8a45433a
commit 593e91a4bc
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
5 changed files with 144 additions and 34 deletions

View File

@ -45,6 +45,13 @@ defaults to 100. `--fail-level low|medium|high` instead uses the corresponding
threshold from the fully resolved configuration. The two options are mutually
exclusive.
Input failures are isolated per document whenever iteration can continue. Valid
documents still emit all findings, while any partial input failure makes the
final status `2` (taking precedence over threshold status `1`). Human and
concise input errors go to stderr. JSON mode emits versioned `input-error`
records on stdout alongside `analysis` records and leaves stderr empty unless a
fatal setup failure prevents the output contract from starting.
Confidence is a summary of observable configured evidence. It is not an
AI-authorship probability. See [Heuristics and confidence](heuristics.md) for
the formula and [Limitations](limitations.md) before using thresholds in review

View File

@ -16,7 +16,7 @@ use john_slop::{
};
use miette::{IntoDiagnostic, Result, WrapErr};
use output::{OutputFormat, render_analysis};
use output::{OutputFormat, render_analysis, render_input_error};
/// Inspect files, directories, or ZIP-based documents for low-care signals.
#[derive(Debug, Parser)]
@ -136,8 +136,9 @@ enum FailLevel {
fn main() -> ExitCode {
let cli = Cli::parse();
match run(&cli) {
Ok(true) => ExitCode::from(1),
Ok(false) => ExitCode::SUCCESS,
Ok(outcome) if outcome.input_errors => ExitCode::from(2),
Ok(outcome) if outcome.threshold_reached => ExitCode::from(1),
Ok(_) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("{error:?}");
ExitCode::from(2)
@ -145,8 +146,14 @@ fn main() -> ExitCode {
}
}
/// Execute one parsed detector invocation and report whether its threshold fired.
fn run(cli: &Cli) -> Result<bool> {
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct RunOutcome {
threshold_reached: bool,
input_errors: bool,
}
/// Execute one parsed detector invocation and retain threshold/error state.
fn run(cli: &Cli) -> Result<RunOutcome> {
let loader = ConfigLoader::discover(cli.config_name.as_ref(), cli.config.clone())?
.with_overrides(ConfigOverrides {
input: InputOverrides {
@ -168,42 +175,59 @@ fn run(cli: &Cli) -> Result<bool> {
}
ConfigCommand::Paths => print_config_paths(&loader),
}
return Ok(false);
return Ok(RunOutcome::default());
}
let config = loader.load()?;
let detector = Detector::new(&config).into_diagnostic()?;
let ignore = compile_ignore_globs(&cli.ignore)?;
let failure_threshold = fail_threshold(cli, &config);
let mut threshold_reached = false;
let mut outcome = RunOutcome::default();
if cli.stdin {
let document = read_stdin(std::io::stdin().lock(), &config.input).into_diagnostic()?;
let context = cli
.stdin_filename
.as_ref()
.map_or_else(AnalysisContext::prose, |path| {
AnalysisContext::for_path(path.clone())
});
if !is_ignored(&ignore, cli.stdin_filename.as_deref()) {
let analysis = detector.analyze_with(&document.content, &context);
print!(
"{}",
render_analysis(cli.output, &document.source, &document.content, &analysis)
.into_diagnostic()?
);
threshold_reached |= analysis.confidence.score() >= failure_threshold;
match read_stdin(std::io::stdin().lock(), &config.input) {
Ok(document) => {
let context = cli
.stdin_filename
.as_ref()
.map_or_else(AnalysisContext::prose, |path| {
AnalysisContext::for_path(path.clone())
});
if !is_ignored(&ignore, cli.stdin_filename.as_deref()) {
let analysis = detector.analyze_with(&document.content, &context);
print!(
"{}",
render_analysis(cli.output, &document.source, &document.content, &analysis)
.into_diagnostic()?
);
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
}
}
Err(error) => {
emit_input_error(cli.output, "<stdin>", &error.to_string())?;
outcome.input_errors = true;
}
}
}
for path in &cli.paths {
let documents = read_inputs(&config, path)
.into_diagnostic()
.wrap_err_with(|| format!("failed to open requested input `{}`", path.display()))?;
let documents = match read_inputs(&config, path) {
Ok(documents) => documents,
Err(error) => {
emit_input_error(cli.output, &path.display().to_string(), &error.to_string())?;
outcome.input_errors = true;
continue;
}
};
for document in documents {
let document = document
.into_diagnostic()
.wrap_err("failed to read an input document")?;
let document = match document {
Ok(document) => document,
Err(error) => {
emit_input_error(cli.output, &path.display().to_string(), &error.to_string())?;
outcome.input_errors = true;
continue;
}
};
if is_ignored(&ignore, source_path(&document.source)) {
continue;
}
@ -214,11 +238,21 @@ fn run(cli: &Cli) -> Result<bool> {
render_analysis(cli.output, &document.source, &document.content, &analysis)
.into_diagnostic()?
);
threshold_reached |= analysis.confidence.score() >= failure_threshold;
outcome.threshold_reached |= analysis.confidence.score() >= failure_threshold;
}
}
Ok(threshold_reached)
Ok(outcome)
}
fn emit_input_error(format: OutputFormat, source: &str, message: &str) -> Result<()> {
let rendered = render_input_error(format, source, message).into_diagnostic()?;
if format == OutputFormat::Json {
print!("{rendered}");
} else {
eprint!("{rendered}");
}
Ok(())
}
const fn fail_threshold(cli: &Cli, config: &Config) -> u8 {

View File

@ -26,7 +26,33 @@ pub(super) fn render_analysis(
match format {
OutputFormat::Human => Ok(render_human(source_id, source, analysis)),
OutputFormat::Concise => Ok(render_concise(source_id, analysis)),
OutputFormat::Json => serde_json::to_string(&JsonAnalysis::new(source_id, analysis)),
OutputFormat::Json => {
serde_json::to_string(&JsonAnalysis::new(source_id, analysis)).map(|mut record| {
record.push('\n');
record
})
}
}
}
pub(super) fn render_input_error(
format: OutputFormat,
source: &str,
message: &str,
) -> Result<String, serde_json::Error> {
match format {
OutputFormat::Human => Ok(format!("{source}: input error: {message}\n")),
OutputFormat::Concise => Ok(format!("{source}: input-error: {message}\n")),
OutputFormat::Json => serde_json::to_string(&JsonInputError {
schema_version: 1,
kind: "input-error",
source,
message,
})
.map(|mut record| {
record.push('\n');
record
}),
}
}
@ -183,6 +209,14 @@ struct JsonSpan {
end: usize,
}
#[derive(Serialize)]
struct JsonInputError<'a> {
schema_version: u8,
kind: &'static str,
source: &'a str,
message: &'a str,
}
#[cfg(test)]
mod tests {
use john_slop::{AnalysisContext, Config, Detector};
@ -225,13 +259,24 @@ mod tests {
&analysis,
)
.unwrap();
let value: serde_json::Value = serde_json::from_str(&output).unwrap();
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
assert_eq!(value["schema_version"], 1);
assert_eq!(value["kind"], "analysis");
assert_eq!(value["source"], "article.md");
assert_eq!(value["findings"][0]["byte_span"]["start"], 8);
assert!(!output.contains('\u{1b}'));
assert!(!output.contains('\n'));
assert_eq!(output.lines().count(), 1);
}
#[test]
fn json_input_errors_are_versioned_line_records() {
let output = render_input_error(OutputFormat::Json, "broken.txt", "invalid UTF-8").unwrap();
let value: serde_json::Value = serde_json::from_str(output.trim_end()).unwrap();
assert_eq!(value["schema_version"], 1);
assert_eq!(value["kind"], "input-error");
assert_eq!(value["source"], "broken.txt");
assert!(!output.contains('\u{1b}'));
}
}

View File

@ -148,7 +148,7 @@ formula are pinned by tests.
diagnostics never contain logs or ANSI escapes.
- [x] Implement documented exit codes: `0` below fail threshold, `1` at/above it,
`2` for usage/configuration/I/O failure. Add `--fail-level`.
- [ ] Report per-document input errors without hiding findings from other
- [x] Report per-document input errors without hiding findings from other
documents; define whether partial failure returns `2` and test it.
- [ ] Add CLI integration tests for spans containing multibyte UTF-8, multiple
files, stdin, missing paths, malformed configs, stable ordering, JSON schema,

View File

@ -93,3 +93,27 @@ fn config_paths_lists_embedded_and_platform_sources() {
assert!(stdout.contains("config.toml"));
assert!(stdout.contains("<not selected>"));
}
#[test]
fn per_document_errors_preserve_other_json_results_and_force_exit_two() {
let directory = tempdir().expect("temp dir");
fs::write(directory.path().join("a-invalid.txt"), [0xff]).unwrap();
fs::write(directory.path().join("b-finding.txt"), "review—this").unwrap();
let output = detector()
.args(["--output", "json"])
.arg(directory.path())
.output()
.expect("run detector");
assert_eq!(output.status.code(), Some(2));
assert!(output.stderr.is_empty());
let records = String::from_utf8(output.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).unwrap())
.collect::<Vec<_>>();
assert_eq!(records.len(), 2);
assert!(records.iter().any(|record| record["kind"] == "input-error"));
assert!(records.iter().any(|record| record["kind"] == "analysis"));
}