john-slop/tests/cli.rs

605 lines
19 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use std::{
fs,
io::Write,
process::{Command, Output, Stdio},
};
use tempfile::tempdir;
fn detector() -> Command {
Command::new(env!("CARGO_BIN_EXE_john-slop"))
}
fn detector_with_stdin(arguments: &[&str], input: &[u8]) -> Output {
let mut child = detector()
.args(arguments)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn detector");
child
.stdin
.take()
.unwrap()
.write_all(input)
.expect("write detector stdin");
child.wait_with_output().expect("wait for detector")
}
fn json_records(output: &Output) -> Vec<serde_json::Value> {
String::from_utf8(output.stdout.clone())
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect()
}
#[test]
fn help_is_successful_and_names_the_command() {
let output = detector().arg("--help").output().expect("run detector");
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
insta::assert_snapshot!(String::from_utf8(output.stdout).expect("UTF-8 stdout"));
}
#[test]
fn config_help_lists_generation_and_inspection_commands() {
let output = detector()
.args(["config", "--help"])
.output()
.expect("run config help");
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
assert!(output.status.success());
assert!(stdout.contains("init"));
assert!(stdout.contains("init-diagnostics"));
assert!(stdout.contains("show"));
assert!(stdout.contains("paths"));
}
#[test]
fn invalid_evidence_score_is_a_usage_error() {
let output = detector()
.args(["--fail-score", "101", "file.txt"])
.output()
.expect("run detector");
assert_eq!(output.status.code(), Some(2));
insta::assert_snapshot!(String::from_utf8(output.stderr).expect("UTF-8 stderr"));
}
#[test]
fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() {
let output = detector_with_stdin(&["--stdin"], "suspicious—dash".as_bytes());
let stdout = String::from_utf8(output.stdout).expect("UTF-8 detector output");
assert!(output.status.success());
assert!(output.stderr.is_empty());
assert!(stdout.contains("unicode-ascii-replacement"));
#[cfg(feature = "fancy-diagnostics")]
{
assert!(stdout.contains('\u{1b}'));
assert!(stdout.contains('╭'));
}
#[cfg(not(feature = "fancy-diagnostics"))]
assert!(!stdout.contains('\u{1b}'));
}
#[test]
fn findings_control_the_threshold_exit_code() {
let directory = tempdir().expect("temp dir");
let clean = directory.path().join("clean.txt");
let suspicious = directory.path().join("suspicious.txt");
fs::write(&clean, "ordinary-word").expect("write clean fixture");
fs::write(&suspicious, "suspicious—dash").expect("write suspicious fixture");
let clean_output = detector()
.args(["--fail-score", "10"])
.arg(&clean)
.output()
.expect("run clean detector");
let suspicious_output = detector()
.args(["--fail-score", "10"])
.arg(&suspicious)
.output()
.expect("run suspicious detector");
assert_eq!(clean_output.status.code(), Some(0));
assert_eq!(suspicious_output.status.code(), Some(1));
}
#[test]
fn config_show_emits_resolved_toml_with_explicit_overrides() {
let directory = tempdir().expect("temp dir");
let config = directory.path().join("project.toml");
fs::write(&config, "[thresholds]\nlow = 23\n").expect("write config");
let output = detector()
.env("XDG_CONFIG_HOME", directory.path())
.args(["config", "show"])
.arg("--config")
.arg(&config)
.output()
.expect("run config show");
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
let resolved = toml::from_str::<toml::Value>(&stdout).expect("valid TOML");
assert_eq!(resolved["schema_version"].as_integer(), Some(1));
assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23));
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_884);
}
#[test]
fn config_init_writes_valid_defaults_and_requires_force_to_overwrite() {
let directory = tempdir().expect("temp dir");
let config = directory.path().join("config.toml");
let config_home = directory.path().join("config-home");
let created = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "init"])
.output()
.expect("generate default config");
assert!(created.status.success());
assert_eq!(created.stdout, b"wrote ./config.toml\n");
let generated = fs::read_to_string(&config).expect("read generated config");
let parsed = toml::from_str::<toml::Value>(&generated).expect("valid generated TOML");
assert_eq!(parsed["schema_version"].as_integer(), Some(1));
assert_eq!(
parsed["diagnostics"]["categories"]["prompt-leakage"].as_bool(),
Some(true)
);
let loaded = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "show", "--config", "config.toml"])
.output()
.expect("load generated config");
assert!(
loaded.status.success(),
"{}",
String::from_utf8_lossy(&loaded.stderr)
);
let refused = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "init"])
.output()
.expect("refuse overwrite");
assert_eq!(refused.status.code(), Some(2));
assert!(String::from_utf8_lossy(&refused.stderr).contains("refusing to overwrite"));
assert_eq!(fs::read_to_string(&config).unwrap(), generated);
fs::write(&config, "not valid TOML").unwrap();
let forced = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "generate", "--force"])
.output()
.expect("force overwrite");
assert!(forced.status.success());
assert_eq!(fs::read_to_string(&config).unwrap(), generated);
}
#[test]
fn diagnostics_init_generates_a_loadable_runtime_catalog() {
let directory = tempdir().expect("temp dir");
let catalog = directory.path().join("diagnostics.toml");
let config_home = directory.path().join("config-home");
let created = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "init-diagnostics"])
.output()
.expect("generate diagnostic catalog");
assert!(created.status.success());
assert_eq!(created.stdout, b"wrote ./diagnostics.toml\n");
let generated = fs::read_to_string(&catalog).unwrap();
let parsed = toml::from_str::<toml::Value>(&generated).expect("valid catalog TOML");
assert_eq!(parsed["schema_version"].as_integer(), Some(1));
assert_eq!(parsed["rules"].as_array().unwrap().len(), 1);
let loaded = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "show", "--diagnostics", "diagnostics.toml"])
.output()
.expect("load generated catalog");
assert!(
loaded.status.success(),
"{}",
String::from_utf8_lossy(&loaded.stderr)
);
let resolved = toml::from_slice::<toml::Value>(&loaded.stdout).unwrap();
assert_eq!(resolved["rules"].as_array().unwrap().len(), 20);
let refused = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "init-diagnostics"])
.output()
.unwrap();
assert_eq!(refused.status.code(), Some(2));
assert_eq!(fs::read_to_string(&catalog).unwrap(), generated);
fs::write(&catalog, "invalid").unwrap();
let forced = detector()
.current_dir(directory.path())
.env("XDG_CONFIG_HOME", &config_home)
.args(["config", "diagnostics-init", "--force"])
.output()
.unwrap();
assert!(forced.status.success());
assert_eq!(fs::read_to_string(&catalog).unwrap(), generated);
}
#[test]
fn explicit_runtime_catalog_adds_a_diagnostic_without_recompilation() {
let directory = tempdir().expect("temp dir");
let source = directory.path().join("article.md");
let catalog = directory.path().join("diagnostics.toml");
fs::write(&source, "REVIEW BEFORE PUBLISHING\n").unwrap();
fs::write(
&catalog,
r#"schema_version = 1
[[rules]]
id = "custom-review-marker"
category = "style"
severity = "warning"
matcher = "exact-phrase"
pattern = "REVIEW BEFORE PUBLISHING"
weight = 8.0
message = "a project review marker remains"
help = "Resolve or remove the marker before publishing."
occurrence_cap = 1
enabled = true
scope = { content = ["prose", "source-code"], extensions = [] }
"#,
)
.unwrap();
let output = detector()
.env(
"XDG_CONFIG_HOME",
directory.path().join("empty-config-home"),
)
.args(["--output", "json", "--diagnostics"])
.arg(&catalog)
.arg(&source)
.output()
.expect("run detector with runtime catalog");
assert!(output.status.success());
let records = json_records(&output);
assert_eq!(records[0]["findings"].as_array().unwrap().len(), 1);
assert_eq!(records[0]["findings"][0]["rule_id"], "custom-review-marker");
}
#[test]
fn category_and_individual_configuration_disable_cli_diagnostics() {
let directory = tempdir().expect("temp dir");
let source = directory.path().join("article.md");
fs::write(&source, "As an AI language model, I can help.\n").unwrap();
let category_config = directory.path().join("category.toml");
fs::write(
&category_config,
"[diagnostics.categories]\nprompt-leakage = false\n",
)
.unwrap();
let category = detector()
.env(
"XDG_CONFIG_HOME",
directory.path().join("empty-config-home"),
)
.args(["--output", "json"])
.arg("--config")
.arg(&category_config)
.arg(&source)
.output()
.unwrap();
assert!(category.status.success());
assert!(
json_records(&category)[0]["findings"]
.as_array()
.unwrap()
.is_empty()
);
let rule_config = directory.path().join("rule.toml");
fs::write(
&rule_config,
"[[rules]]\nid = \"assistant-self-identification\"\nenabled = false\n",
)
.unwrap();
let individual = detector()
.env(
"XDG_CONFIG_HOME",
directory.path().join("empty-config-home"),
)
.args(["--output", "json"])
.arg("--config")
.arg(&rule_config)
.arg(&source)
.output()
.unwrap();
assert!(individual.status.success());
assert!(
json_records(&individual)[0]["findings"]
.as_array()
.unwrap()
.is_empty()
);
}
#[test]
fn config_paths_lists_embedded_and_platform_sources() {
let directory = tempdir().expect("temp dir");
let output = detector()
.env("XDG_CONFIG_HOME", directory.path())
.args(["config", "paths"])
.output()
.expect("run config paths");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
assert!(stdout.contains("<embedded:assets/config.toml>"));
assert!(stdout.contains("diagnostics.toml"));
assert!(stdout.contains("explicit_diagnostics"));
assert!(stdout.contains("config.toml"));
assert!(stdout.contains("<not selected>"));
}
#[test]
fn per_document_errors_preserve_other_json_results_and_force_exit_two() {
let directory = tempdir().expect("temp dir");
fs::write(directory.path().join("a-invalid.txt"), [0xff]).unwrap();
fs::write(directory.path().join("b-finding.txt"), "review—this").unwrap();
let output = detector()
.args(["--output", "json"])
.arg(directory.path())
.output()
.expect("run detector");
assert_eq!(output.status.code(), Some(2));
assert!(output.stderr.is_empty());
let records = json_records(&output);
assert_eq!(records.len(), 2);
let error = records
.iter()
.find(|record| record["kind"] == "input-error")
.unwrap();
assert!(error["source"].as_str().unwrap().ends_with("a-invalid.txt"));
assert!(records.iter().any(|record| record["kind"] == "analysis"));
}
#[test]
fn json_spans_are_utf8_bytes_and_schema_is_complete() {
let output = detector_with_stdin(
&[
"--stdin",
"--stdin-filename",
"article.md",
"--output",
"json",
],
"é—".as_bytes(),
);
assert_eq!(output.status.code(), Some(0));
assert!(output.stderr.is_empty());
let records = json_records(&output);
assert_eq!(records.len(), 1);
let record = &records[0];
assert_eq!(record["schema_version"], 2);
assert!(record["summary"]["evidence_score"].is_number());
assert!(record["summary"].get("confidence_score").is_none());
assert_eq!(record["kind"], "analysis");
assert_eq!(record["source"], "<stdin>");
assert_eq!(record["summary"]["total"], 1);
assert_eq!(record["findings"][0]["byte_span"]["start"], 2);
assert_eq!(record["findings"][0]["byte_span"]["end"], 5);
assert_eq!(record["findings"][0]["matched_text"], "—");
assert!(record["findings"][0]["help"].is_string());
assert!(!String::from_utf8_lossy(&output.stdout).contains('\u{1b}'));
}
#[test]
fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() {
let source = "pаss\u{200b}word";
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
assert!(output.status.success());
let record = &json_records(&output)[0];
assert_eq!(record["summary"]["total"], 2);
assert_eq!(
record["findings"][0]["rule_id"],
"unicode-ascii-replacement"
);
assert_eq!(record["findings"][0]["matched_text"], "а");
assert_eq!(record["findings"][0]["fix"]["replacement"], "a");
assert_eq!(
record["findings"][1]["rule_id"],
"unicode-invisible-control"
);
assert_eq!(
&source[record["findings"][1]["byte_span"]["start"]
.as_u64()
.unwrap() as usize
..record["findings"][1]["byte_span"]["end"].as_u64().unwrap() as usize],
"\u{200b}"
);
assert!(record["findings"][1]["fix"].is_null());
}
#[test]
fn stdin_filename_controls_source_code_scope() {
let text = b"// TODO: implement the parser";
let prose = detector_with_stdin(&["--stdin", "--output", "json"], text);
let rust = detector_with_stdin(
&[
"--stdin",
"--stdin-filename",
"parser.rs",
"--output",
"json",
],
text,
);
assert_eq!(json_records(&prose)[0]["summary"]["total"], 0);
assert_eq!(
json_records(&rust)[0]["findings"][0]["rule_id"],
"source-placeholder-todo"
);
}
#[test]
fn multiple_files_and_findings_keep_requested_and_span_order() {
let directory = tempdir().unwrap();
let first = directory.path().join("z-first.txt");
let second = directory.path().join("a-second.txt");
fs::write(&first, "— then As an AI language model").unwrap();
fs::write(&second, "clean text").unwrap();
let output = detector()
.args(["--output", "json"])
.arg(&first)
.arg(&second)
.output()
.unwrap();
let records = json_records(&output);
assert_eq!(records.len(), 2);
assert_eq!(records[0]["source"], first.display().to_string());
assert_eq!(records[1]["source"], second.display().to_string());
let findings = records[0]["findings"].as_array().unwrap();
assert_eq!(findings.len(), 2);
assert!(
findings[0]["byte_span"]["start"].as_u64() < findings[1]["byte_span"]["start"].as_u64()
);
}
#[test]
fn directory_output_is_sorted_by_path() {
let directory = tempdir().unwrap();
let later = directory.path().join("z-later.txt");
let earlier = directory.path().join("a-earlier.txt");
fs::write(&later, "later").unwrap();
fs::write(&earlier, "earlier").unwrap();
let output = detector()
.args(["--output", "json"])
.arg(directory.path())
.output()
.unwrap();
let records = json_records(&output);
assert_eq!(records[0]["source"], earlier.display().to_string());
assert_eq!(records[1]["source"], later.display().to_string());
}
#[test]
fn missing_path_does_not_hide_a_later_document() {
let directory = tempdir().unwrap();
let missing = directory.path().join("missing.txt");
let valid = directory.path().join("valid.txt");
fs::write(&valid, "clean text").unwrap();
let output = detector()
.args(["--output", "json"])
.arg(&missing)
.arg(&valid)
.output()
.unwrap();
let records = json_records(&output);
assert_eq!(output.status.code(), Some(2));
assert_eq!(records.len(), 2);
assert_eq!(records[0]["kind"], "input-error");
assert_eq!(records[1]["kind"], "analysis");
}
#[test]
fn malformed_configuration_is_fatal_before_json_output() {
let directory = tempdir().unwrap();
let config = directory.path().join("broken.toml");
let input = directory.path().join("input.txt");
fs::write(&config, "[thresholds\nlow = 10").unwrap();
fs::write(&input, "clean text").unwrap();
let output = detector()
.args(["--output", "json", "--config"])
.arg(&config)
.arg(&input)
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(output.stdout.is_empty());
assert!(!output.stderr.is_empty());
}
#[test]
fn fail_level_and_input_errors_cover_all_exit_codes() {
let directory = tempdir().unwrap();
let clean = directory.path().join("clean.txt");
let finding = directory.path().join("finding.txt");
fs::write(&clean, "specific reviewed text").unwrap();
fs::write(&finding, "As an AI language model").unwrap();
let success = detector().arg(&clean).output().unwrap();
let threshold = detector()
.args(["--fail-level", "low"])
.arg(&finding)
.output()
.unwrap();
let operational = detector()
.arg(directory.path().join("missing.txt"))
.output()
.unwrap();
assert_eq!(success.status.code(), Some(0));
assert_eq!(threshold.status.code(), Some(1));
assert_eq!(operational.status.code(), Some(2));
}
#[test]
fn ignore_globs_and_document_limits_apply_to_discovered_inputs() {
let directory = tempdir().unwrap();
fs::write(directory.path().join("ignored.txt"), "review—this").unwrap();
fs::write(directory.path().join("kept.txt"), "short").unwrap();
let ignored = detector()
.args(["--output", "json", "--ignore", "**/ignored.txt"])
.arg(directory.path())
.output()
.unwrap();
let limited = detector()
.args(["--output", "json", "--max-document-bytes", "3"])
.arg(directory.path().join("kept.txt"))
.output()
.unwrap();
let records = json_records(&ignored);
assert_eq!(records.len(), 1);
assert!(records[0]["source"].as_str().unwrap().ends_with("kept.txt"));
assert_eq!(limited.status.code(), Some(2));
assert_eq!(json_records(&limited)[0]["kind"], "input-error");
}