605 lines
19 KiB
Rust
605 lines
19 KiB
Rust
use std::{
|
||
fs,
|
||
io::Write,
|
||
process::{Command, Output, Stdio},
|
||
};
|
||
|
||
use tempfile::tempdir;
|
||
|
||
fn detector() -> Command {
|
||
Command::new(env!("CARGO_BIN_EXE_john-slop"))
|
||
}
|
||
|
||
fn detector_with_stdin(arguments: &[&str], input: &[u8]) -> Output {
|
||
let mut child = detector()
|
||
.args(arguments)
|
||
.stdin(Stdio::piped())
|
||
.stdout(Stdio::piped())
|
||
.stderr(Stdio::piped())
|
||
.spawn()
|
||
.expect("spawn detector");
|
||
child
|
||
.stdin
|
||
.take()
|
||
.unwrap()
|
||
.write_all(input)
|
||
.expect("write detector stdin");
|
||
child.wait_with_output().expect("wait for detector")
|
||
}
|
||
|
||
fn json_records(output: &Output) -> Vec<serde_json::Value> {
|
||
String::from_utf8(output.stdout.clone())
|
||
.unwrap()
|
||
.lines()
|
||
.map(|line| serde_json::from_str(line).unwrap())
|
||
.collect()
|
||
}
|
||
|
||
#[test]
|
||
fn help_is_successful_and_names_the_command() {
|
||
let output = detector().arg("--help").output().expect("run detector");
|
||
|
||
assert!(
|
||
output.status.success(),
|
||
"{}",
|
||
String::from_utf8_lossy(&output.stderr)
|
||
);
|
||
insta::assert_snapshot!(String::from_utf8(output.stdout).expect("UTF-8 stdout"));
|
||
}
|
||
|
||
#[test]
|
||
fn config_help_lists_generation_and_inspection_commands() {
|
||
let output = detector()
|
||
.args(["config", "--help"])
|
||
.output()
|
||
.expect("run config help");
|
||
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
|
||
|
||
assert!(output.status.success());
|
||
assert!(stdout.contains("init"));
|
||
assert!(stdout.contains("init-diagnostics"));
|
||
assert!(stdout.contains("show"));
|
||
assert!(stdout.contains("paths"));
|
||
}
|
||
|
||
#[test]
|
||
fn invalid_evidence_score_is_a_usage_error() {
|
||
let output = detector()
|
||
.args(["--fail-score", "101", "file.txt"])
|
||
.output()
|
||
.expect("run detector");
|
||
|
||
assert_eq!(output.status.code(), Some(2));
|
||
insta::assert_snapshot!(String::from_utf8(output.stderr).expect("UTF-8 stderr"));
|
||
}
|
||
|
||
#[test]
|
||
fn default_output_is_colored_graphical_miette_with_a_plain_feature_fallback() {
|
||
let output = detector_with_stdin(&["--stdin"], "suspicious—dash".as_bytes());
|
||
let stdout = String::from_utf8(output.stdout).expect("UTF-8 detector output");
|
||
|
||
assert!(output.status.success());
|
||
assert!(output.stderr.is_empty());
|
||
assert!(stdout.contains("unicode-ascii-replacement"));
|
||
#[cfg(feature = "fancy-diagnostics")]
|
||
{
|
||
assert!(stdout.contains('\u{1b}'));
|
||
assert!(stdout.contains('╭'));
|
||
}
|
||
#[cfg(not(feature = "fancy-diagnostics"))]
|
||
assert!(!stdout.contains('\u{1b}'));
|
||
}
|
||
|
||
#[test]
|
||
fn findings_control_the_threshold_exit_code() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let clean = directory.path().join("clean.txt");
|
||
let suspicious = directory.path().join("suspicious.txt");
|
||
fs::write(&clean, "ordinary-word").expect("write clean fixture");
|
||
fs::write(&suspicious, "suspicious—dash").expect("write suspicious fixture");
|
||
|
||
let clean_output = detector()
|
||
.args(["--fail-score", "10"])
|
||
.arg(&clean)
|
||
.output()
|
||
.expect("run clean detector");
|
||
let suspicious_output = detector()
|
||
.args(["--fail-score", "10"])
|
||
.arg(&suspicious)
|
||
.output()
|
||
.expect("run suspicious detector");
|
||
|
||
assert_eq!(clean_output.status.code(), Some(0));
|
||
assert_eq!(suspicious_output.status.code(), Some(1));
|
||
}
|
||
|
||
#[test]
|
||
fn config_show_emits_resolved_toml_with_explicit_overrides() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let config = directory.path().join("project.toml");
|
||
fs::write(&config, "[thresholds]\nlow = 23\n").expect("write config");
|
||
|
||
let output = detector()
|
||
.env("XDG_CONFIG_HOME", directory.path())
|
||
.args(["config", "show"])
|
||
.arg("--config")
|
||
.arg(&config)
|
||
.output()
|
||
.expect("run config show");
|
||
|
||
assert!(
|
||
output.status.success(),
|
||
"{}",
|
||
String::from_utf8_lossy(&output.stderr)
|
||
);
|
||
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
|
||
let resolved = toml::from_str::<toml::Value>(&stdout).expect("valid TOML");
|
||
assert_eq!(resolved["schema_version"].as_integer(), Some(1));
|
||
assert_eq!(resolved["thresholds"]["low"].as_integer(), Some(23));
|
||
assert_eq!(resolved["replacements"].as_array().unwrap().len(), 1_884);
|
||
}
|
||
|
||
#[test]
|
||
fn config_init_writes_valid_defaults_and_requires_force_to_overwrite() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let config = directory.path().join("config.toml");
|
||
let config_home = directory.path().join("config-home");
|
||
|
||
let created = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "init"])
|
||
.output()
|
||
.expect("generate default config");
|
||
assert!(created.status.success());
|
||
assert_eq!(created.stdout, b"wrote ./config.toml\n");
|
||
let generated = fs::read_to_string(&config).expect("read generated config");
|
||
let parsed = toml::from_str::<toml::Value>(&generated).expect("valid generated TOML");
|
||
assert_eq!(parsed["schema_version"].as_integer(), Some(1));
|
||
assert_eq!(
|
||
parsed["diagnostics"]["categories"]["prompt-leakage"].as_bool(),
|
||
Some(true)
|
||
);
|
||
|
||
let loaded = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "show", "--config", "config.toml"])
|
||
.output()
|
||
.expect("load generated config");
|
||
assert!(
|
||
loaded.status.success(),
|
||
"{}",
|
||
String::from_utf8_lossy(&loaded.stderr)
|
||
);
|
||
|
||
let refused = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "init"])
|
||
.output()
|
||
.expect("refuse overwrite");
|
||
assert_eq!(refused.status.code(), Some(2));
|
||
assert!(String::from_utf8_lossy(&refused.stderr).contains("refusing to overwrite"));
|
||
assert_eq!(fs::read_to_string(&config).unwrap(), generated);
|
||
|
||
fs::write(&config, "not valid TOML").unwrap();
|
||
let forced = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "generate", "--force"])
|
||
.output()
|
||
.expect("force overwrite");
|
||
assert!(forced.status.success());
|
||
assert_eq!(fs::read_to_string(&config).unwrap(), generated);
|
||
}
|
||
|
||
#[test]
|
||
fn diagnostics_init_generates_a_loadable_runtime_catalog() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let catalog = directory.path().join("diagnostics.toml");
|
||
let config_home = directory.path().join("config-home");
|
||
|
||
let created = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "init-diagnostics"])
|
||
.output()
|
||
.expect("generate diagnostic catalog");
|
||
assert!(created.status.success());
|
||
assert_eq!(created.stdout, b"wrote ./diagnostics.toml\n");
|
||
let generated = fs::read_to_string(&catalog).unwrap();
|
||
let parsed = toml::from_str::<toml::Value>(&generated).expect("valid catalog TOML");
|
||
assert_eq!(parsed["schema_version"].as_integer(), Some(1));
|
||
assert_eq!(parsed["rules"].as_array().unwrap().len(), 1);
|
||
|
||
let loaded = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "show", "--diagnostics", "diagnostics.toml"])
|
||
.output()
|
||
.expect("load generated catalog");
|
||
assert!(
|
||
loaded.status.success(),
|
||
"{}",
|
||
String::from_utf8_lossy(&loaded.stderr)
|
||
);
|
||
let resolved = toml::from_slice::<toml::Value>(&loaded.stdout).unwrap();
|
||
assert_eq!(resolved["rules"].as_array().unwrap().len(), 20);
|
||
|
||
let refused = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "init-diagnostics"])
|
||
.output()
|
||
.unwrap();
|
||
assert_eq!(refused.status.code(), Some(2));
|
||
assert_eq!(fs::read_to_string(&catalog).unwrap(), generated);
|
||
|
||
fs::write(&catalog, "invalid").unwrap();
|
||
let forced = detector()
|
||
.current_dir(directory.path())
|
||
.env("XDG_CONFIG_HOME", &config_home)
|
||
.args(["config", "diagnostics-init", "--force"])
|
||
.output()
|
||
.unwrap();
|
||
assert!(forced.status.success());
|
||
assert_eq!(fs::read_to_string(&catalog).unwrap(), generated);
|
||
}
|
||
|
||
#[test]
|
||
fn explicit_runtime_catalog_adds_a_diagnostic_without_recompilation() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let source = directory.path().join("article.md");
|
||
let catalog = directory.path().join("diagnostics.toml");
|
||
fs::write(&source, "REVIEW BEFORE PUBLISHING\n").unwrap();
|
||
fs::write(
|
||
&catalog,
|
||
r#"schema_version = 1
|
||
[[rules]]
|
||
id = "custom-review-marker"
|
||
category = "style"
|
||
severity = "warning"
|
||
matcher = "exact-phrase"
|
||
pattern = "REVIEW BEFORE PUBLISHING"
|
||
weight = 8.0
|
||
message = "a project review marker remains"
|
||
help = "Resolve or remove the marker before publishing."
|
||
occurrence_cap = 1
|
||
enabled = true
|
||
scope = { content = ["prose", "source-code"], extensions = [] }
|
||
"#,
|
||
)
|
||
.unwrap();
|
||
|
||
let output = detector()
|
||
.env(
|
||
"XDG_CONFIG_HOME",
|
||
directory.path().join("empty-config-home"),
|
||
)
|
||
.args(["--output", "json", "--diagnostics"])
|
||
.arg(&catalog)
|
||
.arg(&source)
|
||
.output()
|
||
.expect("run detector with runtime catalog");
|
||
|
||
assert!(output.status.success());
|
||
let records = json_records(&output);
|
||
assert_eq!(records[0]["findings"].as_array().unwrap().len(), 1);
|
||
assert_eq!(records[0]["findings"][0]["rule_id"], "custom-review-marker");
|
||
}
|
||
|
||
#[test]
|
||
fn category_and_individual_configuration_disable_cli_diagnostics() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let source = directory.path().join("article.md");
|
||
fs::write(&source, "As an AI language model, I can help.\n").unwrap();
|
||
|
||
let category_config = directory.path().join("category.toml");
|
||
fs::write(
|
||
&category_config,
|
||
"[diagnostics.categories]\nprompt-leakage = false\n",
|
||
)
|
||
.unwrap();
|
||
let category = detector()
|
||
.env(
|
||
"XDG_CONFIG_HOME",
|
||
directory.path().join("empty-config-home"),
|
||
)
|
||
.args(["--output", "json"])
|
||
.arg("--config")
|
||
.arg(&category_config)
|
||
.arg(&source)
|
||
.output()
|
||
.unwrap();
|
||
assert!(category.status.success());
|
||
assert!(
|
||
json_records(&category)[0]["findings"]
|
||
.as_array()
|
||
.unwrap()
|
||
.is_empty()
|
||
);
|
||
|
||
let rule_config = directory.path().join("rule.toml");
|
||
fs::write(
|
||
&rule_config,
|
||
"[[rules]]\nid = \"assistant-self-identification\"\nenabled = false\n",
|
||
)
|
||
.unwrap();
|
||
let individual = detector()
|
||
.env(
|
||
"XDG_CONFIG_HOME",
|
||
directory.path().join("empty-config-home"),
|
||
)
|
||
.args(["--output", "json"])
|
||
.arg("--config")
|
||
.arg(&rule_config)
|
||
.arg(&source)
|
||
.output()
|
||
.unwrap();
|
||
assert!(individual.status.success());
|
||
assert!(
|
||
json_records(&individual)[0]["findings"]
|
||
.as_array()
|
||
.unwrap()
|
||
.is_empty()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn config_paths_lists_embedded_and_platform_sources() {
|
||
let directory = tempdir().expect("temp dir");
|
||
let output = detector()
|
||
.env("XDG_CONFIG_HOME", directory.path())
|
||
.args(["config", "paths"])
|
||
.output()
|
||
.expect("run config paths");
|
||
|
||
assert!(output.status.success());
|
||
let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout");
|
||
assert!(stdout.contains("<embedded:assets/config.toml>"));
|
||
assert!(stdout.contains("diagnostics.toml"));
|
||
assert!(stdout.contains("explicit_diagnostics"));
|
||
assert!(stdout.contains("config.toml"));
|
||
assert!(stdout.contains("<not selected>"));
|
||
}
|
||
|
||
#[test]
|
||
fn per_document_errors_preserve_other_json_results_and_force_exit_two() {
|
||
let directory = tempdir().expect("temp dir");
|
||
fs::write(directory.path().join("a-invalid.txt"), [0xff]).unwrap();
|
||
fs::write(directory.path().join("b-finding.txt"), "review—this").unwrap();
|
||
|
||
let output = detector()
|
||
.args(["--output", "json"])
|
||
.arg(directory.path())
|
||
.output()
|
||
.expect("run detector");
|
||
|
||
assert_eq!(output.status.code(), Some(2));
|
||
assert!(output.stderr.is_empty());
|
||
let records = json_records(&output);
|
||
assert_eq!(records.len(), 2);
|
||
let error = records
|
||
.iter()
|
||
.find(|record| record["kind"] == "input-error")
|
||
.unwrap();
|
||
assert!(error["source"].as_str().unwrap().ends_with("a-invalid.txt"));
|
||
assert!(records.iter().any(|record| record["kind"] == "analysis"));
|
||
}
|
||
|
||
#[test]
|
||
fn json_spans_are_utf8_bytes_and_schema_is_complete() {
|
||
let output = detector_with_stdin(
|
||
&[
|
||
"--stdin",
|
||
"--stdin-filename",
|
||
"article.md",
|
||
"--output",
|
||
"json",
|
||
],
|
||
"é—".as_bytes(),
|
||
);
|
||
|
||
assert_eq!(output.status.code(), Some(0));
|
||
assert!(output.stderr.is_empty());
|
||
let records = json_records(&output);
|
||
assert_eq!(records.len(), 1);
|
||
let record = &records[0];
|
||
assert_eq!(record["schema_version"], 2);
|
||
assert!(record["summary"]["evidence_score"].is_number());
|
||
assert!(record["summary"].get("confidence_score").is_none());
|
||
assert_eq!(record["kind"], "analysis");
|
||
assert_eq!(record["source"], "<stdin>");
|
||
assert_eq!(record["summary"]["total"], 1);
|
||
assert_eq!(record["findings"][0]["byte_span"]["start"], 2);
|
||
assert_eq!(record["findings"][0]["byte_span"]["end"], 5);
|
||
assert_eq!(record["findings"][0]["matched_text"], "—");
|
||
assert!(record["findings"][0]["help"].is_string());
|
||
assert!(!String::from_utf8_lossy(&output.stdout).contains('\u{1b}'));
|
||
}
|
||
|
||
#[test]
|
||
fn unicode_review_diagnostics_have_safe_fixes_and_exact_spans() {
|
||
let source = "pаss\u{200b}word";
|
||
let output = detector_with_stdin(&["--stdin", "--output", "json"], source.as_bytes());
|
||
|
||
assert!(output.status.success());
|
||
let record = &json_records(&output)[0];
|
||
assert_eq!(record["summary"]["total"], 2);
|
||
assert_eq!(
|
||
record["findings"][0]["rule_id"],
|
||
"unicode-ascii-replacement"
|
||
);
|
||
assert_eq!(record["findings"][0]["matched_text"], "а");
|
||
assert_eq!(record["findings"][0]["fix"]["replacement"], "a");
|
||
assert_eq!(
|
||
record["findings"][1]["rule_id"],
|
||
"unicode-invisible-control"
|
||
);
|
||
assert_eq!(
|
||
&source[record["findings"][1]["byte_span"]["start"]
|
||
.as_u64()
|
||
.unwrap() as usize
|
||
..record["findings"][1]["byte_span"]["end"].as_u64().unwrap() as usize],
|
||
"\u{200b}"
|
||
);
|
||
assert!(record["findings"][1]["fix"].is_null());
|
||
}
|
||
|
||
#[test]
|
||
fn stdin_filename_controls_source_code_scope() {
|
||
let text = b"// TODO: implement the parser";
|
||
let prose = detector_with_stdin(&["--stdin", "--output", "json"], text);
|
||
let rust = detector_with_stdin(
|
||
&[
|
||
"--stdin",
|
||
"--stdin-filename",
|
||
"parser.rs",
|
||
"--output",
|
||
"json",
|
||
],
|
||
text,
|
||
);
|
||
|
||
assert_eq!(json_records(&prose)[0]["summary"]["total"], 0);
|
||
assert_eq!(
|
||
json_records(&rust)[0]["findings"][0]["rule_id"],
|
||
"source-placeholder-todo"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn multiple_files_and_findings_keep_requested_and_span_order() {
|
||
let directory = tempdir().unwrap();
|
||
let first = directory.path().join("z-first.txt");
|
||
let second = directory.path().join("a-second.txt");
|
||
fs::write(&first, "— then As an AI language model").unwrap();
|
||
fs::write(&second, "clean text").unwrap();
|
||
|
||
let output = detector()
|
||
.args(["--output", "json"])
|
||
.arg(&first)
|
||
.arg(&second)
|
||
.output()
|
||
.unwrap();
|
||
let records = json_records(&output);
|
||
|
||
assert_eq!(records.len(), 2);
|
||
assert_eq!(records[0]["source"], first.display().to_string());
|
||
assert_eq!(records[1]["source"], second.display().to_string());
|
||
let findings = records[0]["findings"].as_array().unwrap();
|
||
assert_eq!(findings.len(), 2);
|
||
assert!(
|
||
findings[0]["byte_span"]["start"].as_u64() < findings[1]["byte_span"]["start"].as_u64()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn directory_output_is_sorted_by_path() {
|
||
let directory = tempdir().unwrap();
|
||
let later = directory.path().join("z-later.txt");
|
||
let earlier = directory.path().join("a-earlier.txt");
|
||
fs::write(&later, "later").unwrap();
|
||
fs::write(&earlier, "earlier").unwrap();
|
||
|
||
let output = detector()
|
||
.args(["--output", "json"])
|
||
.arg(directory.path())
|
||
.output()
|
||
.unwrap();
|
||
let records = json_records(&output);
|
||
|
||
assert_eq!(records[0]["source"], earlier.display().to_string());
|
||
assert_eq!(records[1]["source"], later.display().to_string());
|
||
}
|
||
|
||
#[test]
|
||
fn missing_path_does_not_hide_a_later_document() {
|
||
let directory = tempdir().unwrap();
|
||
let missing = directory.path().join("missing.txt");
|
||
let valid = directory.path().join("valid.txt");
|
||
fs::write(&valid, "clean text").unwrap();
|
||
|
||
let output = detector()
|
||
.args(["--output", "json"])
|
||
.arg(&missing)
|
||
.arg(&valid)
|
||
.output()
|
||
.unwrap();
|
||
let records = json_records(&output);
|
||
|
||
assert_eq!(output.status.code(), Some(2));
|
||
assert_eq!(records.len(), 2);
|
||
assert_eq!(records[0]["kind"], "input-error");
|
||
assert_eq!(records[1]["kind"], "analysis");
|
||
}
|
||
|
||
#[test]
|
||
fn malformed_configuration_is_fatal_before_json_output() {
|
||
let directory = tempdir().unwrap();
|
||
let config = directory.path().join("broken.toml");
|
||
let input = directory.path().join("input.txt");
|
||
fs::write(&config, "[thresholds\nlow = 10").unwrap();
|
||
fs::write(&input, "clean text").unwrap();
|
||
|
||
let output = detector()
|
||
.args(["--output", "json", "--config"])
|
||
.arg(&config)
|
||
.arg(&input)
|
||
.output()
|
||
.unwrap();
|
||
|
||
assert_eq!(output.status.code(), Some(2));
|
||
assert!(output.stdout.is_empty());
|
||
assert!(!output.stderr.is_empty());
|
||
}
|
||
|
||
#[test]
|
||
fn fail_level_and_input_errors_cover_all_exit_codes() {
|
||
let directory = tempdir().unwrap();
|
||
let clean = directory.path().join("clean.txt");
|
||
let finding = directory.path().join("finding.txt");
|
||
fs::write(&clean, "specific reviewed text").unwrap();
|
||
fs::write(&finding, "As an AI language model").unwrap();
|
||
|
||
let success = detector().arg(&clean).output().unwrap();
|
||
let threshold = detector()
|
||
.args(["--fail-level", "low"])
|
||
.arg(&finding)
|
||
.output()
|
||
.unwrap();
|
||
let operational = detector()
|
||
.arg(directory.path().join("missing.txt"))
|
||
.output()
|
||
.unwrap();
|
||
|
||
assert_eq!(success.status.code(), Some(0));
|
||
assert_eq!(threshold.status.code(), Some(1));
|
||
assert_eq!(operational.status.code(), Some(2));
|
||
}
|
||
|
||
#[test]
|
||
fn ignore_globs_and_document_limits_apply_to_discovered_inputs() {
|
||
let directory = tempdir().unwrap();
|
||
fs::write(directory.path().join("ignored.txt"), "review—this").unwrap();
|
||
fs::write(directory.path().join("kept.txt"), "short").unwrap();
|
||
|
||
let ignored = detector()
|
||
.args(["--output", "json", "--ignore", "**/ignored.txt"])
|
||
.arg(directory.path())
|
||
.output()
|
||
.unwrap();
|
||
let limited = detector()
|
||
.args(["--output", "json", "--max-document-bytes", "3"])
|
||
.arg(directory.path().join("kept.txt"))
|
||
.output()
|
||
.unwrap();
|
||
|
||
let records = json_records(&ignored);
|
||
assert_eq!(records.len(), 1);
|
||
assert!(records[0]["source"].as_str().unwrap().ends_with("kept.txt"));
|
||
assert_eq!(limited.status.code(), Some(2));
|
||
assert_eq!(json_records(&limited)[0]["kind"], "input-error");
|
||
}
|