diff --git a/README.md b/README.md index 02740d7..36dac38 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,41 @@ -# JohnSlop +# John Slop -Detect AI slop markers +John Slop is an early-stage, local-first heuristic linter for concrete signals +of low-effort or insufficiently reviewed text and source code. It is intended to +highlight evidence for human review, not to prove that content was written by AI +or assign an authorship probability. +## Current prototype -# Links -* [https://en.wikipedia.org/wiki/Wikipedia%3ASigns_of_AI_writing] -* [https://en.wikipedia.org/wiki/Wikipedia:WikiProject_AI_Cleanup] -* [https://www.theverge.com/report/756810/wikipedia-ai-slop-policies-community-speedy-deletion] +The repository currently contains proof-of-concept code that: + +- reads UTF-8 text from a file or recursively from a directory; +- reads text entries from ZIP-based files, including `.zip`, `.docx`, `.xlsx`, + and `.pptx`; +- rejects missing paths and invalid UTF-8 with a diagnostic error; +- detects a configured-in-code set of hyphen-like Unicode characters; and +- demonstrates source-labelled terminal diagnostics with `miette`. + +The implementation is being restructured into a reusable library plus detector, +formatter, and LSP binaries. See [tasks.md](tasks.md) for the ordered backlog and +[AGENTS.md](AGENTS.md) for the design and contribution rules. + +## Development + +Run the current test and quality suite with: + +```sh +cargo fmt --all -- --check +cargo clippy --all-targets --all-features -- -D warnings +cargo test --all-targets --all-features +cargo doc --no-deps +``` + +Behavior is intentionally captured with regression tests before the refactor so +that later changes are deliberate and reviewable. + +## Background reading + +- [Wikipedia: Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia%3ASigns_of_AI_writing) +- [Wikipedia: WikiProject AI Cleanup](https://en.wikipedia.org/wiki/Wikipedia:WikiProject_AI_Cleanup) +- [The Verge: Wikipedia editors are battling AI slop](https://www.theverge.com/report/756810/wikipedia-ai-slop-policies-community-speedy-deletion) diff --git a/src/errors.rs b/src/errors.rs index 8c5373d..bab0ff5 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -90,6 +90,7 @@ pub enum FormattingError { InconsistentListStyle, } +/// Print the proof-of-concept formatting diagnostic used by the current binary. pub fn try_report() { let a: miette::Report = FormattingError::StrangeUTFDetected { character: "this sentence is first ocntext\nnew line with ยป\nboibob\n\nbob".into(), diff --git a/src/reader.rs b/src/reader.rs index 48032ea..f702b6f 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -39,7 +39,9 @@ use zip::ZipArchive; /// reader instead of the whole string. #[derive(Debug, Clone)] pub struct FileContent { + /// Path of the source file or archive entry. pub path: PathBuf, + /// Complete UTF-8 text read from the source. pub content: String, } @@ -49,8 +51,11 @@ pub struct FileContent { /// simple โ€“ it holds all file descriptors in a vector and yields them one by /// one. This makes it easy to use in a `for` loop while still keeping the /// implementation short. +#[derive(Debug)] pub struct FileDescriptorIterator { + /// Documents that have already been loaded by the proof-of-concept reader. files: Vec, + /// Index of the next document to yield. index: usize, } @@ -98,7 +103,7 @@ impl Iterator for FileDescriptorIterator { /// /// Errors are wrapped with `miette` so that they can be printed nicely to the /// console. -pub fn universal_read(file_path: &PathBuf) -> Result { +pub fn universal_read(file_path: &Path) -> Result { if file_path.is_file() { // If the file looks like a zip we handle it specially if is_zip(file_path) { @@ -109,7 +114,7 @@ pub fn universal_read(file_path: &PathBuf) -> Result { // Plain file โ€“ just read it let content = read_file(file_path)?; return Ok(FileDescriptorIterator::new(vec![FileContent { - path: file_path.clone(), + path: file_path.to_path_buf(), content, }])); } diff --git a/src/reader/tests.rs b/src/reader/tests.rs index 82aa4bc..1bcbc6b 100644 --- a/src/reader/tests.rs +++ b/src/reader/tests.rs @@ -36,7 +36,7 @@ fn test_read_single_file() { let file_path = dir.path().join("test.txt"); write_file(&file_path, "line1\nline2"); - let it = universal_read(&file_path.to_path_buf()).expect("universal_read"); + let it = universal_read(&file_path).expect("universal_read"); let mut collected = Vec::new(); for r in it { collected.push(r.expect("file read")); @@ -46,6 +46,42 @@ fn test_read_single_file() { assert_eq!(collected[0].content, "line1\nline2"); } +#[test] +fn test_read_file_rejects_invalid_utf8() { + let dir = tempdir().expect("temp dir"); + let file_path = dir.path().join("invalid.txt"); + fs::write(&file_path, [0xff, 0xfe, 0xfd]).expect("write invalid UTF-8 fixture"); + + let error = read_file(&file_path).expect_err("invalid UTF-8 should fail"); + + assert!( + error.to_string().contains("Failed to read"), + "unexpected error: {error}" + ); + assert!( + error + .to_string() + .contains("stream did not contain valid UTF-8"), + "unexpected error: {error}" + ); +} + +#[test] +fn test_universal_read_rejects_missing_path() { + let dir = tempdir().expect("temp dir"); + let missing_path = dir.path().join("missing.txt"); + + let error = universal_read(&missing_path).expect_err("missing path should fail"); + + assert_eq!( + error.to_string(), + format!( + "The path `{}` is neither a file, a directory nor a recognised zip archive", + missing_path.display() + ) + ); +} + #[test] fn test_read_folder() { let dir = tempdir().expect("temp dir"); @@ -55,7 +91,7 @@ fn test_read_folder() { write_file(&dir.path().join("a.txt"), "a"); write_file(&sub.join("b.txt"), "b"); - let it = universal_read(&dir.path().to_path_buf()).expect("universal_read"); + let it = universal_read(dir.path()).expect("universal_read"); let mut paths: Vec = it.map(|r| r.expect("file read").path).collect(); paths.sort(); assert_eq!(paths, vec![dir.path().join("a.txt"), sub.join("b.txt")]); @@ -67,7 +103,7 @@ fn test_read_zip() { let zip_path = dir.path().join("archive.zip"); create_zip(&zip_path).expect("create zip"); - let it = universal_read(&zip_path.to_path_buf()).expect("universal_read"); + let it = universal_read(&zip_path).expect("universal_read"); let mut entries: Vec = it.map(|r| r.expect("file read").content).collect(); entries.sort(); assert_eq!( diff --git a/tasks.md b/tasks.md index 08d3186..bd3ebce 100644 --- a/tasks.md +++ b/tasks.md @@ -23,7 +23,7 @@ audit in CI. ## Milestone 0 โ€” Baseline and decisions -- [ ] Capture current behavior with tests before moving code: plain file, +- [x] Capture current behavior with tests before moving code: plain file, recursive directory, ZIP member, invalid UTF-8, missing path, and basic Unicode matching. - [ ] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme