From 27a8494dc7c53fcd1a53835eee06a730476f4e49 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Thu, 4 Dec 2025 17:27:50 +0100 Subject: [PATCH] Shitcoded some base functions, too tired to do any useful shit --- Cargo.toml | 7 ++ src/detector.rs | 92 +++++++++++++++++ src/errors.rs | 100 ++++++++++++++++++ src/main.rs | 30 +++--- src/reader.rs | 246 ++++++++++++++++++++++++++++++++++++++++++++ src/reader/tests.rs | 89 ++++++++++++++++ 6 files changed, 551 insertions(+), 13 deletions(-) create mode 100644 src/detector.rs create mode 100644 src/errors.rs create mode 100644 src/reader.rs create mode 100644 src/reader/tests.rs diff --git a/Cargo.toml b/Cargo.toml index 48e6313..1f62092 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,4 +5,11 @@ edition = "2024" [dependencies] miette = { version = "7.6.0", features = ["fancy"] } +once_cell = "1.21.3" +rayon = "1.11.0" +regex = "1.12.2" +tempfile = "3.23.0" thiserror = "2.0.17" +unicode-segmentation = "1.12.0" +walkdir = "2.5.0" +zip = "6.0.0" diff --git a/src/detector.rs b/src/detector.rs new file mode 100644 index 0000000..68e4e83 --- /dev/null +++ b/src/detector.rs @@ -0,0 +1,92 @@ +//! All detection function & modules +//! +use once_cell::sync::Lazy; +use regex::{Regex, RegexBuilder}; + +/// A tiny “search‑engine” that knows a *set* of Unicode code‑points. +/// It returns the **byte** offset of every match (so the positions are +/// deterministic even if you run the search in parallel). +static HYPHEN_REGEX: Lazy = Lazy::new(|| { + // All the Unicode characters that look like a hyphen. + // (U+002D, U+2010, U+2011, U+2012, U+2013, U+2014, U+2015, …) + let pattern = r"[\u002D\u2010\u2011\u2012\u2013\u2014\u2015\u2212\uFE58\uFE63\uFF0D]"; + RegexBuilder::new(pattern) + .unicode(true) // enable Unicode semantics + .build() + .expect("regex build failed") +}); + +/// Find *all* occurrences of the characters in `re` inside `text`. +/// Returns a vector of `(byte_start, byte_end, matched_char)` tuples. +fn find_all(re: &Regex, text: &str) -> Vec<(usize, usize, char)> { + re.find_iter(text) + .map(|m| (m.start(), m.end(), text[m.range()].chars().next().unwrap())) + .collect() +} + +// If you have many different “special” characters you can keep a +// `Vec<(Regex, &'static str)>` and run the search in parallel with `rayon`. + +#[cfg(test)] +mod tests { + use super::*; + use rayon::prelude::*; + use unicode_segmentation::UnicodeSegmentation; + + #[test] + fn it_finds_all_hyphens() { + let s = "a‑b—c‑—d‑"; + // The string contains the following code points: + // U+2011 (NON‑BREAKING HYPHEN) -> a‑b + // U+2014 (EM DASH) -> —c + // U+2011 again -> d‑ + // U+2014 again -> — + let mut results = find_all(&HYPHEN_REGEX, s); + + // sort to make the assertion deterministic + results.sort_by_key(|r| r.0); + assert_eq!( + results, + vec![ + (1, 4, '‑'), + (5, 8, '—'), + (9, 12, '‑'), + (12, 15, '—'), + (16, 19, '‑') + ] + ); + } + + #[test] + fn it_is_thread_safe_and_can_run_in_parallel() { + // A big dummy string that contains a bunch of hyphen variants. + let s = (0..10_000) + .map(|i| if i % 3 == 0 { "—" } else { "-" }) + .collect::(); + + // Pretend we have two different regexes to search (just one here). + let regexes = vec![&*HYPHEN_REGEX]; + + // Parallel search (one thread per regex – not the fastest, but + // shows the “share‑regex‑between‑threads” pattern). + let results: Vec> = regexes.par_iter().map(|re| find_all(re, &s)).collect(); + + // All regexes returned the same count. + assert_eq!(results[0].len(), 10_000); + } + + #[test] + fn grapheme_positions_are_correct() { + // A string with a single grapheme cluster that is two bytes in UTF‑8. + let s = "a𝌆b"; // 𝌆 = U+1D306, 3 bytes + + // Find the middle grapheme (the diamond). + let diamond_start = s.grapheme_indices(true).nth(1).unwrap().0; + assert_eq!(diamond_start, 1); // the byte offset of the start of 𝌆 + + // Find it with our regex (if we had a regex that matches 𝌆). + let diamond_re = Regex::new(r"\u{1D306}").unwrap(); + let m = diamond_re.find(s).unwrap(); + assert_eq!(m.start(), diamond_start); + } +} diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..8c5373d --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,100 @@ +//! # AI‑Detection Errors +//! +//! This module contains the error types used by the AI‑writing detection +//! logic. The high‑level `AiDetectionError` groups the *four main +//! “signs of AI‑writing”* that editors look for on Wikipedia. Each +//! variant may carry a more detailed sub‑error – e.g. the `FormattingClue` +//! variant contains a `FormattingError` that describes the exact +//! formatting problem that was spotted. +//! +//! The design aims for: +//! * **Type safety** – callers can match on the category instead of parsing +//! a string. +//! * **Extensibility** – new sub‑errors can be added without touching +//! the public API. +//! * **Human‑readability** – `thiserror::Error` automatically provides a +//! clear message for logging or UI display. +//! +use miette::{Diagnostic, SourceSpan}; +use thiserror::Error; + +/// Errors that signal a suspected AI‑generated passage. +/// +/// Each variant maps to one of the four “signs of AI‑writing” categories +#[derive(Error, Debug)] +pub enum AiDetectionError { + /// The text shows common stylistic patterns that LLMs tend to produce. + #[error("AI‑generated text: stylistic patterns detected")] + StylisticPattern, + + /// Formatting idiosyncrasies (e.g., odd punctuation, inconsistent heading styles) + /// that point to an automated draft. + #[error("AI‑generated text: formatting clues detected")] + FormattingClue(#[from] FormattingError), + + /// Content‑related quirks such as hallucinations, fabricated facts, + /// or other substantive inconsistencies. + #[error("AI‑generated text: content‑related quirks detected")] + ContentQuirk, + + /// Issues with context or nuance—e.g., the passage feels out of place + /// within the surrounding article or uses language that the editor knows + /// is atypical for human writers. + #[error("AI‑generated text: context & nuance issues detected")] + ContextNuance, +} + +/// Sub‑errors that describe the specific formatting problem +/// that was found in a suspected AI passage. +#[derive(Error, Debug, Diagnostic)] +pub enum FormattingError { + /// The passage contains Unicode code points that are not normally + /// used in articles (e.g. zero‑width joiners, control + /// characters, or characters from a different writing system). + #[error("unexpected or strange Unicode detected ({character})")] + StrangeUTFDetected { + /// The offending character + #[source_code] + character: String, + /// Source range + #[label("Pb")] + span: SourceSpan, + }, + + /// The passage contains an overuse of emoticons or emoji, which + /// is highly unlikely in a document. + #[error("overuse of emoticons detected {number} emoticons over {range} characters")] + OveruseOfEmoticons { + /// List of all offending emoticons + emoticons: Vec, + /// Number of offenders + number: usize, + /// Offending range + range: usize, + }, + + /// Markdown syntax (`*`, `#`, `>`, etc.) appears in a place where + /// a document normally contains plain prose, indicating an + /// automated template was not cleaned up. + #[error("markdown syntax found in an unexpected location")] + MarkdownInUnusualPlace, + + /// A bullet list is broken by a paragraph or a line break that + /// does not belong to the list. + #[error("list contains unexpected line breaks")] + UnexpectedListBreak, + + /// The article uses a different list style (e.g. Roman numerals + /// instead of Arabic digits) in the middle of a paragraph. + #[error("inconsistent list style detected")] + InconsistentListStyle, +} + +pub fn try_report() { + let a: miette::Report = FormattingError::StrangeUTFDetected { + character: "this sentence is first ocntext\nnew line with »\nboibob\n\nbob".into(), + span: (45..46).into(), + } + .into(); + println!("{:?}", a); +} diff --git a/src/main.rs b/src/main.rs index 6035e11..339a037 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,15 @@ -// ────────────────────────────────────────────────────────────────────── -// Cargo.toml -// ────────────────────────────────────────────────────────────────────── -// [dependencies] -// miette = "7" -// ────────────────────────────────────────────────────────────────────── - +//!Fill this doc +//! +#![warn(clippy::missing_docs_in_private_items)] +#![allow(dead_code)] use miette::{LabeledSpan, miette}; +use crate::errors::try_report; + +mod detector; +mod errors; +mod reader; + /// Creates a diagnostic for the ambiguous Unicode character U+2011 /// (non‑breaking hyphen) that can be confused with the ordinary hyphen U+002D. fn unicode_confusion_report() -> miette::Report { @@ -51,13 +54,14 @@ fn ai_generated_report() -> miette::Report { } fn main() -> Result<(), Box> { - // Show the Unicode‑confusion diagnostic - let report1 = unicode_confusion_report(); - println!("{:?}\n", report1); + // // Show the Unicode‑confusion diagnostic + // let report1 = unicode_confusion_report(); + // println!("{:?}\n", report1); - // Show the AI‑generated‑text diagnostic - let report2 = ai_generated_report(); - println!("{:?}", report2); + // // Show the AI‑generated‑text diagnostic + // let report2 = ai_generated_report(); + // println!("{:?}", report2); + try_report(); Ok(()) } diff --git a/src/reader.rs b/src/reader.rs new file mode 100644 index 0000000..48032ea --- /dev/null +++ b/src/reader.rs @@ -0,0 +1,246 @@ +//! # Multimode reader +//! +//! This module implements a *very* small proof‑of‑concept file reader that can handle +//! individual files, directories and simple ZIP‑based archives (e.g. DOCX, XLSX, +//! …). The public API is intentionally minimal – just enough to +//! demonstrate how you can read a file line‑by‑line or perform a +//! search‑&‑replace operation. +//! +//! The key types are: +//! * `FileDescriptorIterator` – an iterator that yields a `FileContent` for every +//! readable file found under a given path. +//! * `FileContent` – a tiny wrapper that keeps the file’s path and its +//! entire textual content. +//! +//! The helper functions below give a quick way to read a single file, walk a +//! directory or unpack a ZIP archive. All functions return `miette::Result` so +//! that any error can be displayed in a user‑friendly way. + +#[cfg(test)] +mod tests; + +use std::{ + fs::File, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +use miette::{Result, miette}; +use walkdir::WalkDir; +use zip::ZipArchive; + +/// Information about a single file. +/// +/// * `path` – the absolute (or relative) path to the file. +/// * `content` – the file’s content as a UTF‑8 string. +/// +/// The struct is intentionally lightweight; for the POC we only store the +/// contents as a `String`. In a real‑world library you might want to keep a +/// reader instead of the whole string. +#[derive(Debug, Clone)] +pub struct FileContent { + pub path: PathBuf, + pub content: String, +} + +/// Iterator over files found under a path. +/// +/// The iterator will return one `FileContent` per file. It is intentionally +/// simple – it holds all file descriptors in a vector and yields them one by +/// one. This makes it easy to use in a `for` loop while still keeping the +/// implementation short. +pub struct FileDescriptorIterator { + files: Vec, + index: usize, +} + +impl FileDescriptorIterator { + /// Create a new iterator from a vector of `FileContent`s. + pub fn new(files: Vec) -> Self { + Self { files, index: 0 } + } + + /// Return the number of files that will be yielded. + pub fn len(&self) -> usize { + self.files.len() + } + + /// Return `true` if the iterator has no more items. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Iterator for FileDescriptorIterator { + type Item = Result; + + fn next(&mut self) -> Option { + if self.index >= self.files.len() { + return None; + } + let res = self.files[self.index].clone(); + self.index += 1; + Some(Ok(res)) + } +} + +/// Universal reader. +/// +/// * `file_path` – a path that can point to a file, a directory or a ZIP +/// archive. +/// * Returns a `FileDescriptorIterator` that yields the files contained in the +/// input. If the input is a plain file, the iterator will contain a single +/// element. +/// +/// The function recognises ZIP files by their extension (`.zip`, `.docx`, +/// `.xlsx`, …). If the file is not a recognised ZIP format it is treated as +/// a plain file. +/// +/// Errors are wrapped with `miette` so that they can be printed nicely to the +/// console. +pub fn universal_read(file_path: &PathBuf) -> Result { + if file_path.is_file() { + // If the file looks like a zip we handle it specially + if is_zip(file_path) { + let zip_files = read_zip(file_path)?; + return Ok(FileDescriptorIterator::new(zip_files)); + } + + // Plain file – just read it + let content = read_file(file_path)?; + return Ok(FileDescriptorIterator::new(vec![FileContent { + path: file_path.clone(), + content, + }])); + } + + if file_path.is_dir() { + let dir_files = read_folder(file_path)?; + return Ok(FileDescriptorIterator::new(dir_files)); + } + + Err(miette!( + "The path `{}` is neither a file, a directory nor a recognised zip archive", + file_path.display() + )) +} + +/// Read a single file into a `String`. +/// +/// The file is opened and its content is read as UTF‑8. If the file contains +/// non‑UTF‑8 data an error is returned. +pub fn read_file(file_path: &Path) -> Result { + let mut file = File::open(file_path) + .map_err(|e| miette!("Failed to open `{}`: {}", file_path.display(), e))?; + let mut content = String::new(); + file.read_to_string(&mut content) + .map_err(|e| miette!("Failed to read `{}`: {}", file_path.display(), e))?; + Ok(content) +} + +/// Recursively walk a directory and read all regular files. +/// +/// The function returns a `Vec` where each element contains the +/// path and the full textual content of a file. Hidden files and directories +/// are *not* filtered out – they are processed like any other entry. +pub fn read_folder(dir_path: &Path) -> Result> { + let mut files = Vec::new(); + + for entry in WalkDir::new(dir_path) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + { + let path = entry.path().to_path_buf(); + let content = read_file(&path)?; + files.push(FileContent { path, content }); + } + + Ok(files) +} + +/// Return `true` if the file has an extension that we treat as a ZIP archive. +/// For simplicity we just check the first part of the extension (e.g. “zip” +/// for “docx”). +fn is_zip(path: &Path) -> bool { + path.extension() + .and_then(|s| s.to_str()) + .map(|ext| { + let ext_lower = ext.to_ascii_lowercase(); + ext_lower == "zip" || ext_lower == "docx" || ext_lower == "xlsx" || ext_lower == "pptx" + }) + .unwrap_or(false) +} + +/// Read all entries from a ZIP archive. +/// +/// The function extracts each entry’s name and content. If an entry is a +/// directory it is ignored – only regular files are returned. +/// +/// The ZIP file must contain text files. For non‑text files you would need +/// a more sophisticated approach (e.g. decoding binary data or using a +/// specific codec). +pub fn read_zip(zip_path: &Path) -> Result> { + let file = File::open(zip_path) + .map_err(|e| miette!("Failed to open ZIP `{}`: {}", zip_path.display(), e))?; + + let mut archive = ZipArchive::new(file) + .map_err(|e| miette!("Failed to read ZIP `{}`: {}", zip_path.display(), e))?; + + let mut files = Vec::new(); + + for i in 0..archive.len() { + let mut entry = archive.by_index(i).map_err(|e| { + miette!( + "Failed to read entry {} in `{}`: {}", + i, + zip_path.display(), + e + ) + })?; + + // Skip directories + if entry.is_dir() { + continue; + } + + let mut content = String::new(); + entry.read_to_string(&mut content).map_err(|e| { + miette!( + "Failed to read entry `{}` in `{}`: {}", + entry.name(), + zip_path.display(), + e + ) + })?; + + files.push(FileContent { + path: PathBuf::from(entry.name()), + content, + }); + } + + Ok(files) +} + +/// Search‑and‑replace in a single file. +/// +/// The function reads the file, replaces all occurrences of `target` with +/// `replacement` and writes the modified text back to the original file. +/// If the file cannot be read or written an error is returned. +/// +/// # Examples +/// +/// ```ignore +/// let path = Path::new("example.txt"); +/// replace_in_file(path, "foo", "bar")?; +/// ``` +pub fn replace_in_file(path: &Path, target: &str, replacement: &str) -> Result<()> { + let content = read_file(path)?; + let new_content = content.replace(target, replacement); + let mut file = File::create(path) + .map_err(|e| miette!("Failed to open `{}` for writing: {}", path.display(), e))?; + file.write_all(new_content.as_bytes()) + .map_err(|e| miette!("Failed to write `{}`: {}", path.display(), e))?; + Ok(()) +} diff --git a/src/reader/tests.rs b/src/reader/tests.rs new file mode 100644 index 0000000..82aa4bc --- /dev/null +++ b/src/reader/tests.rs @@ -0,0 +1,89 @@ +use super::*; +use std::fs::{self, File}; +use std::io::Write; +use tempfile::tempdir; +use zip::write::FileOptions; + +/// Helper to write a file with the given content +fn write_file(path: &Path, content: &str) { + let mut f = File::create(path).expect("unable to create test file"); + f.write_all(content.as_bytes()) + .expect("unable to write test file"); +} + +/// Create a small ZIP file containing two text entries. +fn create_zip(path: &Path) -> std::io::Result<()> { + let file = File::create(path)?; + let mut zip = zip::ZipWriter::new(file); + + let options: FileOptions<()> = zip::write::FileOptions::default() + .compression_method(zip::CompressionMethod::Stored) + .unix_permissions(0o644); + + zip.start_file("foo.txt", options)?; + zip.write_all(b"hello foo")?; + + zip.start_file("bar.txt", options)?; + zip.write_all(b"hello bar")?; + + zip.finish()?; + Ok(()) +} + +#[test] +fn test_read_single_file() { + let dir = tempdir().expect("temp dir"); + let file_path = dir.path().join("test.txt"); + write_file(&file_path, "line1\nline2"); + + let it = universal_read(&file_path.to_path_buf()).expect("universal_read"); + let mut collected = Vec::new(); + for r in it { + collected.push(r.expect("file read")); + } + assert_eq!(collected.len(), 1); + assert_eq!(collected[0].path, file_path); + assert_eq!(collected[0].content, "line1\nline2"); +} + +#[test] +fn test_read_folder() { + let dir = tempdir().expect("temp dir"); + let sub = dir.path().join("sub"); + fs::create_dir(&sub).expect("mkdir"); + + write_file(&dir.path().join("a.txt"), "a"); + write_file(&sub.join("b.txt"), "b"); + + let it = universal_read(&dir.path().to_path_buf()).expect("universal_read"); + let mut paths: Vec = it.map(|r| r.expect("file read").path).collect(); + paths.sort(); + assert_eq!(paths, vec![dir.path().join("a.txt"), sub.join("b.txt")]); +} + +#[test] +fn test_read_zip() { + let dir = tempdir().expect("temp dir"); + let zip_path = dir.path().join("archive.zip"); + create_zip(&zip_path).expect("create zip"); + + let it = universal_read(&zip_path.to_path_buf()).expect("universal_read"); + let mut entries: Vec = it.map(|r| r.expect("file read").content).collect(); + entries.sort(); + assert_eq!( + entries, + vec!["hello bar".to_string(), "hello foo".to_string()] + ); +} + +#[test] +fn test_replace_in_file() { + let dir = tempdir().expect("temp dir"); + let file_path = dir.path().join("sample.txt"); + write_file(&file_path, "foo bar foo"); + + replace_in_file(&file_path, "foo", "baz").expect("replace"); + + let content = read_file(&file_path).expect("read"); + assert_eq!(content, "baz bar baz"); +}