Shitcoded some base functions, too tired to do any useful shit

This commit is contained in:
Dandaedre 2025-12-04 17:27:50 +01:00
parent 0a5c7b535d
commit 27a8494dc7
6 changed files with 551 additions and 13 deletions

View File

@ -5,4 +5,11 @@ edition = "2024"
[dependencies]
miette = { version = "7.6.0", features = ["fancy"] }
once_cell = "1.21.3"
rayon = "1.11.0"
regex = "1.12.2"
tempfile = "3.23.0"
thiserror = "2.0.17"
unicode-segmentation = "1.12.0"
walkdir = "2.5.0"
zip = "6.0.0"

92
src/detector.rs Normal file
View File

@ -0,0 +1,92 @@
//! All detection function & modules
//!
use once_cell::sync::Lazy;
use regex::{Regex, RegexBuilder};
/// A tiny “search‑engine” that knows a *set* of Unicode code‑points.
/// It returns the **byte** offset of every match (so the positions are
/// deterministic even if you run the search in parallel).
static HYPHEN_REGEX: Lazy<Regex> = Lazy::new(|| {
// All the Unicode characters that look like a hyphen.
// (U+002D, U+2010, U+2011, U+2012, U+2013, U+2014, U+2015, …)
let pattern = r"[\u002D\u2010\u2011\u2012\u2013\u2014\u2015\u2212\uFE58\uFE63\uFF0D]";
RegexBuilder::new(pattern)
.unicode(true) // enable Unicode semantics
.build()
.expect("regex build failed")
});
/// Find *all* occurrences of the characters in `re` inside `text`.
/// Returns a vector of `(byte_start, byte_end, matched_char)` tuples.
fn find_all(re: &Regex, text: &str) -> Vec<(usize, usize, char)> {
re.find_iter(text)
.map(|m| (m.start(), m.end(), text[m.range()].chars().next().unwrap()))
.collect()
}
// If you have many different “special” characters you can keep a
// `Vec<(Regex, &'static str)>` and run the search in parallel with `rayon`.
#[cfg(test)]
mod tests {
use super::*;
use rayon::prelude::*;
use unicode_segmentation::UnicodeSegmentation;
#[test]
fn it_finds_all_hyphens() {
let s = "a‑b—c‑—d‑";
// The string contains the following code points:
// U+2011 (NON‑BREAKING HYPHEN) -> a‑b
// U+2014 (EM DASH) -> —c
// U+2011 again -> d‑
// U+2014 again -> —
let mut results = find_all(&HYPHEN_REGEX, s);
// sort to make the assertion deterministic
results.sort_by_key(|r| r.0);
assert_eq!(
results,
vec![
(1, 4, '‑'),
(5, 8, '—'),
(9, 12, '‑'),
(12, 15, '—'),
(16, 19, '‑')
]
);
}
#[test]
fn it_is_thread_safe_and_can_run_in_parallel() {
// A big dummy string that contains a bunch of hyphen variants.
let s = (0..10_000)
.map(|i| if i % 3 == 0 { "—" } else { "-" })
.collect::<String>();
// Pretend we have two different regexes to search (just one here).
let regexes = vec![&*HYPHEN_REGEX];
// Parallel search (one thread per regex – not the fastest, but
// shows the “share‑regex‑between‑threads” pattern).
let results: Vec<Vec<_>> = regexes.par_iter().map(|re| find_all(re, &s)).collect();
// All regexes returned the same count.
assert_eq!(results[0].len(), 10_000);
}
#[test]
fn grapheme_positions_are_correct() {
// A string with a single grapheme cluster that is two bytes in UTF‑8.
let s = "a𝌆b"; // 𝌆 = U+1D306, 3 bytes
// Find the middle grapheme (the diamond).
let diamond_start = s.grapheme_indices(true).nth(1).unwrap().0;
assert_eq!(diamond_start, 1); // the byte offset of the start of 𝌆
// Find it with our regex (if we had a regex that matches 𝌆).
let diamond_re = Regex::new(r"\u{1D306}").unwrap();
let m = diamond_re.find(s).unwrap();
assert_eq!(m.start(), diamond_start);
}
}

100
src/errors.rs Normal file
View File

@ -0,0 +1,100 @@
//! # AI‑Detection Errors
//!
//! This module contains the error types used by the AI‑writing detection
//! logic. The high‑level `AiDetectionError` groups the *four main
//! “signs of AI‑writing”* that editors look for on Wikipedia. Each
//! variant may carry a more detailed sub‑error – e.g. the `FormattingClue`
//! variant contains a `FormattingError` that describes the exact
//! formatting problem that was spotted.
//!
//! The design aims for:
//! * **Type safety** – callers can match on the category instead of parsing
//! a string.
//! * **Extensibility** – new sub‑errors can be added without touching
//! the public API.
//! * **Human‑readability** – `thiserror::Error` automatically provides a
//! clear message for logging or UI display.
//!
use miette::{Diagnostic, SourceSpan};
use thiserror::Error;
/// Errors that signal a suspected AI‑generated passage.
///
/// Each variant maps to one of the four “signs of AI‑writing” categories
#[derive(Error, Debug)]
pub enum AiDetectionError {
/// The text shows common stylistic patterns that LLMs tend to produce.
#[error("AI‑generated text: stylistic patterns detected")]
StylisticPattern,
/// Formatting idiosyncrasies (e.g., odd punctuation, inconsistent heading styles)
/// that point to an automated draft.
#[error("AI‑generated text: formatting clues detected")]
FormattingClue(#[from] FormattingError),
/// Content‑related quirks such as hallucinations, fabricated facts,
/// or other substantive inconsistencies.
#[error("AI‑generated text: content‑related quirks detected")]
ContentQuirk,
/// Issues with context or nuance—e.g., the passage feels out of place
/// within the surrounding article or uses language that the editor knows
/// is atypical for human writers.
#[error("AI‑generated text: context & nuance issues detected")]
ContextNuance,
}
/// Sub‑errors that describe the specific formatting problem
/// that was found in a suspected AI passage.
#[derive(Error, Debug, Diagnostic)]
pub enum FormattingError {
/// The passage contains Unicode code points that are not normally
/// used in articles (e.g. zero‑width joiners, control
/// characters, or characters from a different writing system).
#[error("unexpected or strange Unicode detected ({character})")]
StrangeUTFDetected {
/// The offending character
#[source_code]
character: String,
/// Source range
#[label("Pb")]
span: SourceSpan,
},
/// The passage contains an overuse of emoticons or emoji, which
/// is highly unlikely in a document.
#[error("overuse of emoticons detected {number} emoticons over {range} characters")]
OveruseOfEmoticons {
/// List of all offending emoticons
emoticons: Vec<String>,
/// Number of offenders
number: usize,
/// Offending range
range: usize,
},
/// Markdown syntax (`*`, `#`, `>`, etc.) appears in a place where
/// a document normally contains plain prose, indicating an
/// automated template was not cleaned up.
#[error("markdown syntax found in an unexpected location")]
MarkdownInUnusualPlace,
/// A bullet list is broken by a paragraph or a line break that
/// does not belong to the list.
#[error("list contains unexpected line breaks")]
UnexpectedListBreak,
/// The article uses a different list style (e.g. Roman numerals
/// instead of Arabic digits) in the middle of a paragraph.
#[error("inconsistent list style detected")]
InconsistentListStyle,
}
pub fn try_report() {
let a: miette::Report = FormattingError::StrangeUTFDetected {
character: "this sentence is first ocntext\nnew line with »\nboibob\n\nbob".into(),
span: (45..46).into(),
}
.into();
println!("{:?}", a);
}

View File

@ -1,12 +1,15 @@
// ──────────────────────────────────────────────────────────────────────
// Cargo.toml
// ──────────────────────────────────────────────────────────────────────
// [dependencies]
// miette = "7"
// ──────────────────────────────────────────────────────────────────────
//!Fill this doc
//!
#![warn(clippy::missing_docs_in_private_items)]
#![allow(dead_code)]
use miette::{LabeledSpan, miette};
use crate::errors::try_report;
mod detector;
mod errors;
mod reader;
/// Creates a diagnostic for the ambiguous Unicode character U+2011
/// (non‑breaking hyphen) that can be confused with the ordinary hyphen U+002D.
fn unicode_confusion_report() -> miette::Report {
@ -51,13 +54,14 @@ fn ai_generated_report() -> miette::Report {
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
// Show the Unicode‑confusion diagnostic
let report1 = unicode_confusion_report();
println!("{:?}\n", report1);
// // Show the Unicode‑confusion diagnostic
// let report1 = unicode_confusion_report();
// println!("{:?}\n", report1);
// Show the AI‑generated‑text diagnostic
let report2 = ai_generated_report();
println!("{:?}", report2);
// // Show the AI‑generated‑text diagnostic
// let report2 = ai_generated_report();
// println!("{:?}", report2);
try_report();
Ok(())
}

246
src/reader.rs Normal file
View File

@ -0,0 +1,246 @@
//! # Multimode reader
//!
//! This module implements a *very* small proof‑of‑concept file reader that can handle
//! individual files, directories and simple ZIP‑based archives (e.g. DOCX, XLSX,
//! …). The public API is intentionally minimal – just enough to
//! demonstrate how you can read a file line‑by‑line or perform a
//! search‑&‑replace operation.
//!
//! The key types are:
//! * `FileDescriptorIterator` – an iterator that yields a `FileContent` for every
//! readable file found under a given path.
//! * `FileContent` – a tiny wrapper that keeps the file’s path and its
//! entire textual content.
//!
//! The helper functions below give a quick way to read a single file, walk a
//! directory or unpack a ZIP archive. All functions return `miette::Result` so
//! that any error can be displayed in a user‑friendly way.
#[cfg(test)]
mod tests;
use std::{
fs::File,
io::{Read, Write},
path::{Path, PathBuf},
};
use miette::{Result, miette};
use walkdir::WalkDir;
use zip::ZipArchive;
/// Information about a single file.
///
/// * `path` – the absolute (or relative) path to the file.
/// * `content` – the file’s content as a UTF‑8 string.
///
/// The struct is intentionally lightweight; for the POC we only store the
/// contents as a `String`. In a real‑world library you might want to keep a
/// reader instead of the whole string.
#[derive(Debug, Clone)]
pub struct FileContent {
pub path: PathBuf,
pub content: String,
}
/// Iterator over files found under a path.
///
/// The iterator will return one `FileContent` per file. It is intentionally
/// simple – it holds all file descriptors in a vector and yields them one by
/// one. This makes it easy to use in a `for` loop while still keeping the
/// implementation short.
pub struct FileDescriptorIterator {
files: Vec<FileContent>,
index: usize,
}
impl FileDescriptorIterator {
/// Create a new iterator from a vector of `FileContent`s.
pub fn new(files: Vec<FileContent>) -> Self {
Self { files, index: 0 }
}
/// Return the number of files that will be yielded.
pub fn len(&self) -> usize {
self.files.len()
}
/// Return `true` if the iterator has no more items.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl Iterator for FileDescriptorIterator {
type Item = Result<FileContent>;
fn next(&mut self) -> Option<Self::Item> {
if self.index >= self.files.len() {
return None;
}
let res = self.files[self.index].clone();
self.index += 1;
Some(Ok(res))
}
}
/// Universal reader.
///
/// * `file_path` – a path that can point to a file, a directory or a ZIP
/// archive.
/// * Returns a `FileDescriptorIterator` that yields the files contained in the
/// input. If the input is a plain file, the iterator will contain a single
/// element.
///
/// The function recognises ZIP files by their extension (`.zip`, `.docx`,
/// `.xlsx`, …). If the file is not a recognised ZIP format it is treated as
/// a plain file.
///
/// Errors are wrapped with `miette` so that they can be printed nicely to the
/// console.
pub fn universal_read(file_path: &PathBuf) -> Result<FileDescriptorIterator> {
if file_path.is_file() {
// If the file looks like a zip we handle it specially
if is_zip(file_path) {
let zip_files = read_zip(file_path)?;
return Ok(FileDescriptorIterator::new(zip_files));
}
// Plain file – just read it
let content = read_file(file_path)?;
return Ok(FileDescriptorIterator::new(vec![FileContent {
path: file_path.clone(),
content,
}]));
}
if file_path.is_dir() {
let dir_files = read_folder(file_path)?;
return Ok(FileDescriptorIterator::new(dir_files));
}
Err(miette!(
"The path `{}` is neither a file, a directory nor a recognised zip archive",
file_path.display()
))
}
/// Read a single file into a `String`.
///
/// The file is opened and its content is read as UTF‑8. If the file contains
/// non‑UTF‑8 data an error is returned.
pub fn read_file(file_path: &Path) -> Result<String> {
let mut file = File::open(file_path)
.map_err(|e| miette!("Failed to open `{}`: {}", file_path.display(), e))?;
let mut content = String::new();
file.read_to_string(&mut content)
.map_err(|e| miette!("Failed to read `{}`: {}", file_path.display(), e))?;
Ok(content)
}
/// Recursively walk a directory and read all regular files.
///
/// The function returns a `Vec<FileContent>` where each element contains the
/// path and the full textual content of a file. Hidden files and directories
/// are *not* filtered out – they are processed like any other entry.
pub fn read_folder(dir_path: &Path) -> Result<Vec<FileContent>> {
let mut files = Vec::new();
for entry in WalkDir::new(dir_path)
.into_iter()
.filter_map(Result::ok)
.filter(|e| e.file_type().is_file())
{
let path = entry.path().to_path_buf();
let content = read_file(&path)?;
files.push(FileContent { path, content });
}
Ok(files)
}
/// Return `true` if the file has an extension that we treat as a ZIP archive.
/// For simplicity we just check the first part of the extension (e.g. “zip”
/// for “docx”).
fn is_zip(path: &Path) -> bool {
path.extension()
.and_then(|s| s.to_str())
.map(|ext| {
let ext_lower = ext.to_ascii_lowercase();
ext_lower == "zip" || ext_lower == "docx" || ext_lower == "xlsx" || ext_lower == "pptx"
})
.unwrap_or(false)
}
/// Read all entries from a ZIP archive.
///
/// The function extracts each entry’s name and content. If an entry is a
/// directory it is ignored – only regular files are returned.
///
/// The ZIP file must contain text files. For non‑text files you would need
/// a more sophisticated approach (e.g. decoding binary data or using a
/// specific codec).
pub fn read_zip(zip_path: &Path) -> Result<Vec<FileContent>> {
let file = File::open(zip_path)
.map_err(|e| miette!("Failed to open ZIP `{}`: {}", zip_path.display(), e))?;
let mut archive = ZipArchive::new(file)
.map_err(|e| miette!("Failed to read ZIP `{}`: {}", zip_path.display(), e))?;
let mut files = Vec::new();
for i in 0..archive.len() {
let mut entry = archive.by_index(i).map_err(|e| {
miette!(
"Failed to read entry {} in `{}`: {}",
i,
zip_path.display(),
e
)
})?;
// Skip directories
if entry.is_dir() {
continue;
}
let mut content = String::new();
entry.read_to_string(&mut content).map_err(|e| {
miette!(
"Failed to read entry `{}` in `{}`: {}",
entry.name(),
zip_path.display(),
e
)
})?;
files.push(FileContent {
path: PathBuf::from(entry.name()),
content,
});
}
Ok(files)
}
/// Search‑and‑replace in a single file.
///
/// The function reads the file, replaces all occurrences of `target` with
/// `replacement` and writes the modified text back to the original file.
/// If the file cannot be read or written an error is returned.
///
/// # Examples
///
/// ```ignore
/// let path = Path::new("example.txt");
/// replace_in_file(path, "foo", "bar")?;
/// ```
pub fn replace_in_file(path: &Path, target: &str, replacement: &str) -> Result<()> {
let content = read_file(path)?;
let new_content = content.replace(target, replacement);
let mut file = File::create(path)
.map_err(|e| miette!("Failed to open `{}` for writing: {}", path.display(), e))?;
file.write_all(new_content.as_bytes())
.map_err(|e| miette!("Failed to write `{}`: {}", path.display(), e))?;
Ok(())
}

89
src/reader/tests.rs Normal file
View File

@ -0,0 +1,89 @@
use super::*;
use std::fs::{self, File};
use std::io::Write;
use tempfile::tempdir;
use zip::write::FileOptions;
/// Helper to write a file with the given content
fn write_file(path: &Path, content: &str) {
let mut f = File::create(path).expect("unable to create test file");
f.write_all(content.as_bytes())
.expect("unable to write test file");
}
/// Create a small ZIP file containing two text entries.
fn create_zip(path: &Path) -> std::io::Result<()> {
let file = File::create(path)?;
let mut zip = zip::ZipWriter::new(file);
let options: FileOptions<()> = zip::write::FileOptions::default()
.compression_method(zip::CompressionMethod::Stored)
.unix_permissions(0o644);
zip.start_file("foo.txt", options)?;
zip.write_all(b"hello foo")?;
zip.start_file("bar.txt", options)?;
zip.write_all(b"hello bar")?;
zip.finish()?;
Ok(())
}
#[test]
fn test_read_single_file() {
let dir = tempdir().expect("temp dir");
let file_path = dir.path().join("test.txt");
write_file(&file_path, "line1\nline2");
let it = universal_read(&file_path.to_path_buf()).expect("universal_read");
let mut collected = Vec::new();
for r in it {
collected.push(r.expect("file read"));
}
assert_eq!(collected.len(), 1);
assert_eq!(collected[0].path, file_path);
assert_eq!(collected[0].content, "line1\nline2");
}
#[test]
fn test_read_folder() {
let dir = tempdir().expect("temp dir");
let sub = dir.path().join("sub");
fs::create_dir(&sub).expect("mkdir");
write_file(&dir.path().join("a.txt"), "a");
write_file(&sub.join("b.txt"), "b");
let it = universal_read(&dir.path().to_path_buf()).expect("universal_read");
let mut paths: Vec<PathBuf> = it.map(|r| r.expect("file read").path).collect();
paths.sort();
assert_eq!(paths, vec![dir.path().join("a.txt"), sub.join("b.txt")]);
}
#[test]
fn test_read_zip() {
let dir = tempdir().expect("temp dir");
let zip_path = dir.path().join("archive.zip");
create_zip(&zip_path).expect("create zip");
let it = universal_read(&zip_path.to_path_buf()).expect("universal_read");
let mut entries: Vec<String> = it.map(|r| r.expect("file read").content).collect();
entries.sort();
assert_eq!(
entries,
vec!["hello bar".to_string(), "hello foo".to_string()]
);
}
#[test]
fn test_replace_in_file() {
let dir = tempdir().expect("temp dir");
let file_path = dir.path().join("sample.txt");
write_file(&file_path, "foo bar foo");
replace_in_file(&file_path, "foo", "baz").expect("replace");
let content = read_file(&file_path).expect("read");
assert_eq!(content, "baz bar baz");
}