Compare commits

...

3 Commits

Author SHA1 Message Date
Dandaedre f8dca15b90 Poc 2026-01-05 14:37:24 +01:00
Dandaedre c10f2b9559 Ignores hidden files 2025-12-08 00:50:43 +01:00
Dandaedre d141f4e0c2 Poc 2025-12-07 15:41:43 +01:00
6 changed files with 204 additions and 55 deletions

1
.gitignore vendored
View File

@ -19,3 +19,4 @@ Cargo.lock
# Added by cargo
/target
bob/

View File

@ -4,5 +4,10 @@ version = "0.1.0"
edition = "2024"
[dependencies]
miette = { version = "7.6.0", features = ["fancy"] }
thiserror = "2.0.17"
anyhow = "1.0.100"
clap = { version = "4.5.53", features = ["derive"] }
rayon = "1.11.0"
regex = "1.12.2"
serde = { version = "1.0.228", features = ["derive"] }
toml = "0.9.8"
walkdir = "2.5.0"

7
config.toml Normal file
View File

@ -0,0 +1,7 @@
[[patterns]]
regex = "\\p{Pd}"
replacement = "-"
[[patterns]]
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
replacement = "'"

7
src/default_config.toml Normal file
View File

@ -0,0 +1,7 @@
[[patterns]]
regex = "\\p{Pd}"
replacement = "-"
[[patterns]]
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
replacement = "'"

View File

@ -1,63 +1,137 @@
// ──────────────────────────────────────────────────────────────────────
// Cargo.toml
// ──────────────────────────────────────────────────────────────────────
// [dependencies]
// miette = "7"
// ──────────────────────────────────────────────────────────────────────
use anyhow::{Context, Result};
use clap::Parser;
use regex::Regex;
use serde::Deserialize;
use std::{fs, path::PathBuf};
use walkdir::{DirEntry, WalkDir};
use miette::{LabeledSpan, miette};
use rayon::prelude::*; // brings `par_iter()` into scope
/// Creates a diagnostic for the ambiguous Unicode character U+2011
/// (non‑breaking hyphen) that can be confused with the ordinary hyphen U+002D.
fn unicode_confusion_report() -> miette::Report {
// The source text you want to annotate
let source = "Validate that the configuration file at path is well‑formed.".to_string();
// Build the report
miette!(
// Label the exact character that is problematic (index 52 in the string)
labels = vec![LabeledSpan::at(52..53, "non‑breaking hyphen U+2011")],
// Help text that tells the user what to do
help = "This file contains ambiguous Unicode characters that might be \n\
confused with other characters. This is often the mark of AI\n\
generated text.",
// Short description shown in the diagnostic title
"Ambiguous Unicode character detected"
)
// Attach the source code to the report so that miette can pretty‑print it
.with_source_code(source)
#[derive(Parser, Debug)]
#[command(name = "john_slop")]
struct Cli {
#[arg( value_parser = clap::value_parser!(PathBuf))]
/// The path to analyse
path: Vec<PathBuf>,
/// Path to a configuration file (optional)
#[arg(short = 'c', long = "config", value_parser = clap::value_parser!(PathBuf))]
config: Option<PathBuf>,
}
/// Creates a diagnostic for text that looks like it was generated by an AI,
/// because it contains the phrase “as of my last knowledge update”.
fn ai_generated_report() -> miette::Report {
// The source text you want to annotate
let source = "As of my last knowledge update in January 2022, a historic \n
peace agreement between the Colombian government and the \
Revolutionary Armed Forces of Colombia"
.to_string();
// Build the report
miette!(
// Label the suspicious phrase (roughly the first 34 characters)
labels = vec![LabeledSpan::at(0..34, "\"as of my last knowledge update\"")],
help = "If the text contains phrases like \"as an AI model\" or \"as of my \n\
last knowledge update\", or if the editor copy‑pasted the \n\
prompt used to generate the text together with the AI \n\
response, the text is almost certainly AI‑generated.",
"Potential AI‑generated content detected"
)
.with_source_code(source)
#[derive(Debug, Deserialize)]
struct Config {
patterns: Vec<Pattern>,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
// Show the Unicode‑confusion diagnostic
let report1 = unicode_confusion_report();
println!("{:?}\n", report1);
#[derive(Debug, Deserialize)]
struct Pattern {
regex: String,
replacement: String,
}
const DEFAULT_TOML: &str = include_str!("default_config.toml");
// Show the AI‑generated‑text diagnostic
let report2 = ai_generated_report();
println!("{:?}", report2);
/// Read the user-supplied file or fall back to the default.
fn load_config(path: &PathBuf) -> Result<Config> {
if path.exists() {
let data = fs::read_to_string(path)
.with_context(|| format!("Could not read config at {:?}", path))?;
toml::from_str(&data).with_context(|| "Failed to parse TOML") // ← 2️⃣
} else {
// If the file is missing, create it from the embedded default.
fs::write(path, DEFAULT_TOML)
.with_context(|| format!("Could not write default config to {:?}", path))?;
// Parse the embedded default to return the same type.
toml::from_str(DEFAULT_TOML).with_context(|| "Failed to parse embedded TOML") // ← 3️⃣
}
}
fn main() -> Result<()> {
// ---------------------------------------------------------
// 1. Parse CLI arguments
// ---------------------------------------------------------
// usage: ./regex_replacer <directory> <config.toml>
let cli = Cli::parse();
let target_dirs = cli.path;
let cfg_path = cli.config.unwrap_or("config.toml".into());
// ---------------------------------------------------------
// 2. Load and deserialize the TOML config
// ---------------------------------------------------------
// let cfg_contents = std::fs::read_to_string(&cfg_path)
// let cfg: Config = toml::from_str(&cfg_contents)
// .with_context(|| format!("TOML parsing failed for {:?}", cfg_path))?;
let cfg = load_config(&cfg_path)
.with_context(|| format!("Could not read config file {:?}", cfg_path))?;
// ---------------------------------------------------------
// 3. Compile all regexes once (pre-compute for speed)
// ---------------------------------------------------------
let compiled_patterns: Result<Vec<(Regex, String)>> = cfg
.patterns
.into_iter()
.map(|p| {
let re = Regex::new(&p.regex).with_context(|| format!("Invalid regex: {}", p.regex))?;
Ok((re, p.replacement))
})
.collect();
let compiled_patterns = compiled_patterns?;
// ---------------------------------------------------------
// 4. Walk the directory tree - collect all files
// ---------------------------------------------------------
let files: Vec<PathBuf> = collect_files(&target_dirs);
// ---------------------------------------------------------
// 5. Process files in parallel
// ---------------------------------------------------------
files.par_iter().for_each(|path| {
if let Err(err) = process_file(path, &compiled_patterns) {
eprintln!("⚠️ Failed on {:?}: {}", path, err);
}
});
Ok(())
}
/// Apply all regex→replacement rules to one file and overwrite it.
fn process_file(path: &PathBuf, patterns: &[(Regex, String)]) -> Result<()> {
let content =
std::fs::read_to_string(path).with_context(|| format!("Failed to read {:?}", path))?;
// Apply every pattern in order
let transformed = patterns.iter().fold(content, |acc, (re, repl)| {
re.replace_all(&acc, repl).to_string()
});
// Write the new content back to the file
std::fs::write(path, transformed).with_context(|| format!("Failed to write to {:?}", path))?;
Ok(())
}
/// Returns `true` for every entry that is **not** hidden.
fn is_not_hidden(entry: &DirEntry) -> bool {
// `entry.file_name()` is a `OsStr`; converting it to a string
// is cheap because we only need to check the first byte.
!entry.file_name().to_string_lossy().starts_with('.')
}
/// Collect all regular files that are reachable from *any* directory
/// in `target_dirs` and are not hidden.
pub fn collect_files(target_dirs: &[PathBuf]) -> Vec<PathBuf> {
target_dirs
.iter()
.flat_map(|root| {
// `WalkDir::new(root)` produces an iterator over *all*
// entries below `root`.
// We apply the same filters that were used in the original
// single‑directory version.
WalkDir::new(root)
.into_iter()
.filter_entry(is_not_hidden) // skip hidden directories
.filter_map(Result::ok) // ignore I/O errors
.filter(|e| e.file_type().is_file()) // keep only files
.map(|e| e.path().to_path_buf())
})
.collect()
}

55
test/files.md Normal file
View File

@ -0,0 +1,55 @@
# Basic ASCII hyphen
foo-bar
bar-foo
# Unicode dash variants - all are in the Pd category
foo-bar # EN DASH (U+2013)
foo-bar # EM DASH (U+2014)
foo-bar # HYPHEN-MINUS (U+2010)
foo-bar # FIGURE DASH (U+2012)
foo--bar # Two EN DASHes side-by-side
foo---------bar # Mix of all
# Other punctuation that *looks* like a dash but is not Pd
foo·bar # BULLET (U+00B7)
foo/ bar # SOLIDUS (U+002F)
foo- bar # EM DASH followed by a non-breaking space (U+00A0)
# Edge cases
foo- -bar # ASCII hyphen, space, en dash, space, bar
-foo-bar- # Hyphens at the start and end
foo--bar- # Mixed dashes with trailing en dash
# Test words surrounded by dashes - word boundaries
-foo-
foo-
-bar
--foo--
foo--bar
foo--bar
foo---bar
# Dashes embedded in words (should NOT be replaced if you keep word-boundaries)
f-oo-b-ar # “f-oo” (hyphen inside a word)
fo-ob-ar # “fo-ob” (en dash inside a word)
# Dashes as punctuation in a sentence
"foo - bar" # En dash surrounded by spaces
"foo-bar" # No spaces
"foo-bar" # EM dash, no spaces
"foo-bar" # FIGURE dash
"foo-bar" # HYPHEN-MINUS
# Mix of Unicode whitespace
foo - bar # U+202F NARROW NO-BREAK SPACE
foo\u2005-bar # U+2005 MEDIUM MATHEMATICAL SPACE
# Multiple dashes with other punctuation
foo--bar # Two ASCII hyphens
foo--bar # EM DASH + EM DASH
foo---bar # HYPHEN-MINUS + EM DASH + FIGURE DASH
# Apostrophes
Here's a “quote” with a mix of apostrophes: 'Don't' “re-write” 'my' “code”, ʹand ʽtʾe-sʾtʽ! Also, 'like this', or a full-width ', or a low-9 'and a high-reversed 'example'. (End of line.)
"`This line should stay intact`"