Compare commits
3 Commits
main
...
poc-replac
| Author | SHA1 | Date |
|---|---|---|
|
|
f8dca15b90 | |
|
|
c10f2b9559 | |
|
|
d141f4e0c2 |
|
|
@ -19,3 +19,4 @@ Cargo.lock
|
|||
# Added by cargo
|
||||
|
||||
/target
|
||||
bob/
|
||||
|
|
|
|||
|
|
@ -4,5 +4,10 @@ version = "0.1.0"
|
|||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
miette = { version = "7.6.0", features = ["fancy"] }
|
||||
thiserror = "2.0.17"
|
||||
anyhow = "1.0.100"
|
||||
clap = { version = "4.5.53", features = ["derive"] }
|
||||
rayon = "1.11.0"
|
||||
regex = "1.12.2"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
toml = "0.9.8"
|
||||
walkdir = "2.5.0"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
[[patterns]]
|
||||
regex = "\\p{Pd}"
|
||||
replacement = "-"
|
||||
[[patterns]]
|
||||
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
|
||||
replacement = "'"
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
[[patterns]]
|
||||
regex = "\\p{Pd}"
|
||||
replacement = "-"
|
||||
[[patterns]]
|
||||
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
|
||||
replacement = "'"
|
||||
|
||||
180
src/main.rs
180
src/main.rs
|
|
@ -1,63 +1,137 @@
|
|||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Cargo.toml
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// [dependencies]
|
||||
// miette = "7"
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use regex::Regex;
|
||||
use serde::Deserialize;
|
||||
use std::{fs, path::PathBuf};
|
||||
use walkdir::{DirEntry, WalkDir};
|
||||
|
||||
use miette::{LabeledSpan, miette};
|
||||
use rayon::prelude::*; // brings `par_iter()` into scope
|
||||
|
||||
/// Creates a diagnostic for the ambiguous Unicode character U+2011
|
||||
/// (non‑breaking hyphen) that can be confused with the ordinary hyphen U+002D.
|
||||
fn unicode_confusion_report() -> miette::Report {
|
||||
// The source text you want to annotate
|
||||
let source = "Validate that the configuration file at path is well‑formed.".to_string();
|
||||
|
||||
// Build the report
|
||||
miette!(
|
||||
// Label the exact character that is problematic (index 52 in the string)
|
||||
labels = vec![LabeledSpan::at(52..53, "non‑breaking hyphen U+2011")],
|
||||
// Help text that tells the user what to do
|
||||
help = "This file contains ambiguous Unicode characters that might be \n\
|
||||
confused with other characters. This is often the mark of AI\n\
|
||||
generated text.",
|
||||
// Short description shown in the diagnostic title
|
||||
"Ambiguous Unicode character detected"
|
||||
)
|
||||
// Attach the source code to the report so that miette can pretty‑print it
|
||||
.with_source_code(source)
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "john_slop")]
|
||||
struct Cli {
|
||||
#[arg( value_parser = clap::value_parser!(PathBuf))]
|
||||
/// The path to analyse
|
||||
path: Vec<PathBuf>,
|
||||
/// Path to a configuration file (optional)
|
||||
#[arg(short = 'c', long = "config", value_parser = clap::value_parser!(PathBuf))]
|
||||
config: Option<PathBuf>,
|
||||
}
|
||||
|
||||
/// Creates a diagnostic for text that looks like it was generated by an AI,
|
||||
/// because it contains the phrase “as of my last knowledge update”.
|
||||
fn ai_generated_report() -> miette::Report {
|
||||
// The source text you want to annotate
|
||||
let source = "As of my last knowledge update in January 2022, a historic \n
|
||||
peace agreement between the Colombian government and the \
|
||||
Revolutionary Armed Forces of Colombia"
|
||||
.to_string();
|
||||
|
||||
// Build the report
|
||||
miette!(
|
||||
// Label the suspicious phrase (roughly the first 34 characters)
|
||||
labels = vec![LabeledSpan::at(0..34, "\"as of my last knowledge update\"")],
|
||||
help = "If the text contains phrases like \"as an AI model\" or \"as of my \n\
|
||||
last knowledge update\", or if the editor copy‑pasted the \n\
|
||||
prompt used to generate the text together with the AI \n\
|
||||
response, the text is almost certainly AI‑generated.",
|
||||
"Potential AI‑generated content detected"
|
||||
)
|
||||
.with_source_code(source)
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Config {
|
||||
patterns: Vec<Pattern>,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Show the Unicode‑confusion diagnostic
|
||||
let report1 = unicode_confusion_report();
|
||||
println!("{:?}\n", report1);
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Pattern {
|
||||
regex: String,
|
||||
replacement: String,
|
||||
}
|
||||
const DEFAULT_TOML: &str = include_str!("default_config.toml");
|
||||
|
||||
// Show the AI‑generated‑text diagnostic
|
||||
let report2 = ai_generated_report();
|
||||
println!("{:?}", report2);
|
||||
/// Read the user-supplied file or fall back to the default.
|
||||
fn load_config(path: &PathBuf) -> Result<Config> {
|
||||
if path.exists() {
|
||||
let data = fs::read_to_string(path)
|
||||
.with_context(|| format!("Could not read config at {:?}", path))?;
|
||||
toml::from_str(&data).with_context(|| "Failed to parse TOML") // ← 2️⃣
|
||||
} else {
|
||||
// If the file is missing, create it from the embedded default.
|
||||
fs::write(path, DEFAULT_TOML)
|
||||
.with_context(|| format!("Could not write default config to {:?}", path))?;
|
||||
// Parse the embedded default to return the same type.
|
||||
toml::from_str(DEFAULT_TOML).with_context(|| "Failed to parse embedded TOML") // ← 3️⃣
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// ---------------------------------------------------------
|
||||
// 1. Parse CLI arguments
|
||||
// ---------------------------------------------------------
|
||||
// usage: ./regex_replacer <directory> <config.toml>
|
||||
let cli = Cli::parse();
|
||||
|
||||
let target_dirs = cli.path;
|
||||
let cfg_path = cli.config.unwrap_or("config.toml".into());
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 2. Load and deserialize the TOML config
|
||||
// ---------------------------------------------------------
|
||||
// let cfg_contents = std::fs::read_to_string(&cfg_path)
|
||||
// let cfg: Config = toml::from_str(&cfg_contents)
|
||||
// .with_context(|| format!("TOML parsing failed for {:?}", cfg_path))?;
|
||||
let cfg = load_config(&cfg_path)
|
||||
.with_context(|| format!("Could not read config file {:?}", cfg_path))?;
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 3. Compile all regexes once (pre-compute for speed)
|
||||
// ---------------------------------------------------------
|
||||
let compiled_patterns: Result<Vec<(Regex, String)>> = cfg
|
||||
.patterns
|
||||
.into_iter()
|
||||
.map(|p| {
|
||||
let re = Regex::new(&p.regex).with_context(|| format!("Invalid regex: {}", p.regex))?;
|
||||
Ok((re, p.replacement))
|
||||
})
|
||||
.collect();
|
||||
let compiled_patterns = compiled_patterns?;
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// 4. Walk the directory tree - collect all files
|
||||
// ---------------------------------------------------------
|
||||
let files: Vec<PathBuf> = collect_files(&target_dirs);
|
||||
// ---------------------------------------------------------
|
||||
// 5. Process files in parallel
|
||||
// ---------------------------------------------------------
|
||||
files.par_iter().for_each(|path| {
|
||||
if let Err(err) = process_file(path, &compiled_patterns) {
|
||||
eprintln!("⚠️ Failed on {:?}: {}", path, err);
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply all regex→replacement rules to one file and overwrite it.
|
||||
fn process_file(path: &PathBuf, patterns: &[(Regex, String)]) -> Result<()> {
|
||||
let content =
|
||||
std::fs::read_to_string(path).with_context(|| format!("Failed to read {:?}", path))?;
|
||||
|
||||
// Apply every pattern in order
|
||||
let transformed = patterns.iter().fold(content, |acc, (re, repl)| {
|
||||
re.replace_all(&acc, repl).to_string()
|
||||
});
|
||||
|
||||
// Write the new content back to the file
|
||||
std::fs::write(path, transformed).with_context(|| format!("Failed to write to {:?}", path))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns `true` for every entry that is **not** hidden.
|
||||
fn is_not_hidden(entry: &DirEntry) -> bool {
|
||||
// `entry.file_name()` is a `OsStr`; converting it to a string
|
||||
// is cheap because we only need to check the first byte.
|
||||
!entry.file_name().to_string_lossy().starts_with('.')
|
||||
}
|
||||
|
||||
/// Collect all regular files that are reachable from *any* directory
|
||||
/// in `target_dirs` and are not hidden.
|
||||
pub fn collect_files(target_dirs: &[PathBuf]) -> Vec<PathBuf> {
|
||||
target_dirs
|
||||
.iter()
|
||||
.flat_map(|root| {
|
||||
// `WalkDir::new(root)` produces an iterator over *all*
|
||||
// entries below `root`.
|
||||
// We apply the same filters that were used in the original
|
||||
// single‑directory version.
|
||||
WalkDir::new(root)
|
||||
.into_iter()
|
||||
.filter_entry(is_not_hidden) // skip hidden directories
|
||||
.filter_map(Result::ok) // ignore I/O errors
|
||||
.filter(|e| e.file_type().is_file()) // keep only files
|
||||
.map(|e| e.path().to_path_buf())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
# Basic ASCII hyphen
|
||||
foo-bar
|
||||
bar-foo
|
||||
|
||||
# Unicode dash variants - all are in the Pd category
|
||||
foo-bar # EN DASH (U+2013)
|
||||
foo-bar # EM DASH (U+2014)
|
||||
foo-bar # HYPHEN-MINUS (U+2010)
|
||||
foo-bar # FIGURE DASH (U+2012)
|
||||
foo--bar # Two EN DASHes side-by-side
|
||||
foo---------bar # Mix of all
|
||||
|
||||
# Other punctuation that *looks* like a dash but is not Pd
|
||||
foo·bar # BULLET (U+00B7)
|
||||
foo/ bar # SOLIDUS (U+002F)
|
||||
foo- bar # EM DASH followed by a non-breaking space (U+00A0)
|
||||
|
||||
# Edge cases
|
||||
foo- -bar # ASCII hyphen, space, en dash, space, bar
|
||||
-foo-bar- # Hyphens at the start and end
|
||||
foo--bar- # Mixed dashes with trailing en dash
|
||||
|
||||
|
||||
# Test words surrounded by dashes - word boundaries
|
||||
-foo-
|
||||
foo-
|
||||
-bar
|
||||
--foo--
|
||||
foo--bar
|
||||
foo--bar
|
||||
foo---bar
|
||||
|
||||
# Dashes embedded in words (should NOT be replaced if you keep word-boundaries)
|
||||
f-oo-b-ar # “f-oo” (hyphen inside a word)
|
||||
fo-ob-ar # “fo-ob” (en dash inside a word)
|
||||
|
||||
# Dashes as punctuation in a sentence
|
||||
"foo - bar" # En dash surrounded by spaces
|
||||
"foo-bar" # No spaces
|
||||
"foo-bar" # EM dash, no spaces
|
||||
"foo-bar" # FIGURE dash
|
||||
"foo-bar" # HYPHEN-MINUS
|
||||
|
||||
# Mix of Unicode whitespace
|
||||
foo - bar # U+202F NARROW NO-BREAK SPACE
|
||||
foo\u2005-bar # U+2005 MEDIUM MATHEMATICAL SPACE
|
||||
|
||||
# Multiple dashes with other punctuation
|
||||
foo--bar # Two ASCII hyphens
|
||||
foo--bar # EM DASH + EM DASH
|
||||
foo---bar # HYPHEN-MINUS + EM DASH + FIGURE DASH
|
||||
|
||||
# Apostrophes
|
||||
Here's a “quote” with a mix of apostrophes: 'Don't' “re-write” 'my' “code”, ʹand ʽtʾe-sʾtʽ! Also, 'like this', or a full-width ', or a low-9 'and a high-reversed 'example'. (End of line.)
|
||||
"`This line should stay intact`"
|
||||
Loading…
Reference in New Issue