Compare commits
3 Commits
main
...
poc-replac
| Author | SHA1 | Date |
|---|---|---|
|
|
f8dca15b90 | |
|
|
c10f2b9559 | |
|
|
d141f4e0c2 |
|
|
@ -19,3 +19,4 @@ Cargo.lock
|
||||||
# Added by cargo
|
# Added by cargo
|
||||||
|
|
||||||
/target
|
/target
|
||||||
|
bob/
|
||||||
|
|
|
||||||
|
|
@ -4,5 +4,10 @@ version = "0.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
miette = { version = "7.6.0", features = ["fancy"] }
|
anyhow = "1.0.100"
|
||||||
thiserror = "2.0.17"
|
clap = { version = "4.5.53", features = ["derive"] }
|
||||||
|
rayon = "1.11.0"
|
||||||
|
regex = "1.12.2"
|
||||||
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
|
toml = "0.9.8"
|
||||||
|
walkdir = "2.5.0"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
[[patterns]]
|
||||||
|
regex = "\\p{Pd}"
|
||||||
|
replacement = "-"
|
||||||
|
[[patterns]]
|
||||||
|
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
|
||||||
|
replacement = "'"
|
||||||
|
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
[[patterns]]
|
||||||
|
regex = "\\p{Pd}"
|
||||||
|
replacement = "-"
|
||||||
|
[[patterns]]
|
||||||
|
regex = "[\u2018\u2019\u201A\u201B\u02BC\u02BB\u00B4\uFF40\uFF07]"
|
||||||
|
replacement = "'"
|
||||||
|
|
||||||
180
src/main.rs
180
src/main.rs
|
|
@ -1,63 +1,137 @@
|
||||||
// ──────────────────────────────────────────────────────────────────────
|
use anyhow::{Context, Result};
|
||||||
// Cargo.toml
|
use clap::Parser;
|
||||||
// ──────────────────────────────────────────────────────────────────────
|
use regex::Regex;
|
||||||
// [dependencies]
|
use serde::Deserialize;
|
||||||
// miette = "7"
|
use std::{fs, path::PathBuf};
|
||||||
// ──────────────────────────────────────────────────────────────────────
|
use walkdir::{DirEntry, WalkDir};
|
||||||
|
|
||||||
use miette::{LabeledSpan, miette};
|
use rayon::prelude::*; // brings `par_iter()` into scope
|
||||||
|
|
||||||
/// Creates a diagnostic for the ambiguous Unicode character U+2011
|
#[derive(Parser, Debug)]
|
||||||
/// (non‑breaking hyphen) that can be confused with the ordinary hyphen U+002D.
|
#[command(name = "john_slop")]
|
||||||
fn unicode_confusion_report() -> miette::Report {
|
struct Cli {
|
||||||
// The source text you want to annotate
|
#[arg( value_parser = clap::value_parser!(PathBuf))]
|
||||||
let source = "Validate that the configuration file at path is well‑formed.".to_string();
|
/// The path to analyse
|
||||||
|
path: Vec<PathBuf>,
|
||||||
// Build the report
|
/// Path to a configuration file (optional)
|
||||||
miette!(
|
#[arg(short = 'c', long = "config", value_parser = clap::value_parser!(PathBuf))]
|
||||||
// Label the exact character that is problematic (index 52 in the string)
|
config: Option<PathBuf>,
|
||||||
labels = vec![LabeledSpan::at(52..53, "non‑breaking hyphen U+2011")],
|
|
||||||
// Help text that tells the user what to do
|
|
||||||
help = "This file contains ambiguous Unicode characters that might be \n\
|
|
||||||
confused with other characters. This is often the mark of AI\n\
|
|
||||||
generated text.",
|
|
||||||
// Short description shown in the diagnostic title
|
|
||||||
"Ambiguous Unicode character detected"
|
|
||||||
)
|
|
||||||
// Attach the source code to the report so that miette can pretty‑print it
|
|
||||||
.with_source_code(source)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a diagnostic for text that looks like it was generated by an AI,
|
#[derive(Debug, Deserialize)]
|
||||||
/// because it contains the phrase “as of my last knowledge update”.
|
struct Config {
|
||||||
fn ai_generated_report() -> miette::Report {
|
patterns: Vec<Pattern>,
|
||||||
// The source text you want to annotate
|
|
||||||
let source = "As of my last knowledge update in January 2022, a historic \n
|
|
||||||
peace agreement between the Colombian government and the \
|
|
||||||
Revolutionary Armed Forces of Colombia"
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
// Build the report
|
|
||||||
miette!(
|
|
||||||
// Label the suspicious phrase (roughly the first 34 characters)
|
|
||||||
labels = vec![LabeledSpan::at(0..34, "\"as of my last knowledge update\"")],
|
|
||||||
help = "If the text contains phrases like \"as an AI model\" or \"as of my \n\
|
|
||||||
last knowledge update\", or if the editor copy‑pasted the \n\
|
|
||||||
prompt used to generate the text together with the AI \n\
|
|
||||||
response, the text is almost certainly AI‑generated.",
|
|
||||||
"Potential AI‑generated content detected"
|
|
||||||
)
|
|
||||||
.with_source_code(source)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
#[derive(Debug, Deserialize)]
|
||||||
// Show the Unicode‑confusion diagnostic
|
struct Pattern {
|
||||||
let report1 = unicode_confusion_report();
|
regex: String,
|
||||||
println!("{:?}\n", report1);
|
replacement: String,
|
||||||
|
}
|
||||||
|
const DEFAULT_TOML: &str = include_str!("default_config.toml");
|
||||||
|
|
||||||
// Show the AI‑generated‑text diagnostic
|
/// Read the user-supplied file or fall back to the default.
|
||||||
let report2 = ai_generated_report();
|
fn load_config(path: &PathBuf) -> Result<Config> {
|
||||||
println!("{:?}", report2);
|
if path.exists() {
|
||||||
|
let data = fs::read_to_string(path)
|
||||||
|
.with_context(|| format!("Could not read config at {:?}", path))?;
|
||||||
|
toml::from_str(&data).with_context(|| "Failed to parse TOML") // ← 2️⃣
|
||||||
|
} else {
|
||||||
|
// If the file is missing, create it from the embedded default.
|
||||||
|
fs::write(path, DEFAULT_TOML)
|
||||||
|
.with_context(|| format!("Could not write default config to {:?}", path))?;
|
||||||
|
// Parse the embedded default to return the same type.
|
||||||
|
toml::from_str(DEFAULT_TOML).with_context(|| "Failed to parse embedded TOML") // ← 3️⃣
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// 1. Parse CLI arguments
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// usage: ./regex_replacer <directory> <config.toml>
|
||||||
|
let cli = Cli::parse();
|
||||||
|
|
||||||
|
let target_dirs = cli.path;
|
||||||
|
let cfg_path = cli.config.unwrap_or("config.toml".into());
|
||||||
|
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// 2. Load and deserialize the TOML config
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// let cfg_contents = std::fs::read_to_string(&cfg_path)
|
||||||
|
// let cfg: Config = toml::from_str(&cfg_contents)
|
||||||
|
// .with_context(|| format!("TOML parsing failed for {:?}", cfg_path))?;
|
||||||
|
let cfg = load_config(&cfg_path)
|
||||||
|
.with_context(|| format!("Could not read config file {:?}", cfg_path))?;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// 3. Compile all regexes once (pre-compute for speed)
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
let compiled_patterns: Result<Vec<(Regex, String)>> = cfg
|
||||||
|
.patterns
|
||||||
|
.into_iter()
|
||||||
|
.map(|p| {
|
||||||
|
let re = Regex::new(&p.regex).with_context(|| format!("Invalid regex: {}", p.regex))?;
|
||||||
|
Ok((re, p.replacement))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let compiled_patterns = compiled_patterns?;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// 4. Walk the directory tree - collect all files
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
let files: Vec<PathBuf> = collect_files(&target_dirs);
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
// 5. Process files in parallel
|
||||||
|
// ---------------------------------------------------------
|
||||||
|
files.par_iter().for_each(|path| {
|
||||||
|
if let Err(err) = process_file(path, &compiled_patterns) {
|
||||||
|
eprintln!("⚠️ Failed on {:?}: {}", path, err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Apply all regex→replacement rules to one file and overwrite it.
|
||||||
|
fn process_file(path: &PathBuf, patterns: &[(Regex, String)]) -> Result<()> {
|
||||||
|
let content =
|
||||||
|
std::fs::read_to_string(path).with_context(|| format!("Failed to read {:?}", path))?;
|
||||||
|
|
||||||
|
// Apply every pattern in order
|
||||||
|
let transformed = patterns.iter().fold(content, |acc, (re, repl)| {
|
||||||
|
re.replace_all(&acc, repl).to_string()
|
||||||
|
});
|
||||||
|
|
||||||
|
// Write the new content back to the file
|
||||||
|
std::fs::write(path, transformed).with_context(|| format!("Failed to write to {:?}", path))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns `true` for every entry that is **not** hidden.
|
||||||
|
fn is_not_hidden(entry: &DirEntry) -> bool {
|
||||||
|
// `entry.file_name()` is a `OsStr`; converting it to a string
|
||||||
|
// is cheap because we only need to check the first byte.
|
||||||
|
!entry.file_name().to_string_lossy().starts_with('.')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collect all regular files that are reachable from *any* directory
|
||||||
|
/// in `target_dirs` and are not hidden.
|
||||||
|
pub fn collect_files(target_dirs: &[PathBuf]) -> Vec<PathBuf> {
|
||||||
|
target_dirs
|
||||||
|
.iter()
|
||||||
|
.flat_map(|root| {
|
||||||
|
// `WalkDir::new(root)` produces an iterator over *all*
|
||||||
|
// entries below `root`.
|
||||||
|
// We apply the same filters that were used in the original
|
||||||
|
// single‑directory version.
|
||||||
|
WalkDir::new(root)
|
||||||
|
.into_iter()
|
||||||
|
.filter_entry(is_not_hidden) // skip hidden directories
|
||||||
|
.filter_map(Result::ok) // ignore I/O errors
|
||||||
|
.filter(|e| e.file_type().is_file()) // keep only files
|
||||||
|
.map(|e| e.path().to_path_buf())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
# Basic ASCII hyphen
|
||||||
|
foo-bar
|
||||||
|
bar-foo
|
||||||
|
|
||||||
|
# Unicode dash variants - all are in the Pd category
|
||||||
|
foo-bar # EN DASH (U+2013)
|
||||||
|
foo-bar # EM DASH (U+2014)
|
||||||
|
foo-bar # HYPHEN-MINUS (U+2010)
|
||||||
|
foo-bar # FIGURE DASH (U+2012)
|
||||||
|
foo--bar # Two EN DASHes side-by-side
|
||||||
|
foo---------bar # Mix of all
|
||||||
|
|
||||||
|
# Other punctuation that *looks* like a dash but is not Pd
|
||||||
|
foo·bar # BULLET (U+00B7)
|
||||||
|
foo/ bar # SOLIDUS (U+002F)
|
||||||
|
foo- bar # EM DASH followed by a non-breaking space (U+00A0)
|
||||||
|
|
||||||
|
# Edge cases
|
||||||
|
foo- -bar # ASCII hyphen, space, en dash, space, bar
|
||||||
|
-foo-bar- # Hyphens at the start and end
|
||||||
|
foo--bar- # Mixed dashes with trailing en dash
|
||||||
|
|
||||||
|
|
||||||
|
# Test words surrounded by dashes - word boundaries
|
||||||
|
-foo-
|
||||||
|
foo-
|
||||||
|
-bar
|
||||||
|
--foo--
|
||||||
|
foo--bar
|
||||||
|
foo--bar
|
||||||
|
foo---bar
|
||||||
|
|
||||||
|
# Dashes embedded in words (should NOT be replaced if you keep word-boundaries)
|
||||||
|
f-oo-b-ar # “f-oo” (hyphen inside a word)
|
||||||
|
fo-ob-ar # “fo-ob” (en dash inside a word)
|
||||||
|
|
||||||
|
# Dashes as punctuation in a sentence
|
||||||
|
"foo - bar" # En dash surrounded by spaces
|
||||||
|
"foo-bar" # No spaces
|
||||||
|
"foo-bar" # EM dash, no spaces
|
||||||
|
"foo-bar" # FIGURE dash
|
||||||
|
"foo-bar" # HYPHEN-MINUS
|
||||||
|
|
||||||
|
# Mix of Unicode whitespace
|
||||||
|
foo - bar # U+202F NARROW NO-BREAK SPACE
|
||||||
|
foo\u2005-bar # U+2005 MEDIUM MATHEMATICAL SPACE
|
||||||
|
|
||||||
|
# Multiple dashes with other punctuation
|
||||||
|
foo--bar # Two ASCII hyphens
|
||||||
|
foo--bar # EM DASH + EM DASH
|
||||||
|
foo---bar # HYPHEN-MINUS + EM DASH + FIGURE DASH
|
||||||
|
|
||||||
|
# Apostrophes
|
||||||
|
Here's a “quote” with a mix of apostrophes: 'Don't' “re-write” 'my' “code”, ʹand ʽtʾe-sʾtʽ! Also, 'like this', or a full-width ', or a low-9 'and a high-reversed 'example'. (End of line.)
|
||||||
|
"`This line should stay intact`"
|
||||||
Loading…
Reference in New Issue