feat: harden replacement engine
Rust CI JohnSlop passed all CI checks.
Gitea/john_slop/pipeline/head This commit looks good
Details
Rust CI JohnSlop passed all CI checks.
Gitea/john_slop/pipeline/head This commit looks good
Details
This commit is contained in:
parent
1bacd8a75d
commit
f1daf590f4
|
|
@ -10,8 +10,9 @@ max_archive_entries = 10000
|
|||
max_compression_ratio = 100
|
||||
max_walk_depth = 64
|
||||
|
||||
# A replacement entry has a stable lowercase kebab-case ID, exactly one source
|
||||
# character, an ASCII target, a rationale, and an enabled switch. Example:
|
||||
# A replacement entry has a stable lowercase kebab-case ID, non-empty Unicode
|
||||
# source text, an ASCII target, a rationale, and an enabled switch. When enabled
|
||||
# sources share a prefix, the longest source wins. Example:
|
||||
# [[replacements]]
|
||||
# id = "example-symbol"
|
||||
# source = "→"
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
//! Inspect the edits shared by detector, formatter, and future LSP adapters.
|
||||
|
||||
use john_slop::{Config, plan_replacements};
|
||||
use john_slop::{Config, apply_replacements, plan_replacements};
|
||||
|
||||
fn main() {
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let source = "Review—then subtract 5−2.";
|
||||
let edits = plan_replacements(&Config::embedded(), source);
|
||||
|
||||
println!("source: {source}");
|
||||
for edit in edits {
|
||||
for edit in &edits {
|
||||
println!(
|
||||
"replace bytes {}..{} with {:?}",
|
||||
edit.byte_span.start, edit.byte_span.end, edit.replacement
|
||||
);
|
||||
}
|
||||
println!("result: {}", apply_replacements(source, &edits)?);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -305,11 +305,17 @@ pub enum ConfigValidationError {
|
|||
/// Repeated ID.
|
||||
id: String,
|
||||
},
|
||||
/// Two replacement rules target the same source character.
|
||||
#[error("replacement source `{character}` is configured more than once")]
|
||||
/// Two replacement rules target the same source text.
|
||||
#[error("replacement source `{text}` is configured more than once")]
|
||||
DuplicateReplacementSource {
|
||||
/// Conflicting source character.
|
||||
character: char,
|
||||
/// Conflicting source text.
|
||||
text: String,
|
||||
},
|
||||
/// A replacement source is empty or contains only ASCII.
|
||||
#[error("replacement `{id}` must have a non-empty, non-ASCII source")]
|
||||
InvalidReplacementSource {
|
||||
/// Replacement rule ID.
|
||||
id: String,
|
||||
},
|
||||
/// A replacement target is empty or contains non-ASCII text.
|
||||
#[error("replacement `{id}` must have a non-empty ASCII target")]
|
||||
|
|
@ -511,7 +517,7 @@ struct ThresholdLayer {
|
|||
#[serde(deny_unknown_fields)]
|
||||
struct ReplacementPatch {
|
||||
id: String,
|
||||
source: Option<char>,
|
||||
source: Option<String>,
|
||||
replacement: Option<String>,
|
||||
rationale: Option<String>,
|
||||
enabled: Option<bool>,
|
||||
|
|
@ -743,8 +749,8 @@ impl ConfigPaths {
|
|||
pub struct ReplacementRule {
|
||||
/// Stable identifier used by diagnostics and overrides.
|
||||
pub id: String,
|
||||
/// Unicode character to replace.
|
||||
pub source: char,
|
||||
/// Non-empty Unicode text to replace.
|
||||
pub source: String,
|
||||
/// ASCII replacement text.
|
||||
pub replacement: String,
|
||||
/// Why this replacement is safe or desirable.
|
||||
|
|
@ -956,9 +962,14 @@ fn validate_replacements(replacements: &[ReplacementRule]) -> Result<(), ConfigV
|
|||
id: rule.id.clone(),
|
||||
});
|
||||
}
|
||||
if !sources.insert(rule.source) {
|
||||
if rule.source.is_empty() || rule.source.is_ascii() {
|
||||
return Err(ConfigValidationError::InvalidReplacementSource {
|
||||
id: rule.id.clone(),
|
||||
});
|
||||
}
|
||||
if !sources.insert(rule.source.as_str()) {
|
||||
return Err(ConfigValidationError::DuplicateReplacementSource {
|
||||
character: rule.source,
|
||||
text: rule.source.clone(),
|
||||
});
|
||||
}
|
||||
if rule.replacement.is_empty() || !rule.replacement.is_ascii() {
|
||||
|
|
@ -1109,7 +1120,7 @@ mod tests {
|
|||
let sources = config
|
||||
.replacements
|
||||
.iter()
|
||||
.map(|rule| rule.source)
|
||||
.map(|rule| rule.source.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(config.replacements.len(), 10);
|
||||
|
|
@ -1279,7 +1290,7 @@ weight = 9.5
|
|||
.find(|rule| rule.id == "em-dash")
|
||||
.unwrap();
|
||||
assert!(!replacement.enabled);
|
||||
assert_eq!(replacement.source, '—');
|
||||
assert_eq!(replacement.source, "—");
|
||||
assert_eq!(replacement.replacement, "-");
|
||||
assert_eq!(config.replacements.last().unwrap().id, "right-arrow");
|
||||
assert_eq!(config.rules.len(), 11);
|
||||
|
|
@ -1302,12 +1313,19 @@ weight = 9.5
|
|||
));
|
||||
|
||||
let mut config = Config::embedded();
|
||||
config.replacements[1].source = config.replacements[0].source;
|
||||
config.replacements[1].source = config.replacements[0].source.clone();
|
||||
assert!(matches!(
|
||||
config.validate(),
|
||||
Err(ConfigValidationError::DuplicateReplacementSource { .. })
|
||||
));
|
||||
|
||||
let mut config = Config::embedded();
|
||||
config.replacements[0].source.clear();
|
||||
assert!(matches!(
|
||||
config.validate(),
|
||||
Err(ConfigValidationError::InvalidReplacementSource { .. })
|
||||
));
|
||||
|
||||
let mut config = Config::embedded();
|
||||
config.replacements[0].replacement = "—".to_owned();
|
||||
assert!(matches!(
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ pub use model::{
|
|||
Analysis, AnalysisSummary, Category, Confidence, ConfidenceBand, Finding, FindingSeverity,
|
||||
RuleId, SeverityCounts, SourceId, TextEdit,
|
||||
};
|
||||
pub use replacement::ReplacementError;
|
||||
|
||||
/// Load embedded defaults plus optional platform user configuration.
|
||||
pub fn load_config() -> Result<Config, ConfigError> {
|
||||
|
|
@ -45,6 +46,14 @@ pub fn plan_replacements(config: &Config, source: &str) -> Vec<TextEdit> {
|
|||
replacement::plan(config, source)
|
||||
}
|
||||
|
||||
/// Validate and apply text edits against their original UTF-8 source snapshot.
|
||||
pub fn apply_replacements<'source>(
|
||||
source: &'source str,
|
||||
edits: &[TextEdit],
|
||||
) -> Result<std::borrow::Cow<'source, str>, ReplacementError> {
|
||||
replacement::apply(source, edits)
|
||||
}
|
||||
|
||||
/// Read documents from a file, directory, or supported ZIP-based container.
|
||||
pub fn read_inputs(config: &Config, path: &Path) -> InputResult<Documents> {
|
||||
input::read(path, &config.input)
|
||||
|
|
|
|||
|
|
@ -1,22 +1,120 @@
|
|||
//! Pure planning of configured text replacements.
|
||||
//! Pure planning and application of configured text replacements.
|
||||
|
||||
use std::{borrow::Cow, ops::Range};
|
||||
|
||||
use miette::Diagnostic;
|
||||
|
||||
use crate::{Config, TextEdit};
|
||||
|
||||
/// An edit set cannot be applied safely to its source snapshot.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error, Diagnostic)]
|
||||
#[diagnostic(code(john_slop::replacement::invalid_edit))]
|
||||
pub enum ReplacementError {
|
||||
/// A range is empty, reversed, or extends beyond the source.
|
||||
#[error("replacement byte range {start}..{end} is invalid for a {source_len}-byte source")]
|
||||
InvalidRange {
|
||||
/// Inclusive start byte supplied by the edit.
|
||||
start: usize,
|
||||
/// Exclusive end byte supplied by the edit.
|
||||
end: usize,
|
||||
/// Source snapshot length in bytes.
|
||||
source_len: usize,
|
||||
},
|
||||
/// A range endpoint does not lie on a UTF-8 character boundary.
|
||||
#[error("replacement byte offset {offset} is not a UTF-8 character boundary")]
|
||||
InvalidCharacterBoundary {
|
||||
/// Invalid endpoint.
|
||||
offset: usize,
|
||||
},
|
||||
/// Two edits cover at least one common source byte.
|
||||
#[error("replacement ranges overlap at byte {offset}")]
|
||||
OverlappingEdits {
|
||||
/// Start of the later edit inside an earlier edit.
|
||||
offset: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// Plan deterministic non-overlapping edits without modifying the source.
|
||||
pub fn plan(config: &Config, source: &str) -> Vec<TextEdit> {
|
||||
source
|
||||
.char_indices()
|
||||
.filter_map(|(start, character)| {
|
||||
let rule = config
|
||||
.replacements
|
||||
.iter()
|
||||
.find(|rule| rule.enabled && rule.source == character)?;
|
||||
Some(TextEdit {
|
||||
byte_span: start..start + character.len_utf8(),
|
||||
let mut rules = config
|
||||
.replacements
|
||||
.iter()
|
||||
.filter(|rule| rule.enabled)
|
||||
.collect::<Vec<_>>();
|
||||
rules.sort_by(|left, right| {
|
||||
right
|
||||
.source
|
||||
.len()
|
||||
.cmp(&left.source.len())
|
||||
.then_with(|| left.id.cmp(&right.id))
|
||||
});
|
||||
|
||||
let mut edits = Vec::new();
|
||||
let mut offset = 0;
|
||||
while offset < source.len() {
|
||||
if let Some(rule) = rules
|
||||
.iter()
|
||||
.find(|rule| source[offset..].starts_with(&rule.source))
|
||||
{
|
||||
let end = offset + rule.source.len();
|
||||
edits.push(TextEdit {
|
||||
byte_span: offset..end,
|
||||
replacement: rule.replacement.clone(),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
offset = end;
|
||||
} else {
|
||||
offset += source[offset..].chars().next().map_or(0, char::len_utf8);
|
||||
}
|
||||
}
|
||||
edits
|
||||
}
|
||||
|
||||
/// Validate and apply edits from the end of the source toward the beginning.
|
||||
///
|
||||
/// With no edits, the returned [`Cow`] borrows the original source without an
|
||||
/// allocation. All ranges refer to the original source snapshot.
|
||||
pub fn apply<'source>(
|
||||
source: &'source str,
|
||||
edits: &[TextEdit],
|
||||
) -> Result<Cow<'source, str>, ReplacementError> {
|
||||
if edits.is_empty() {
|
||||
return Ok(Cow::Borrowed(source));
|
||||
}
|
||||
|
||||
let mut ordered = edits.iter().collect::<Vec<_>>();
|
||||
ordered.sort_by(|left, right| range_order(&left.byte_span, &right.byte_span));
|
||||
let mut previous_end = 0;
|
||||
for edit in &ordered {
|
||||
let range = &edit.byte_span;
|
||||
if range.start >= range.end || range.end > source.len() {
|
||||
return Err(ReplacementError::InvalidRange {
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
source_len: source.len(),
|
||||
});
|
||||
}
|
||||
for offset in [range.start, range.end] {
|
||||
if !source.is_char_boundary(offset) {
|
||||
return Err(ReplacementError::InvalidCharacterBoundary { offset });
|
||||
}
|
||||
}
|
||||
if range.start < previous_end {
|
||||
return Err(ReplacementError::OverlappingEdits {
|
||||
offset: range.start,
|
||||
});
|
||||
}
|
||||
previous_end = range.end;
|
||||
}
|
||||
|
||||
let mut output = source.to_owned();
|
||||
for edit in ordered.into_iter().rev() {
|
||||
output.replace_range(edit.byte_span.clone(), &edit.replacement);
|
||||
}
|
||||
Ok(Cow::Owned(output))
|
||||
}
|
||||
|
||||
fn range_order(left: &Range<usize>, right: &Range<usize>) -> std::cmp::Ordering {
|
||||
(left.start, left.end).cmp(&(right.start, right.end))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -58,4 +156,82 @@ mod tests {
|
|||
assert!(plan(&config, "").is_empty());
|
||||
assert!(plan(&config, "ordinary-ascii").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longest_source_wins_independently_of_configuration_order() {
|
||||
let mut config = Config::embedded();
|
||||
config.replacements.insert(
|
||||
0,
|
||||
crate::ReplacementRule {
|
||||
id: "double-em-dash".to_owned(),
|
||||
source: "——".to_owned(),
|
||||
replacement: "--".to_owned(),
|
||||
rationale: "Preserve a doubled dash.".to_owned(),
|
||||
enabled: true,
|
||||
},
|
||||
);
|
||||
config.validate().unwrap();
|
||||
|
||||
let edits = plan(&config, "a——b—c");
|
||||
|
||||
assert_eq!(edits.len(), 2);
|
||||
assert_eq!(edits[0].byte_span, 1..7);
|
||||
assert_eq!(edits[0].replacement, "--");
|
||||
assert_eq!(edits[1].byte_span, 8..11);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_edits_in_reverse_and_preserves_line_endings() {
|
||||
let source = "a—b\r\nc−d\n";
|
||||
let edits = plan(&Config::embedded(), source);
|
||||
|
||||
assert_eq!(apply(source, &edits).unwrap(), "a-b\r\nc-d\n");
|
||||
assert_eq!(
|
||||
apply(&apply(source, &edits).unwrap(), &[]).unwrap(),
|
||||
"a-b\r\nc-d\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_edits_borrow_the_original_source() {
|
||||
let source = "ordinary ASCII";
|
||||
|
||||
assert!(matches!(apply(source, &[]).unwrap(), Cow::Borrowed(value) if value == source));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_boundaries_ranges_and_overlaps() {
|
||||
let invalid_boundary = [TextEdit {
|
||||
byte_span: 1..2,
|
||||
replacement: "-".to_owned(),
|
||||
}];
|
||||
assert!(matches!(
|
||||
apply("é", &invalid_boundary),
|
||||
Err(ReplacementError::InvalidCharacterBoundary { offset: 1 })
|
||||
));
|
||||
|
||||
let invalid_range = [TextEdit {
|
||||
byte_span: 0..3,
|
||||
replacement: "-".to_owned(),
|
||||
}];
|
||||
assert!(matches!(
|
||||
apply("ab", &invalid_range),
|
||||
Err(ReplacementError::InvalidRange { .. })
|
||||
));
|
||||
|
||||
let overlapping = [
|
||||
TextEdit {
|
||||
byte_span: 0..2,
|
||||
replacement: "x".to_owned(),
|
||||
},
|
||||
TextEdit {
|
||||
byte_span: 1..3,
|
||||
replacement: "y".to_owned(),
|
||||
},
|
||||
];
|
||||
assert!(matches!(
|
||||
apply("abc", &overlapping),
|
||||
Err(ReplacementError::OverlappingEdits { offset: 1 })
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::{collections::BTreeSet, fs, path::Path, process::Command};
|
||||
|
||||
use john_slop::{AnalysisContext, Config, Detector, plan_replacements};
|
||||
use john_slop::{AnalysisContext, Config, Detector, apply_replacements, plan_replacements};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
|
@ -24,11 +24,9 @@ fn load_manifest(root: &Path) -> Manifest {
|
|||
}
|
||||
|
||||
fn apply_planned_edits(source: &str, config: &Config) -> String {
|
||||
let mut output = source.to_owned();
|
||||
for edit in plan_replacements(config, source).into_iter().rev() {
|
||||
output.replace_range(edit.byte_span, &edit.replacement);
|
||||
}
|
||||
output
|
||||
apply_replacements(source, &plan_replacements(config, source))
|
||||
.expect("apply valid planned edits")
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in New Issue