test: correct Unicode offset expectations

This commit is contained in:
Dandaedre 2026-09-29 12:10:17 +02:00
parent acfb775877
commit 4d142d60ed
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
3 changed files with 16 additions and 16 deletions

View File

@ -34,6 +34,10 @@ cargo doc --no-deps
Behavior is intentionally captured with regression tests before the refactor so
that later changes are deliberate and reviewable.
Detector spans use zero-based UTF-8 byte offsets. Presentation layers will
convert those canonical offsets to terminal or LSP coordinates at their
boundaries.
## Background reading
- [Wikipedia: Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia%3ASigns_of_AI_writing)

View File

@ -36,11 +36,8 @@ mod tests {
#[test]
fn it_finds_all_hyphens() {
let s = "a‑b—c‑—d‑";
// The string contains the following code points:
// U+2011 (NON‑BREAKING HYPHEN) -> a‑b
// U+2014 (EM DASH) -> —c
// U+2011 again -> d‑
// U+2014 again -> —
// The string contains U+2011, U+2014, U+2011, U+2014, and U+2011 in
// that order. Each matched code point occupies three UTF-8 bytes.
let mut results = find_all(&HYPHEN_REGEX, s);
// sort to make the assertion deterministic
@ -76,17 +73,16 @@ mod tests {
}
#[test]
fn grapheme_positions_are_correct() {
// A string with a single grapheme cluster that is two bytes in UTF‑8.
let s = "a𝌆b"; // 𝌆 = U+1D306, 3 bytes
fn regex_offsets_are_utf8_byte_offsets() {
// U+1D306 is one grapheme cluster and occupies four bytes in UTF-8.
let s = "a𝌆b";
// Find the middle grapheme (the diamond).
let diamond_start = s.grapheme_indices(true).nth(1).unwrap().0;
assert_eq!(diamond_start, 1); // the byte offset of the start of 𝌆
let symbol_start = s.grapheme_indices(true).nth(1).unwrap().0;
assert_eq!(symbol_start, 1);
// Find it with our regex (if we had a regex that matches 𝌆).
let diamond_re = Regex::new(r"\u{1D306}").unwrap();
let m = diamond_re.find(s).unwrap();
assert_eq!(m.start(), diamond_start);
let symbol_regex = Regex::new(r"\u{1D306}").unwrap();
let matched = symbol_regex.find(s).unwrap();
assert_eq!(matched.range(), 1..5);
assert_eq!(matched.start(), symbol_start);
}
}

View File

@ -26,7 +26,7 @@ audit in CI.
- [x] Capture current behavior with tests before moving code: plain file,
recursive directory, ZIP member, invalid UTF-8, missing path, and basic Unicode
matching.
- [ ] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme
- [x] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme
terminology) so test names reflect what is actually asserted.
- [ ] Record initial decisions in `docs/decisions/`: evidence confidence is not
authorship probability; UTF-8 byte spans are canonical; formatter writes are