From 4d142d60ed2ef5e519178846593a1ec0fb49212c Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 29 Sep 2026 12:10:17 +0200 Subject: [PATCH] test: correct Unicode offset expectations --- README.md | 4 ++++ src/detector.rs | 26 +++++++++++--------------- tasks.md | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 36dac38..4c7a93d 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ cargo doc --no-deps Behavior is intentionally captured with regression tests before the refactor so that later changes are deliberate and reviewable. +Detector spans use zero-based UTF-8 byte offsets. Presentation layers will +convert those canonical offsets to terminal or LSP coordinates at their +boundaries. + ## Background reading - [Wikipedia: Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia%3ASigns_of_AI_writing) diff --git a/src/detector.rs b/src/detector.rs index 68e4e83..f08fdbc 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -36,11 +36,8 @@ mod tests { #[test] fn it_finds_all_hyphens() { let s = "a‑b—c‑—d‑"; - // The string contains the following code points: - // U+2011 (NON‑BREAKING HYPHEN) -> a‑b - // U+2014 (EM DASH) -> —c - // U+2011 again -> d‑ - // U+2014 again -> — + // The string contains U+2011, U+2014, U+2011, U+2014, and U+2011 in + // that order. Each matched code point occupies three UTF-8 bytes. let mut results = find_all(&HYPHEN_REGEX, s); // sort to make the assertion deterministic @@ -76,17 +73,16 @@ mod tests { } #[test] - fn grapheme_positions_are_correct() { - // A string with a single grapheme cluster that is two bytes in UTF‑8. - let s = "a𝌆b"; // 𝌆 = U+1D306, 3 bytes + fn regex_offsets_are_utf8_byte_offsets() { + // U+1D306 is one grapheme cluster and occupies four bytes in UTF-8. + let s = "a𝌆b"; - // Find the middle grapheme (the diamond). - let diamond_start = s.grapheme_indices(true).nth(1).unwrap().0; - assert_eq!(diamond_start, 1); // the byte offset of the start of 𝌆 + let symbol_start = s.grapheme_indices(true).nth(1).unwrap().0; + assert_eq!(symbol_start, 1); - // Find it with our regex (if we had a regex that matches 𝌆). - let diamond_re = Regex::new(r"\u{1D306}").unwrap(); - let m = diamond_re.find(s).unwrap(); - assert_eq!(m.start(), diamond_start); + let symbol_regex = Regex::new(r"\u{1D306}").unwrap(); + let matched = symbol_regex.find(s).unwrap(); + assert_eq!(matched.range(), 1..5); + assert_eq!(matched.start(), symbol_start); } } diff --git a/tasks.md b/tasks.md index bd3ebce..bd4900f 100644 --- a/tasks.md +++ b/tasks.md @@ -26,7 +26,7 @@ audit in CI. - [x] Capture current behavior with tests before moving code: plain file, recursive directory, ZIP member, invalid UTF-8, missing path, and basic Unicode matching. -- [ ] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme +- [x] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme terminology) so test names reflect what is actually asserted. - [ ] Record initial decisions in `docs/decisions/`: evidence confidence is not authorship probability; UTF-8 byte spans are canonical; formatter writes are