test: correct Unicode offset expectations
This commit is contained in:
parent
acfb775877
commit
4d142d60ed
|
|
@ -34,6 +34,10 @@ cargo doc --no-deps
|
|||
Behavior is intentionally captured with regression tests before the refactor so
|
||||
that later changes are deliberate and reviewable.
|
||||
|
||||
Detector spans use zero-based UTF-8 byte offsets. Presentation layers will
|
||||
convert those canonical offsets to terminal or LSP coordinates at their
|
||||
boundaries.
|
||||
|
||||
## Background reading
|
||||
|
||||
- [Wikipedia: Signs of AI writing](https://en.wikipedia.org/wiki/Wikipedia%3ASigns_of_AI_writing)
|
||||
|
|
|
|||
|
|
@ -36,11 +36,8 @@ mod tests {
|
|||
#[test]
|
||||
fn it_finds_all_hyphens() {
|
||||
let s = "a‑b—c‑—d‑";
|
||||
// The string contains the following code points:
|
||||
// U+2011 (NON‑BREAKING HYPHEN) -> a‑b
|
||||
// U+2014 (EM DASH) -> —c
|
||||
// U+2011 again -> d‑
|
||||
// U+2014 again -> —
|
||||
// The string contains U+2011, U+2014, U+2011, U+2014, and U+2011 in
|
||||
// that order. Each matched code point occupies three UTF-8 bytes.
|
||||
let mut results = find_all(&HYPHEN_REGEX, s);
|
||||
|
||||
// sort to make the assertion deterministic
|
||||
|
|
@ -76,17 +73,16 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn grapheme_positions_are_correct() {
|
||||
// A string with a single grapheme cluster that is two bytes in UTF‑8.
|
||||
let s = "a𝌆b"; // 𝌆 = U+1D306, 3 bytes
|
||||
fn regex_offsets_are_utf8_byte_offsets() {
|
||||
// U+1D306 is one grapheme cluster and occupies four bytes in UTF-8.
|
||||
let s = "a𝌆b";
|
||||
|
||||
// Find the middle grapheme (the diamond).
|
||||
let diamond_start = s.grapheme_indices(true).nth(1).unwrap().0;
|
||||
assert_eq!(diamond_start, 1); // the byte offset of the start of 𝌆
|
||||
let symbol_start = s.grapheme_indices(true).nth(1).unwrap().0;
|
||||
assert_eq!(symbol_start, 1);
|
||||
|
||||
// Find it with our regex (if we had a regex that matches 𝌆).
|
||||
let diamond_re = Regex::new(r"\u{1D306}").unwrap();
|
||||
let m = diamond_re.find(s).unwrap();
|
||||
assert_eq!(m.start(), diamond_start);
|
||||
let symbol_regex = Regex::new(r"\u{1D306}").unwrap();
|
||||
let matched = symbol_regex.find(s).unwrap();
|
||||
assert_eq!(matched.range(), 1..5);
|
||||
assert_eq!(matched.start(), symbol_start);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -26,7 +26,7 @@ audit in CI.
|
|||
- [x] Capture current behavior with tests before moving code: plain file,
|
||||
recursive directory, ZIP member, invalid UTF-8, missing path, and basic Unicode
|
||||
matching.
|
||||
- [ ] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme
|
||||
- [x] Fix the misleading Unicode tests/comments (UTF-8 byte lengths and grapheme
|
||||
terminology) so test names reflect what is actually asserted.
|
||||
- [ ] Record initial decisions in `docs/decisions/`: evidence confidence is not
|
||||
authorship probability; UTF-8 byte spans are canonical; formatter writes are
|
||||
|
|
|
|||
Loading…
Reference in New Issue