Correct case folding

Case folding had a serious bug causing some code points to fold to the
wrong values. This affected codepoints in FoldRanges with modulo 2 or
more which should not change when case folded, but erroneously did.

The affected code points are:

The 24 odd code points in the range U+100 - U+12E
The 3 odd code points in the range U+132 - U+136
The 7 even code points in the range U+139 - U+147
The 23 odd code points in the range U+14A - U+176
The 3 odd code points in the range U+1A0 - U+1A4
The 8 even code points in the range U+1CB - U+1DB
The 9 odd code points in the range U+1DE - U+1EE
The 20 odd code points in the range U+1F8 - U+21E
The 9 odd code points in the range U+222 - U+232
The 5 odd code points in the range U+246 - U+24E
The 12 odd code points in the range U+3D8 - U+3EE
The 17 odd code points in the range U+460 - U+480
The 27 odd code points in the range U+48A - U+4BE
The 6 even code points in the range U+4C1 - U+4CD
The 48 odd code points in the range U+4D0 - U+52E
The 75 odd code points in the range U+1E00 - U+1E94
The 48 odd code points in the range U+1EA0 - U+1EFE
The 3 even code points in the range U+1F59 - U+1F5F
The 50 odd code points in the range U+2C80 - U+2CE2
The 23 odd code points in the range U+A640 - U+A66C
The 14 odd code points in the range U+A680 - U+A69A
The 7 odd code points in the range U+A722 - U+A72E
The 31 odd code points in the range U+A732 - U+A76E
The 5 odd code points in the range U+A77E - U+A786
The 10 odd code points in the range U+A796 - U+A7A8
The 8 odd code points in the range U+A7B4 - U+A7C2
The code points U+17A, U+17C, U+182, U+184, U+1B4, U+1B9, U+1BA, U+1BB,
U+1F2, U+1F4, U+370, U+372, U+2C68, U+2C6A, U+2CEC, U+A77A, U+A790, U+A792,
U+A7C8, U+A7D6, U+A7D8

Author: ridiculousfish

src/unicode.rs

@@ -4,6 +4,8 @@ use crate::util::SliceHelp;
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
 use core::cmp::Ordering;
+#[cfg(test)]
+use std::collections::HashMap;
 
 // CodePointRange packs a code point and a length together into a u32.
 // We currently do not need to store any information about code points in plane 16 (U+100000),
@@ -100,7 +102,8 @@ pub(crate) struct FoldRange {
 impl FoldRange {
     #[inline(always)]
     pub const fn from(start: u32, length: u32, delta: i32, modulo: u8) -> Self {
-        let mask = (1 << modulo) - 1;
+        const_assert_true!(modulo.is_power_of_two());
+        let mask = (modulo - 1) as i32;
         const_assert_true!(mask < (1 << PREDICATE_MASK_BITS));
         const_assert_true!(((delta << PREDICATE_MASK_BITS) >> PREDICATE_MASK_BITS) == delta);
         let extra = mask | (delta << PREDICATE_MASK_BITS);
@@ -126,7 +129,7 @@ impl FoldRange {
 
     #[inline(always)]
     fn predicate_mask(&self) -> u32 {
-        (self.extra as u32) & PREDICATE_MASK_BITS
+        (self.extra as u32) & ((1 << PREDICATE_MASK_BITS) - 1)
     }
 
     fn add_delta(&self, cu: u32) -> u32 {
@@ -346,3 +349,71 @@ pub(crate) fn is_character_class(c: u32, property_escape: &PropertyEscape) -> bo
         false
     }
 }
+
+#[test]
+fn test_folds() {
+    for c in 0..0x41 {
+        assert_eq!(fold(c), c);
+    }
+    for c in 0x41..=0x5A {
+        assert_eq!(fold(c), c + 0x20);
+    }
+    assert_eq!(fold(0xB5), 0x3BC);
+    assert_eq!(fold(0xC0), 0xE0);
+
+    assert_eq!(fold(0x1B8), 0x1B9);
+    assert_eq!(fold(0x1B9), 0x1B9);
+    assert_eq!(fold(0x1BA), 0x1BA);
+    assert_eq!(fold(0x1BB), 0x1BB);
+    assert_eq!(fold(0x1BC), 0x1BD);
+    assert_eq!(fold(0x1BD), 0x1BD);
+
+    for c in 0x1F8..0x21F {
+        if c % 2 == 0 {
+            assert_eq!(fold(c), c + 1);
+        } else {
+            assert_eq!(fold(c), c);
+        }
+    }
+
+    assert_eq!(fold(0x37F), 0x3F3);
+    assert_eq!(fold(0x380), 0x380);
+    assert_eq!(fold(0x16E40), 0x16E60);
+    assert_eq!(fold(0x16E41), 0x16E61);
+    assert_eq!(fold(0x16E42), 0x16E62);
+    assert_eq!(fold(0x1E900), 0x1E922);
+    assert_eq!(fold(0x1E901), 0x1E923);
+    for c in 0xF0000..=0x10FFFF {
+        assert_eq!(fold(c), c);
+    }
+}
+
+#[test]
+fn test_fold_idempotent() {
+    for c in 0..=0x10FFFF {
+        let fc = fold(c);
+        let ffc = fold(fc);
+        assert_eq!(ffc, fc);
+    }
+}
+
+#[test]
+fn test_unfold_chars() {
+    // Map from folded char to the chars that folded to it.
+    let mut fold_map: HashMap<u32, Vec<u32>> = HashMap::new();
+    for c in 0..=0x10FFFF {
+        let fc = fold(c);
+        fold_map.entry(fc).or_insert_with(Vec::new).push(c);
+    }
+
+    // Sort them all.
+    for v in fold_map.values_mut() {
+        v.sort_unstable();
+    }
+
+    for c in 0..=0x10FFFF {
+        let mut unfolded = unfold_char(c);
+        unfolded.sort_unstable();
+        assert_eq!(unfolded, fold_map[&fold(c)]);
+    }
+}

tests/tests.rs

@@ -1537,3 +1537,46 @@ fn test_escapes_folding_tc(tc: TestConfig) {
     tc.test_match_succeeds(r"\u{61}", "i", "a");
     tc.test_match_succeeds(r"\u{61}", "i", "A");
 }
+
+#[test]
+fn test_high_folds() {
+    test_with_configs(test_high_folds_tc)
+}
+
+fn test_high_folds_tc(tc: TestConfig) {
+    // Regression test for bogus folding.
+    // We incorrectly folded certain characters in delta blocks:
+    // we folded U+100 to U+101 (correctly) but then U+101 to U+102 (wrong).
+    tc.test_match_succeeds(r"\u{100}", "", "\u{100}");
+    tc.test_match_succeeds(r"\u{100}", "i", "\u{100}");
+
+    tc.test_match_fails(r"\u{100}", "", "\u{101}");
+    tc.test_match_succeeds(r"\u{100}", "i", "\u{101}");
+
+    tc.test_match_succeeds(r"\u{101}", "", "\u{101}");
+    tc.test_match_succeeds(r"\u{101}", "i", "\u{101}");
+
+    tc.test_match_fails(r"\u{101}", "", "\u{102}");
+    tc.test_match_fails(r"\u{101}", "i", "\u{102}");
+
+    // Exercise a "mod 4 range":
+    //   U+1B8 folds to U+1B9
+    //   U+1BC folds to U+1BD
+    // Codepoints between fold to themselves.
+    tc.test_match_succeeds(r"\u{1B8}", "", "\u{1B8}");
+    tc.test_match_succeeds(r"\u{1B8}", "i", "\u{1B8}");
+    tc.test_match_fails(r"\u{1B8}", "", "\u{1B9}");
+    tc.test_match_succeeds(r"\u{1B8}", "i", "\u{1B9}");
+    tc.test_match_succeeds(r"\u{1B9}", "", "\u{1B9}");
+    tc.test_match_succeeds(r"\u{1B9}", "i", "\u{1B9}");
+    tc.test_match_fails(r"\u{1B9}", "", "\u{1BA}");
+    tc.test_match_fails(r"\u{1B9}", "i", "\u{1BA}");
+    tc.test_match_succeeds(r"\u{1BC}", "", "\u{1BC}");
+    tc.test_match_succeeds(r"\u{1BC}", "i", "\u{1BC}");
+    tc.test_match_fails(r"\u{1BC}", "", "\u{1BD}");
+    tc.test_match_succeeds(r"\u{1BC}", "i", "\u{1BD}");
+    tc.test_match_succeeds(r"\u{1BD}", "", "\u{1BD}");
+    tc.test_match_succeeds(r"\u{1BD}", "i", "\u{1BD}");
+    tc.test_match_fails(r"\u{1BD}", "", "\u{1BE}");
+    tc.test_match_fails(r"\u{1BD}", "i", "\u{1BE}");
+}