Merge pull request #106 from Earlopain/fix-utf8-escapes-in-sets

Fix UTF8 escapes in character classes

Author: jaynetics

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- correctly emit backslash-escaped UTF8 characters in character classes as one token (#104)
+
 ## [2.11.2] - 2025-08-12 - Janosch Müller
 
 ### Added

lib/regexp_parser/scanner/scanner.rl

@@ -247,7 +247,7 @@
     # Treat all remaining escapes - those not supported in sets - as literal.
     # (This currently includes \^, \-, \&, \:, although these could potentially
     # be meta chars when not escaped, depending on their position in the set.)
-    any > (escaped_set_alpha, 1) {
+    any > (escaped_set_alpha, 1) | utf8_multibyte > (escaped_set_alpha, 1) {
       emit(:escape, :literal, copy(data, ts-1, te))
       fret;
     };

spec/scanner/sets_spec.rb

@@ -66,6 +66,7 @@
   include_examples 'scan', '[\R]',                  1 => [:escape, :literal,         '\R',         1, 3]
   include_examples 'scan', '[\X]',                  1 => [:escape, :literal,         '\X',         1, 3]
   include_examples 'scan', '[\B]',                  1 => [:escape, :literal,         '\B',         1, 3]
+  include_examples 'scan', '[\💎]',                 1 => [:escape, :literal,         '\💎',        1, 3]
 
   include_examples 'scan', /[\d]/,                  1 => [:type,   :digit,           '\d',         1, 3]
   include_examples 'scan', /[\da-z]/,               1 => [:type,   :digit,           '\d',         1, 3]