[eggex] Separate range checks for bytes and code points

- disallow NUL byte \y00
  - aka code point 0 \u{0}
- keep the ASCII limitation on bytes, so 128-255 are disallowed
- relax limitation on code points - we can match arbitrary code points
  now
  - because of the recent setlocale() change, which checks that we the libc locale is UTF-8
  - if OILS_LOCALE_OK, then all bets are off

TODO: document this in doc/eggex.md, and the reference

Author: Andy C

spec/ysh-regex-bytes-chars.test.sh

@@ -130,7 +130,8 @@ no
 
 #### Can't match code points \u{ff} because they don't translate to valid ERE
 shopt -s ysh:all
-var pat2 = /[ \u{00} - \u{ff} ]/;
+
+var pat2 = /[ \u{1} - \u{3bc} ]/;
 
 # This causes an error
 echo $pat2
@@ -141,8 +142,8 @@ echo $pat2
 var pat1 = /[ \u{ff} ]/;
 
 echo $pat1 | od -A n -t x1
-if (b'\x7f' ~ pat) { echo yes } else { echo no }
-if (b'\x7e' ~ pat) { echo yes } else { echo no }
+if (b'\y7f' ~ pat1) { echo yes } else { echo no }
+if (b'\y7e' ~ pat1) { echo yes } else { echo no }
 
 ## status: 1
 ## stdout-json: ""
@@ -209,21 +210,16 @@ shopt -s ysh:all
 # ascii works
 pp test_ (b'\y7f' ~ / [\x7f] /)
 pp test_ (b'\y7e' ~ / [\x7f] /)
-echo
-
-# BUG before disabling: high byte doesn't work?    Because of utf-8 locale
-#
-# We translate \x80 to a the byte b'\y80', but it still doesn't work
-#
-# We COULD allow this with LANG=C, but for now don't bother.  I think using an
-# ERE string is probably better.
 
-pp test_ (b'\y80' ~ / [\x80] /)
-pp test_ (b'\yff' ~ / [\xff] /)
+= str( / [\y80]/ )
 
-= str( / [\x80]/ )
+#pp test_ (b'\y80' ~ / [\y80] /)
+#pp test_ (b'\yff' ~ / [\yff] /)
 
+## status: 1
 ## STDOUT:
+(Bool)   true
+(Bool)   false
 ## END
 
 #### High bytes 0x80 0xff can be matched with plain ERE and LC_ALL=C
@@ -245,3 +241,39 @@ EOF
 (Bool)   true
 (Bool)   false
 ## END
+
+#### Code points like \u{3bc} can be matched
+
+var pat = / [\u{3bc}] /
+pp test_ (b'a' ~ pat)
+pp test_ (b'\u{3bc}' ~ / [\u{3bc}] /)
+echo
+
+var pat = / [\u{10ffff}] /
+pp test_ (b'a' ~ pat)
+pp test_ (b'\u{10ffff}' ~ pat)
+
+#echo "-$pat-"
+
+## STDOUT:
+(Bool)   false
+(Bool)   true
+
+(Bool)   false
+(Bool)   true
+## END
+
+#### Max code point is disallowed at parse time
+
+pp test_ (/ [\u{10ffff}] /)
+pp test_ (/ [\u{110000}] /)
+
+## STDOUT:
+(Bool)   false
+(Bool)   true
+
+(Bool)   false
+(Bool)   true
+## END
+
+

ysh/regex_translate.py

@@ -19,6 +19,7 @@
 from _devbuild.gen.id_kind_asdl import Id
 from _devbuild.gen.value_asdl import value
 from core.error import e_die, p_die
+from data_lang import j8
 from frontend import lexer
 from mycpp.mylib import log, tagswitch, switch
 from osh import glob_  # for ExtendedRegexEscape
@@ -90,10 +91,14 @@ def _CharCodeToEre(term, parts, special_char_flags):
     if char_int == 0:
         e_die("ERE can't express char code %d" % char_int, term.blame_tok)
 
-    if char_int >= 128 and term.u_braced:  # 128 is 0x80
-        # \u{ff} can't be represented in ERE because we don't know the encoding
-        # \xff can be represented
-        e_die("ERE can't express char code %d" % char_int, term.blame_tok)
+    if term.u_braced:  # \u{3bc} 
+        # max is \u{10ffff} is checked at PARSE time
+        pass
+    else:
+        if char_int >= 128:  # 128 is 0x80
+            # \yff can't be represented in ERE if LC_ALL=utf-8
+            # It should be \u{ff} instead
+            e_die("ERE can't express high bytes %d" % char_int, term.blame_tok)
 
     # note: mycpp doesn't handle
     # special_char_flags[0] |= FLAG_HYPHEN
@@ -108,7 +113,8 @@ def _CharCodeToEre(term, parts, special_char_flags):
     elif char_int == CH_BACKSLASH:
         mask |= FLAG_BACKSLASH
     else:
-        parts.append(chr(char_int))
+        s = j8.Utf8Encode(char_int)
+        parts.append(s)
 
     special_char_flags[0] = mask