[libc unicode] Print warnings if the locale is not UTF-8

spec/sh-usage: Test that env vars like LC_ALL LANG are respected

Update the docs.

Author: Andy C

bin/oils_for_unix.py

@@ -95,19 +95,22 @@ def InitLocale(environ):
         # passing None queries it
         #lo = locale.setlocale(locale.LC_CTYPE, None)
     except pylocale.Error:
-        #print('INVALID')
+        # we could make this nicer
+        print_stderr('oils: setlocale() failed')
         locale_name = ''  # unknown value
     #log('LOC %s', locale_name)
 
     if locale_name not in ('', 'C'):
-        # Check that it's utf-8
+        # Check if the codeset is UTF-8, unless OILS_LOCALE_OK=1 is set
+        if environ.get('OILS_LOCALE_OK') == '1':
+            return
+
         codeset = pylocale.nl_langinfo(pylocale.CODESET)
         #log('codeset %s', codeset)
 
         if not match.IsUtf8Codeset(codeset):
-            # TODO: enable this if not OILS_LOCALE_OK=1
-            #print_stderr('Warning: not UTF-8')
-            pass
+            print_stderr("oils warning: codeset %r doesn't look like UTF-8" % codeset)
+            print_stderr('              Set OILS_LOCALE_OK=1 to remove this message')
 
 
 # TODO: Hook up valid applets (including these) to completion

doc/ref/chap-special-var.md

@@ -239,6 +239,10 @@ When the shell process exists, print GC stats to stderr.
 
 When the shell process exists, print GC stats to this file descriptor.
 
+### `OILS_LOCALE_OK`
+
+Suppress the warning about `libc` locales that are not UTF-8.
+
 ## Float
 
 ### NAN

doc/ref/toc-ysh.md

@@ -380,10 +380,10 @@ X [External Lang] BEGIN   END   when (awk)
   [YSH read]      _reply
   [History]       YSH_HISTFILE
   [Interactive]   OILS_COMP_UI
-  [Oils VM]       OILS_VERSION
+  [Oils VM]       OILS_VERSION        LIB_YSH
                   OILS_GC_THRESHOLD   OILS_GC_ON_EXIT
                   OILS_GC_STATS       OILS_GC_STATS_FD
-                  LIB_YSH
+                  OILS_LOCALE_OK
   [Float]         NAN                 INFINITY
   [Module]        __provide__ 
   [Other Env]     HOME                PATH

doc/unicode.md

@@ -133,37 +133,31 @@ Not unicode aware:
 
 - `strcmp()` does byte-wise and UTF-8 wise comparisons?
 
-### libc functions used
-
-TODO
-
-- Note: GNU readline calls `setlocale()`, which means that the `oils-for-unix`
-  process is affected by the environment
-
 ### Data Languages
 
 - Decoding JSON/J8 validates UTF-8
 - Encoding JSON/J8 decodes and validates UTF-8
   - So we can distinguish valid UTF-8 and invalid bytes like `\yff`
 
-## Implementation Notes
+## libc locale
 
-Unlike bash and CPython, Oils doesn't call `setlocale()`.  (Although GNU
-readline may call it.)
+At startup, Oils calls `setlocale()`, which initializes the global libc locale
+from the environment.  (GNU readline also calls `setlocale()`, but Oils may or
+may not link against GNU readline.)
 
-It's expected that your locale will respect UTF-8.  This is true on most
-distros.  If not, then some string operations will support UTF-8 and some
-won't.
+The locale affects the behavior of say `?` in globs, and `.` in libc regexes.
+
+Oils only supports UTF-8.  If the locale is not UTF-8, Oils prints a warning to
+stderr.  You can silence it with `OILS_LOCALE_OK=1`.
+
+### Some string operations use libc, and some don'
 
 For example:
 
 - String length like `${#s}` is implemented in Oils code, not libc, so it will
   always respect UTF-8.
 - `[[ s =~ $pat ]]` is implemented with libc, so it is affected by the locale
-  settings.  Same with Oils `(x ~ pat)`.
-
-TODO: Oils should support `LANG=C` for some operations, but not `LANG=X` for
-other `X`.
+  settings.  This is also true of YSH `(x ~ pat)`.
 
 ### List of Low-Level UTF-8 Operations
 

spec/sh-usage.test.sh

@@ -122,3 +122,76 @@ status=0
 
 ## N-I bash/dash/mksh/zsh STDOUT:
 ## END
+
+#### Set LC_ALL LC_CTYPE LC_COLLATE LANG - affects glob ?
+
+# note: test/spec-common.sh sets LC_ALL
+unset LC_ALL
+
+touch _x_ _μ_
+
+LC_ALL=C       $SH -c 'echo LC_ALL _?_'
+LC_ALL=C.UTF-8 $SH -c 'echo LC_ALL _?_'
+echo
+
+LC_CTYPE=C       $SH -c 'echo LC_CTYPE _?_'
+LC_CTYPE=C.UTF-8 $SH -c 'echo LC_CTYPE _?_'
+echo
+
+LC_COLLATE=C       $SH -c 'echo LC_COLLATE _?_'
+LC_COLLATE=C.UTF-8 $SH -c 'echo LC_COLLATE _?_'
+echo
+
+LANG=C       $SH -c 'echo LANG _?_'
+LANG=C.UTF-8 $SH -c 'echo LANG _?_'
+
+## STDOUT:
+LC_ALL _x_
+LC_ALL _x_ _μ_
+
+LC_CTYPE _x_
+LC_CTYPE _x_ _μ_
+
+LC_COLLATE _x_
+LC_COLLATE _x_
+
+LANG _x_
+LANG _x_ _μ_
+## END
+
+## N-I dash/mksh STDOUT:
+LC_ALL _x_
+LC_ALL _x_
+
+LC_CTYPE _x_
+LC_CTYPE _x_
+
+LC_COLLATE _x_
+LC_COLLATE _x_
+
+LANG _x_
+LANG _x_
+## END
+
+
+#### LC_ALL=invalid
+
+# note: test/spec-common.sh sets LC_ALL
+unset LC_ALL
+
+touch _x_ _μ_
+
+LC_ALL=invalid $SH -c 'echo LC_ALL _?_' 2> err.txt
+
+#cat err.txt
+wc -l err.txt
+
+## STDOUT:
+LC_ALL _x_
+1 err.txt
+## END
+
+## N-I dash/mksh/zsh STDOUT:
+LC_ALL _x_
+0 err.txt
+## END