Rollup merge of rust-lang#68232 - Mark-Simulacrum:unicode-tables, r=joshtriplett

Optimize size/speed of Unicode datasets

The overall implementation has the same general idea as the prior approach,
which was based on a compressed trie structure, but modified to use less space
(and, coincidentally, be an overall performance improvement).

Sizes | Old | New | New/current
-- | -- | -- | --
Alphabetic | 4616 | 2982 | 64.60%
Case_Ignorable | 3144 | 2112 | 67.18%
Cased | 2376 | 934 | 39.31%
Cc | 19 | 43 | 226.32%
Grapheme_Extend | 3072 | 1734 | 56.45%
Lowercase | 2328 | 985 | 42.31%
N | 2648 | 1239 | 46.79%
Uppercase | 1978 | 934 | 47.22%
White_Space | 241 | 140 | 58.09%
  |   |   |
Total | 20422 | 11103 | 54.37%

This table shows the size of the old and new tables in bytes. The most important
of these tables is "Grapheme_Extend", as it is present in essentially all Rust
programs due to being called from `str`'s Debug impl (`char::escape_debug`). In
a representative case given by this [blog post] for the embedded world, the
shrinking in this PR shrinks the final binary by 1,604 bytes, from 14,440 to
12,836.

The performance of these new tables, based on the (rough) benchmark of linearly
scanning the entire valid set of chars, querying for each `is_*`, is roughly
~50% better, though in some cases is either on par or slightly (3-5%) worse. In
practice, I believe the size benefits of this PR are the main concern. The new
implementation has been tested to be equivalent to the current nightly in terms
of returned values on the set of valid chars.

A (relatively) high-level explanation of the specific compression scheme used
can be found [in the generator].

This is split into three commits -- the first adds the generator which produces
the Rust code for the tables, the second adds support code for the lookup, and
the third actually swaps the current implementation out for the new one.

[blog post]: https://jamesmunns.com/blog/fmt-unreasonably-expensive/
[in the generator]: https://github.com/Mark-Simulacrum/rust/blob/unicode-tables/src/tools/unicode-table-generator/src/raw_emitter.rs

Author: Dylan-DPC

.gitignore

@@ -34,14 +34,7 @@ __pycache__/
 # Created by default with `src/ci/docker/run.sh`:
 /obj/
 /rustllvm/
-/src/libcore/unicode/DerivedCoreProperties.txt
-/src/libcore/unicode/DerivedNormalizationProps.txt
-/src/libcore/unicode/PropList.txt
-/src/libcore/unicode/ReadMe.txt
-/src/libcore/unicode/Scripts.txt
-/src/libcore/unicode/SpecialCasing.txt
-/src/libcore/unicode/UnicodeData.txt
-/src/libcore/unicode/downloaded
+/unicode-downloads
 /target/
 # Generated by compiletest for incremental:
 /tmp/

Cargo.lock

@@ -4953,6 +4953,16 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
 
+[[package]]
+name = "ucd-parse"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca6b52bf4da6512f0f07785a04769222e50d29639e7ecd016b7806fd2de306b4"
+dependencies = [
+ "lazy_static 1.3.0",
+ "regex",
+]
+
 [[package]]
 name = "ucd-trie"
 version = "0.1.1"
@@ -4974,6 +4984,13 @@ dependencies = [
  "version_check 0.1.5",
 ]
 
+[[package]]
+name = "unicode-bdd"
+version = "0.1.0"
+dependencies = [
+ "ucd-parse",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.4"

Cargo.toml

@@ -23,6 +23,7 @@ members = [
   "src/tools/rustfmt",
   "src/tools/miri",
   "src/tools/rustdoc-themes",
+  "src/tools/unicode-table-generator",
 ]
 exclude = [
   "build",

src/libcore/char/methods.rs

@@ -3,7 +3,7 @@
 use crate::slice;
 use crate::str::from_utf8_unchecked_mut;
 use crate::unicode::printable::is_printable;
-use crate::unicode::tables::{conversions, derived_property, general_category, property};
+use crate::unicode::{self, conversions};
 
 use super::*;
 
@@ -552,7 +552,7 @@ impl char {
     pub fn is_alphabetic(self) -> bool {
         match self {
             'a'..='z' | 'A'..='Z' => true,
-            c => c > '\x7f' && derived_property::Alphabetic(c),
+            c => c > '\x7f' && unicode::Alphabetic(c),
         }
     }
 
@@ -583,7 +583,7 @@ impl char {
     pub fn is_lowercase(self) -> bool {
         match self {
             'a'..='z' => true,
-            c => c > '\x7f' && derived_property::Lowercase(c),
+            c => c > '\x7f' && unicode::Lowercase(c),
         }
     }
 
@@ -614,7 +614,7 @@ impl char {
     pub fn is_uppercase(self) -> bool {
         match self {
             'A'..='Z' => true,
-            c => c > '\x7f' && derived_property::Uppercase(c),
+            c => c > '\x7f' && unicode::Uppercase(c),
         }
     }
 
@@ -642,7 +642,7 @@ impl char {
     pub fn is_whitespace(self) -> bool {
         match self {
             ' ' | '\x09'..='\x0d' => true,
-            c => c > '\x7f' && property::White_Space(c),
+            c => c > '\x7f' && unicode::White_Space(c),
         }
     }
 
@@ -693,7 +693,7 @@ impl char {
     #[stable(feature = "rust1", since = "1.0.0")]
     #[inline]
     pub fn is_control(self) -> bool {
-        general_category::Cc(self)
+        unicode::Cc(self)
     }
 
     /// Returns `true` if this `char` has the `Grapheme_Extend` property.
@@ -707,7 +707,7 @@ impl char {
     /// [`DerivedCoreProperties.txt`]: https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
     #[inline]
     pub(crate) fn is_grapheme_extended(self) -> bool {
-        derived_property::Grapheme_Extend(self)
+        unicode::Grapheme_Extend(self)
     }
 
     /// Returns `true` if this `char` has one of the general categories for numbers.
@@ -739,7 +739,7 @@ impl char {
     pub fn is_numeric(self) -> bool {
         match self {
             '0'..='9' => true,
-            c => c > '\x7f' && general_category::N(c),
+            c => c > '\x7f' && unicode::N(c),
         }
     }
 

src/libcore/char/mod.rs

@@ -37,9 +37,9 @@ pub use self::decode::{decode_utf16, DecodeUtf16, DecodeUtf16Error};
 
 // unstable re-exports
 #[unstable(feature = "unicode_version", issue = "49726")]
-pub use crate::unicode::tables::UNICODE_VERSION;
-#[unstable(feature = "unicode_version", issue = "49726")]
 pub use crate::unicode::version::UnicodeVersion;
+#[unstable(feature = "unicode_version", issue = "49726")]
+pub use crate::unicode::UNICODE_VERSION;
 
 use crate::fmt::{self, Write};
 use crate::iter::FusedIterator;

src/libcore/unicode/bool_trie.rs

@@ -1,15 +1,59 @@
 #![unstable(feature = "unicode_internals", issue = "none")]
 #![allow(missing_docs)]
 
-mod bool_trie;
 pub(crate) mod printable;
-pub(crate) mod tables;
+mod unicode_data;
 pub(crate) mod version;
 
+use version::UnicodeVersion;
+
+/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
+/// `char` and `str` methods are based on.
+#[unstable(feature = "unicode_version", issue = "49726")]
+pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {
+    major: unicode_data::UNICODE_VERSION.0,
+    minor: unicode_data::UNICODE_VERSION.1,
+    micro: unicode_data::UNICODE_VERSION.2,
+    _priv: (),
+};
+
 // For use in liballoc, not re-exported in libstd.
 pub mod derived_property {
-    pub use crate::unicode::tables::derived_property::{Case_Ignorable, Cased};
+    pub use super::{Case_Ignorable, Cased};
 }
-pub mod conversions {
-    pub use crate::unicode::tables::conversions::{to_lower, to_upper};
+
+pub use unicode_data::alphabetic::lookup as Alphabetic;
+pub use unicode_data::case_ignorable::lookup as Case_Ignorable;
+pub use unicode_data::cased::lookup as Cased;
+pub use unicode_data::cc::lookup as Cc;
+pub use unicode_data::conversions;
+pub use unicode_data::grapheme_extend::lookup as Grapheme_Extend;
+pub use unicode_data::lowercase::lookup as Lowercase;
+pub use unicode_data::n::lookup as N;
+pub use unicode_data::uppercase::lookup as Uppercase;
+pub use unicode_data::white_space::lookup as White_Space;
+
+#[inline(always)]
+fn range_search<const N: usize, const N1: usize, const N2: usize>(
+    needle: u32,
+    chunk_idx_map: &[u8; N],
+    (last_chunk_idx, last_chunk_mapping): (u16, u8),
+    bitset_chunk_idx: &[[u8; 16]; N1],
+    bitset: &[u64; N2],
+) -> bool {
+    let bucket_idx = (needle / 64) as usize;
+    let chunk_map_idx = bucket_idx / 16;
+    let chunk_piece = bucket_idx % 16;
+    let chunk_idx = if chunk_map_idx >= N {
+        if chunk_map_idx == last_chunk_idx as usize {
+            last_chunk_mapping
+        } else {
+            return false;
+        }
+    } else {
+        chunk_idx_map[chunk_map_idx]
+    };
+    let idx = bitset_chunk_idx[(chunk_idx as usize)][chunk_piece];
+    let word = bitset[(idx as usize)];
+    (word & (1 << (needle % 64) as u64)) != 0
 }