From 8740d5d171fbb2fc9a12e46ecda32a0124badb46 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 20:08:32 +0100 Subject: [PATCH 01/11] Add benchmarks for [u8]::make_ascii_uppercase --- src/libcore/benches/ascii_case.rs | 172 ++++++++++++++++++++++++++++++ src/libcore/benches/lib.rs | 1 + 2 files changed, 173 insertions(+) create mode 100644 src/libcore/benches/ascii_case.rs diff --git a/src/libcore/benches/ascii_case.rs b/src/libcore/benches/ascii_case.rs new file mode 100644 index 0000000000000..43242ca3f364e --- /dev/null +++ b/src/libcore/benches/ascii_case.rs @@ -0,0 +1,172 @@ +// Lower-case ASCII 'a' is the first byte that has its highest bit set after wrap-adding 0x1F: +// +// b'a' + 0x1F == 0x80 == 0b1000_0000 +// b'z' + 0x1F == 0x98 == 0b10011000 +// +// Lower-case ASCII 'z' is the last byte that has its highest bit unset after wrap-adding 0x05: +// +// b'a' + 0x05 == 0x66 == 0b0110_0110 +// b'z' + 0x05 == 0x7F == 0b0111_1111 +// +// … except for 0xFB to 0xFF, but those are in the range of bytes that have the highest bit +// unset again after adding 0x1F. +// +// So `(byte + 0x1f) & !(byte + 5)` has its highest bit set +// iff `byte` is a lower-case ASCII letter. +// +// Lower-case ASCII letters all have the 0x20 bit set. +// (Two positions right of 0x80, the highest bit.) +// Unsetting that bit produces the same letter, in upper-case. +// +// Therefore: +fn branchless_to_ascii_upper_case(byte: u8) -> u8 { + byte & + !( + ( + byte.wrapping_add(0x1f) & + !byte.wrapping_add(0x05) & + 0x80 + ) >> 2 + ) +} + + +macro_rules! benches { + ($( fn $name: ident($arg: ident: &mut [u8]) $body: block )+) => { + benches!(mod short SHORT $($name $arg $body)+); + benches!(mod medium MEDIUM $($name $arg $body)+); + benches!(mod long LONG $($name $arg $body)+); + }; + + (mod $mod_name: ident $input: ident $($name: ident $arg: ident $body: block)+) => { + mod $mod_name { + use super::*; + + $( + #[bench] + fn $name(bencher: &mut Bencher) { + bencher.bytes = $input.len() as u64; + bencher.iter(|| { + let mut vec = $input.as_bytes().to_vec(); + { + let $arg = &mut vec[..]; + $body + } + vec + }) + } + )+ + } + } +} + +use test::black_box; +use test::Bencher; + +benches! { + fn bench00_alloc_only(_bytes: &mut [u8]) {} + + fn bench01_black_box_read_each_byte(bytes: &mut [u8]) { + for byte in bytes { + black_box(*byte); + } + } + + fn bench02_lookup(bytes: &mut [u8]) { + for byte in bytes { + *byte = ASCII_UPPERCASE_MAP[*byte as usize] + } + } + + fn bench03_branch_and_subtract(bytes: &mut [u8]) { + for byte in bytes { + *byte = if b'a' <= *byte && *byte <= b'z' { + *byte - b'a' + b'A' + } else { + *byte + } + } + } + + fn bench04_branch_and_mask(bytes: &mut [u8]) { + for byte in bytes { + *byte = if b'a' <= *byte && *byte <= b'z' { + *byte & !0x20 + } else { + *byte + } + } + } + + fn bench05_branchless(bytes: &mut [u8]) { + for byte in bytes { + *byte = branchless_to_ascii_upper_case(*byte) + } + } + + fn bench06_libcore(bytes: &mut [u8]) { + bytes.make_ascii_uppercase() + } +} + +macro_rules! repeat { + ($s: expr) => { concat!($s, $s, $s, $s, $s, $s, $s, $s, $s, $s) } +} + +const SHORT: &'static str = "Alice's"; +const MEDIUM: &'static str = "Alice's Adventures in Wonderland"; +const LONG: &'static str = repeat!(r#" + La Guida di Bragia, a Ballad Opera for the Marionette Theatre (around 1850) + Alice's Adventures in Wonderland (1865) + Phantasmagoria and Other Poems (1869) + Through the Looking-Glass, and What Alice Found There (includes "Jabberwocky" and "The Walrus and the Carpenter") (1871) + The Hunting of the Snark (1876) + Rhyme? And Reason? (1883) – shares some contents with the 1869 collection, including the long poem "Phantasmagoria" + A Tangled Tale (1885) + Sylvie and Bruno (1889) + Sylvie and Bruno Concluded (1893) + Pillow Problems (1893) + What the Tortoise Said to Achilles (1895) + Three Sunsets and Other Poems (1898) + The Manlet (1903)[106] +"#); + +const ASCII_UPPERCASE_MAP: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + b' ', b'!', b'"', b'#', b'$', b'%', b'&', b'\'', + b'(', b')', b'*', b'+', b',', b'-', b'.', b'/', + b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', + b'8', b'9', b':', b';', b'<', b'=', b'>', b'?', + b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G', + b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O', + b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', + b'X', b'Y', b'Z', b'[', b'\\', b']', b'^', b'_', + b'`', + + b'A', b'B', b'C', b'D', b'E', b'F', b'G', + b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O', + b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', + b'X', b'Y', b'Z', + + b'{', b'|', b'}', b'~', 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +]; + diff --git a/src/libcore/benches/lib.rs b/src/libcore/benches/lib.rs index 48572af611a5b..b72d035b0ce94 100644 --- a/src/libcore/benches/lib.rs +++ b/src/libcore/benches/lib.rs @@ -5,6 +5,7 @@ extern crate core; extern crate test; mod any; +mod ascii_case; mod char; mod hash; mod iter; From ce933f77c865a15670855ac5941fe200752b739f Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 20:16:37 +0100 Subject: [PATCH 02/11] Make u8::to_ascii_lowercase and to_ascii_uppercase branchless --- src/libcore/benches/ascii_case.rs | 22 +----- src/libcore/num/mod.rs | 122 ++++++++++-------------------- 2 files changed, 43 insertions(+), 101 deletions(-) diff --git a/src/libcore/benches/ascii_case.rs b/src/libcore/benches/ascii_case.rs index 43242ca3f364e..1c9691d1ee173 100644 --- a/src/libcore/benches/ascii_case.rs +++ b/src/libcore/benches/ascii_case.rs @@ -1,24 +1,4 @@ -// Lower-case ASCII 'a' is the first byte that has its highest bit set after wrap-adding 0x1F: -// -// b'a' + 0x1F == 0x80 == 0b1000_0000 -// b'z' + 0x1F == 0x98 == 0b10011000 -// -// Lower-case ASCII 'z' is the last byte that has its highest bit unset after wrap-adding 0x05: -// -// b'a' + 0x05 == 0x66 == 0b0110_0110 -// b'z' + 0x05 == 0x7F == 0b0111_1111 -// -// … except for 0xFB to 0xFF, but those are in the range of bytes that have the highest bit -// unset again after adding 0x1F. -// -// So `(byte + 0x1f) & !(byte + 5)` has its highest bit set -// iff `byte` is a lower-case ASCII letter. -// -// Lower-case ASCII letters all have the 0x20 bit set. -// (Two positions right of 0x80, the highest bit.) -// Unsetting that bit produces the same letter, in upper-case. -// -// Therefore: +// See comments in `u8::to_ascii_uppercase` in `src/libcore/num/mod.rs`. fn branchless_to_ascii_upper_case(byte: u8) -> u8 { byte & !( diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs index d93cfbc2a28ac..977e21e0dc89a 100644 --- a/src/libcore/num/mod.rs +++ b/src/libcore/num/mod.rs @@ -3794,7 +3794,39 @@ impl u8 { #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> u8 { - ASCII_UPPERCASE_MAP[*self as usize] + // See benchmarks in src/libcore/benches/ascii_case.rs + + // Lower-case ASCII 'a' is the first byte that has its highest bit set + // after wrap-adding 0x1F: + // + // b'a' + 0x1F == 0x80 == 0b1000_0000 + // b'z' + 0x1F == 0x98 == 0b10011000 + // + // Lower-case ASCII 'z' is the last byte that has its highest bit unset + // after wrap-adding 0x05: + // + // b'a' + 0x05 == 0x66 == 0b0110_0110 + // b'z' + 0x05 == 0x7F == 0b0111_1111 + // + // … except for 0xFB to 0xFF, but those are in the range of bytes + // that have the highest bit unset again after adding 0x1F. + // + // So `(byte + 0x1f) & !(byte + 5)` has its highest bit set + // iff `byte` is a lower-case ASCII letter. + // + // Lower-case ASCII letters all have the 0x20 bit set. + // (Two positions right of 0x80, the highest bit.) + // Unsetting that bit produces the same letter, in upper-case. + // + // Therefore: + *self & + !( + ( + self.wrapping_add(0x1f) & + !self.wrapping_add(0x05) & + 0x80 + ) >> 2 + ) } /// Makes a copy of the value in its ASCII lower case equivalent. @@ -3816,7 +3848,15 @@ impl u8 { #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> u8 { - ASCII_LOWERCASE_MAP[*self as usize] + // See comments in to_ascii_uppercase above. + *self | + ( + ( + self.wrapping_add(0x3f) & + !self.wrapping_add(0x25) & + 0x80 + ) >> 2 + ) } /// Checks that two values are an ASCII case-insensitive match. @@ -4940,84 +4980,6 @@ impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0" // Float -> Float impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] } -static ASCII_LOWERCASE_MAP: [u8; 256] = [ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - b' ', b'!', b'"', b'#', b'$', b'%', b'&', b'\'', - b'(', b')', b'*', b'+', b',', b'-', b'.', b'/', - b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', - b'8', b'9', b':', b';', b'<', b'=', b'>', b'?', - b'@', - - b'a', b'b', b'c', b'd', b'e', b'f', b'g', - b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o', - b'p', b'q', b'r', b's', b't', b'u', b'v', b'w', - b'x', b'y', b'z', - - b'[', b'\\', b']', b'^', b'_', - b'`', b'a', b'b', b'c', b'd', b'e', b'f', b'g', - b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o', - b'p', b'q', b'r', b's', b't', b'u', b'v', b'w', - b'x', b'y', b'z', b'{', b'|', b'}', b'~', 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, -]; - -static ASCII_UPPERCASE_MAP: [u8; 256] = [ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - b' ', b'!', b'"', b'#', b'$', b'%', b'&', b'\'', - b'(', b')', b'*', b'+', b',', b'-', b'.', b'/', - b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', - b'8', b'9', b':', b';', b'<', b'=', b'>', b'?', - b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G', - b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O', - b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', - b'X', b'Y', b'Z', b'[', b'\\', b']', b'^', b'_', - b'`', - - b'A', b'B', b'C', b'D', b'E', b'F', b'G', - b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O', - b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W', - b'X', b'Y', b'Z', - - b'{', b'|', b'}', b'~', 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, -]; - enum AsciiCharacterClass { C, // control Cw, // control whitespace From fbe34cc521a5d0f04dd6afadb5b200f0338f41d9 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 20:40:32 +0100 Subject: [PATCH 03/11] =?UTF-8?q?Add=20benchmark=20for=20not-quite-correct?= =?UTF-8?q?=20=E2=80=9Cfake=20SIMD=E2=80=9D=20make=5Fascii=5Fuppercase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libcore/benches/ascii_case.rs | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/libcore/benches/ascii_case.rs b/src/libcore/benches/ascii_case.rs index 1c9691d1ee173..df82a8cb1d537 100644 --- a/src/libcore/benches/ascii_case.rs +++ b/src/libcore/benches/ascii_case.rs @@ -87,6 +87,52 @@ benches! { fn bench06_libcore(bytes: &mut [u8]) { bytes.make_ascii_uppercase() } + + fn bench07_fake_simd_u32(bytes: &mut [u8]) { + let (before, aligned, after) = unsafe { + bytes.align_to_mut::() + }; + for byte in before { + *byte = branchless_to_ascii_upper_case(*byte) + } + for word in aligned { + // FIXME: this is incorrect for some byte values: + // addition within a byte can carry/overflow into the next byte. + // Test case: b"\xFFz " + *word &= !( + ( + word.wrapping_add(0x1f1f1f1f) & + !word.wrapping_add(0x05050505) & + 0x80808080 + ) >> 2 + ) + } + for byte in after { + *byte = branchless_to_ascii_upper_case(*byte) + } + } + + fn bench08_fake_simd_u64(bytes: &mut [u8]) { + let (before, aligned, after) = unsafe { + bytes.align_to_mut::() + }; + for byte in before { + *byte = branchless_to_ascii_upper_case(*byte) + } + for word in aligned { + // FIXME: like above, this is incorrect for some byte values. + *word &= !( + ( + word.wrapping_add(0x1f1f1f1f_1f1f1f1f) & + !word.wrapping_add(0x05050505_05050505) & + 0x80808080_80808080 + ) >> 2 + ) + } + for byte in after { + *byte = branchless_to_ascii_upper_case(*byte) + } + } } macro_rules! repeat { From e3fb6f89fef4fea94deeac572b0373843560c2d6 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 21:28:33 +0100 Subject: [PATCH 04/11] Tidy --- src/libcore/benches/ascii_case.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libcore/benches/ascii_case.rs b/src/libcore/benches/ascii_case.rs index df82a8cb1d537..dcc87cc45b1e3 100644 --- a/src/libcore/benches/ascii_case.rs +++ b/src/libcore/benches/ascii_case.rs @@ -145,9 +145,11 @@ const LONG: &'static str = repeat!(r#" La Guida di Bragia, a Ballad Opera for the Marionette Theatre (around 1850) Alice's Adventures in Wonderland (1865) Phantasmagoria and Other Poems (1869) - Through the Looking-Glass, and What Alice Found There (includes "Jabberwocky" and "The Walrus and the Carpenter") (1871) + Through the Looking-Glass, and What Alice Found There + (includes "Jabberwocky" and "The Walrus and the Carpenter") (1871) The Hunting of the Snark (1876) - Rhyme? And Reason? (1883) – shares some contents with the 1869 collection, including the long poem "Phantasmagoria" + Rhyme? And Reason? (1883) – shares some contents with the 1869 collection, + including the long poem "Phantasmagoria" A Tangled Tale (1885) Sylvie and Bruno (1889) Sylvie and Bruno Concluded (1893) From 525a043c6bc8a220b7dbdde3eb10bb79f8e99506 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 23:32:36 +0100 Subject: [PATCH 05/11] Rename src/libcore/benches/ascii_case.rs to ascii.rs --- src/libcore/benches/{ascii_case.rs => ascii.rs} | 6 ++++++ src/libcore/benches/lib.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) rename src/libcore/benches/{ascii_case.rs => ascii.rs} (97%) diff --git a/src/libcore/benches/ascii_case.rs b/src/libcore/benches/ascii.rs similarity index 97% rename from src/libcore/benches/ascii_case.rs rename to src/libcore/benches/ascii.rs index dcc87cc45b1e3..2714a8a5c278e 100644 --- a/src/libcore/benches/ascii_case.rs +++ b/src/libcore/benches/ascii.rs @@ -84,6 +84,12 @@ benches! { } } + fn bench05_multiply_by_bool(bytes: &mut [u8]) { + for byte in bytes { + *byte &= !(0x20 * (b'a' <= *byte && *byte <= b'z') as u8) + } + } + fn bench06_libcore(bytes: &mut [u8]) { bytes.make_ascii_uppercase() } diff --git a/src/libcore/benches/lib.rs b/src/libcore/benches/lib.rs index b72d035b0ce94..707cdd5f450ea 100644 --- a/src/libcore/benches/lib.rs +++ b/src/libcore/benches/lib.rs @@ -5,7 +5,7 @@ extern crate core; extern crate test; mod any; -mod ascii_case; +mod ascii; mod char; mod hash; mod iter; From 6d3840b23a185d057cc9792a2516e6a71887f277 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 23:56:50 +0100 Subject: [PATCH 06/11] Add benchmarks for `u8::is_ascii*` --- src/libcore/benches/ascii.rs | 49 +++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/libcore/benches/ascii.rs b/src/libcore/benches/ascii.rs index 2714a8a5c278e..e20b1b9bf3136 100644 --- a/src/libcore/benches/ascii.rs +++ b/src/libcore/benches/ascii.rs @@ -12,7 +12,14 @@ fn branchless_to_ascii_upper_case(byte: u8) -> u8 { macro_rules! benches { - ($( fn $name: ident($arg: ident: &mut [u8]) $body: block )+) => { + ($( fn $name: ident($arg: ident: &mut [u8]) $body: block )+ @iter $( $is_: ident, )+) => { + benches! {@ + $( fn $name($arg: &mut [u8]) $body )+ + $( fn $is_(bytes: &mut [u8]) { bytes.iter().all(u8::$is_) } )+ + } + }; + + (@$( fn $name: ident($arg: ident: &mut [u8]) $body: block )+) => { benches!(mod short SHORT $($name $arg $body)+); benches!(mod medium MEDIUM $($name $arg $body)+); benches!(mod long LONG $($name $arg $body)+); @@ -30,7 +37,7 @@ macro_rules! benches { let mut vec = $input.as_bytes().to_vec(); { let $arg = &mut vec[..]; - $body + black_box($body); } vec }) @@ -44,21 +51,21 @@ use test::black_box; use test::Bencher; benches! { - fn bench00_alloc_only(_bytes: &mut [u8]) {} + fn case00_alloc_only(_bytes: &mut [u8]) {} - fn bench01_black_box_read_each_byte(bytes: &mut [u8]) { + fn case01_black_box_read_each_byte(bytes: &mut [u8]) { for byte in bytes { black_box(*byte); } } - fn bench02_lookup(bytes: &mut [u8]) { + fn case02_lookup(bytes: &mut [u8]) { for byte in bytes { *byte = ASCII_UPPERCASE_MAP[*byte as usize] } } - fn bench03_branch_and_subtract(bytes: &mut [u8]) { + fn case03_branch_and_subtract(bytes: &mut [u8]) { for byte in bytes { *byte = if b'a' <= *byte && *byte <= b'z' { *byte - b'a' + b'A' @@ -68,7 +75,7 @@ benches! { } } - fn bench04_branch_and_mask(bytes: &mut [u8]) { + fn case04_branch_and_mask(bytes: &mut [u8]) { for byte in bytes { *byte = if b'a' <= *byte && *byte <= b'z' { *byte & !0x20 @@ -78,23 +85,17 @@ benches! { } } - fn bench05_branchless(bytes: &mut [u8]) { + fn case05_branchless(bytes: &mut [u8]) { for byte in bytes { *byte = branchless_to_ascii_upper_case(*byte) } } - fn bench05_multiply_by_bool(bytes: &mut [u8]) { - for byte in bytes { - *byte &= !(0x20 * (b'a' <= *byte && *byte <= b'z') as u8) - } - } - - fn bench06_libcore(bytes: &mut [u8]) { + fn case06_libcore(bytes: &mut [u8]) { bytes.make_ascii_uppercase() } - fn bench07_fake_simd_u32(bytes: &mut [u8]) { + fn case07_fake_simd_u32(bytes: &mut [u8]) { let (before, aligned, after) = unsafe { bytes.align_to_mut::() }; @@ -118,7 +119,7 @@ benches! { } } - fn bench08_fake_simd_u64(bytes: &mut [u8]) { + fn case08_fake_simd_u64(bytes: &mut [u8]) { let (before, aligned, after) = unsafe { bytes.align_to_mut::() }; @@ -139,6 +140,20 @@ benches! { *byte = branchless_to_ascii_upper_case(*byte) } } + + @iter + + is_ascii, + is_ascii_alphabetic, + is_ascii_uppercase, + is_ascii_lowercase, + is_ascii_alphanumeric, + is_ascii_digit, + is_ascii_hexdigit, + is_ascii_punctuation, + is_ascii_graphic, + is_ascii_whitespace, + is_ascii_control, } macro_rules! repeat { From b4faa9b45635ae5185d562701d7429e5fbb08c8b Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Mon, 18 Mar 2019 23:57:09 +0100 Subject: [PATCH 07/11] Remove ASCII_CHARACTER_CLASS table, use `match` with range patterns instead. --- src/libcore/num/mod.rs | 75 +++++++++++------------------------------- 1 file changed, 20 insertions(+), 55 deletions(-) diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs index 977e21e0dc89a..64469a4b7e43c 100644 --- a/src/libcore/num/mod.rs +++ b/src/libcore/num/mod.rs @@ -3958,9 +3958,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_alphabetic(&self) -> bool { - if *self >= 0x80 { return false; } - match ASCII_CHARACTER_CLASS[*self as usize] { - L | Lx | U | Ux => true, + match *self { + b'A'...b'Z' | b'a'...b'z' => true, _ => false } } @@ -3994,9 +3993,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_uppercase(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - U | Ux => true, + match *self { + b'A'...b'Z' => true, _ => false } } @@ -4030,9 +4028,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_lowercase(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - L | Lx => true, + match *self { + b'a'...b'z' => true, _ => false } } @@ -4069,9 +4066,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_alphanumeric(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - D | L | Lx | U | Ux => true, + match *self { + b'0'...b'9' | b'A'...b'Z' | b'a'...b'z' => true, _ => false } } @@ -4105,9 +4101,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_digit(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - D => true, + match *self { + b'0'...b'9' => true, _ => false } } @@ -4144,9 +4139,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_hexdigit(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - D | Lx | Ux => true, + match *self { + b'0'...b'9' | b'A'...b'F' | b'a'...b'f' => true, _ => false } } @@ -4184,9 +4178,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_punctuation(&self) -> bool { - if *self >= 0x80 { return false } - match ASCII_CHARACTER_CLASS[*self as usize] { - P => true, + match *self { + b'!'...b'/' | b':'...b'@' | b'['...b'`' | b'{'...b'~' => true, _ => false } } @@ -4220,9 +4213,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_graphic(&self) -> bool { - if *self >= 0x80 { return false; } - match ASCII_CHARACTER_CLASS[*self as usize] { - Ux | U | Lx | L | D | P => true, + match *self { + b'!'...b'~' => true, _ => false } } @@ -4273,9 +4265,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_whitespace(&self) -> bool { - if *self >= 0x80 { return false; } - match ASCII_CHARACTER_CLASS[*self as usize] { - Cw | W => true, + match *self { + b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' => true, _ => false } } @@ -4311,9 +4302,8 @@ impl u8 { #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] #[inline] pub fn is_ascii_control(&self) -> bool { - if *self >= 0x80 { return false; } - match ASCII_CHARACTER_CLASS[*self as usize] { - C | Cw => true, + match *self { + b'\0'...b'\x1F' | b'\x7F' => true, _ => false } } @@ -4979,28 +4969,3 @@ impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0" // Float -> Float impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] } - -enum AsciiCharacterClass { - C, // control - Cw, // control whitespace - W, // whitespace - D, // digit - L, // lowercase - Lx, // lowercase hex digit - U, // uppercase - Ux, // uppercase hex digit - P, // punctuation -} -use self::AsciiCharacterClass::*; - -static ASCII_CHARACTER_CLASS: [AsciiCharacterClass; 128] = [ -// _0 _1 _2 _3 _4 _5 _6 _7 _8 _9 _a _b _c _d _e _f - C, C, C, C, C, C, C, C, C, Cw,Cw,C, Cw,Cw,C, C, // 0_ - C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, // 1_ - W, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, // 2_ - D, D, D, D, D, D, D, D, D, D, P, P, P, P, P, P, // 3_ - P, Ux,Ux,Ux,Ux,Ux,Ux,U, U, U, U, U, U, U, U, U, // 4_ - U, U, U, U, U, U, U, U, U, U, U, P, P, P, P, P, // 5_ - P, Lx,Lx,Lx,Lx,Lx,Lx,L, L, L, L, L, L, L, L, L, // 6_ - L, L, L, L, L, L, L, L, L, L, L, P, P, P, P, C, // 7_ -]; From 4a3241a815e951dfa87e9be7133e8265bc945e57 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Tue, 19 Mar 2019 00:34:27 +0100 Subject: [PATCH 08/11] Benchmark more possibles impls of [u8]::make_ascii_uppercase --- src/libcore/benches/ascii.rs | 84 +++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/src/libcore/benches/ascii.rs b/src/libcore/benches/ascii.rs index e20b1b9bf3136..ce36027394a48 100644 --- a/src/libcore/benches/ascii.rs +++ b/src/libcore/benches/ascii.rs @@ -59,7 +59,7 @@ benches! { } } - fn case02_lookup(bytes: &mut [u8]) { + fn case02_lookup_table(bytes: &mut [u8]) { for byte in bytes { *byte = ASCII_UPPERCASE_MAP[*byte as usize] } @@ -141,6 +141,55 @@ benches! { } } + fn case09_mask_mult_bool_branchy_lookup_table(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + if b >= 0x80 { return false } + match ASCII_CHARACTER_CLASS[b as usize] { + L | Lx => true, + _ => false, + } + } + for byte in bytes { + *byte &= !(0x20 * (is_ascii_lowercase(*byte) as u8)) + } + } + + fn case10_mask_mult_bool_lookup_table(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + match ASCII_CHARACTER_CLASS[b as usize] { + L | Lx => true, + _ => false + } + } + for byte in bytes { + *byte &= !(0x20 * (is_ascii_lowercase(*byte) as u8)) + } + } + + fn case11_mask_mult_bool_match_range(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + match b { + b'a'...b'z' => true, + _ => false + } + } + for byte in bytes { + *byte &= !(0x20 * (is_ascii_lowercase(*byte) as u8)) + } + } + + fn case12_mask_shifted_bool_match_range(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + match b { + b'a'...b'z' => true, + _ => false + } + } + for byte in bytes { + *byte &= !((is_ascii_lowercase(*byte) as u8) << 5) + } + } + @iter is_ascii, @@ -219,3 +268,36 @@ const ASCII_UPPERCASE_MAP: [u8; 256] = [ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, ]; +enum AsciiCharacterClass { + C, // control + Cw, // control whitespace + W, // whitespace + D, // digit + L, // lowercase + Lx, // lowercase hex digit + U, // uppercase + Ux, // uppercase hex digit + P, // punctuation + N, // Non-ASCII +} +use self::AsciiCharacterClass::*; + +static ASCII_CHARACTER_CLASS: [AsciiCharacterClass; 256] = [ +// _0 _1 _2 _3 _4 _5 _6 _7 _8 _9 _a _b _c _d _e _f + C, C, C, C, C, C, C, C, C, Cw,Cw,C, Cw,Cw,C, C, // 0_ + C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, // 1_ + W, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, // 2_ + D, D, D, D, D, D, D, D, D, D, P, P, P, P, P, P, // 3_ + P, Ux,Ux,Ux,Ux,Ux,Ux,U, U, U, U, U, U, U, U, U, // 4_ + U, U, U, U, U, U, U, U, U, U, U, P, P, P, P, P, // 5_ + P, Lx,Lx,Lx,Lx,Lx,Lx,L, L, L, L, L, L, L, L, L, // 6_ + L, L, L, L, L, L, L, L, L, L, L, P, P, P, P, C, // 7_ + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, + N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, +]; From 0ad91f73d92c3b8d3978f8f54c04b8efe3d2e673 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Tue, 19 Mar 2019 00:50:26 +0100 Subject: [PATCH 09/11] Simplify u8::to_ascii_{upp,low}ercase while keeping it fast --- src/libcore/benches/ascii.rs | 24 ++++++++++++++++++- src/libcore/num/mod.rs | 46 ++++-------------------------------- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/src/libcore/benches/ascii.rs b/src/libcore/benches/ascii.rs index ce36027394a48..89e67cca4b754 100644 --- a/src/libcore/benches/ascii.rs +++ b/src/libcore/benches/ascii.rs @@ -1,4 +1,26 @@ -// See comments in `u8::to_ascii_uppercase` in `src/libcore/num/mod.rs`. +// Lower-case ASCII 'a' is the first byte that has its highest bit set +// after wrap-adding 0x1F: +// +// b'a' + 0x1F == 0x80 == 0b1000_0000 +// b'z' + 0x1F == 0x98 == 0b10011000 +// +// Lower-case ASCII 'z' is the last byte that has its highest bit unset +// after wrap-adding 0x05: +// +// b'a' + 0x05 == 0x66 == 0b0110_0110 +// b'z' + 0x05 == 0x7F == 0b0111_1111 +// +// … except for 0xFB to 0xFF, but those are in the range of bytes +// that have the highest bit unset again after adding 0x1F. +// +// So `(byte + 0x1f) & !(byte + 5)` has its highest bit set +// iff `byte` is a lower-case ASCII letter. +// +// Lower-case ASCII letters all have the 0x20 bit set. +// (Two positions right of 0x80, the highest bit.) +// Unsetting that bit produces the same letter, in upper-case. +// +// Therefore: fn branchless_to_ascii_upper_case(byte: u8) -> u8 { byte & !( diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs index 64469a4b7e43c..3fcae6b94b06d 100644 --- a/src/libcore/num/mod.rs +++ b/src/libcore/num/mod.rs @@ -3794,39 +3794,8 @@ impl u8 { #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> u8 { - // See benchmarks in src/libcore/benches/ascii_case.rs - - // Lower-case ASCII 'a' is the first byte that has its highest bit set - // after wrap-adding 0x1F: - // - // b'a' + 0x1F == 0x80 == 0b1000_0000 - // b'z' + 0x1F == 0x98 == 0b10011000 - // - // Lower-case ASCII 'z' is the last byte that has its highest bit unset - // after wrap-adding 0x05: - // - // b'a' + 0x05 == 0x66 == 0b0110_0110 - // b'z' + 0x05 == 0x7F == 0b0111_1111 - // - // … except for 0xFB to 0xFF, but those are in the range of bytes - // that have the highest bit unset again after adding 0x1F. - // - // So `(byte + 0x1f) & !(byte + 5)` has its highest bit set - // iff `byte` is a lower-case ASCII letter. - // - // Lower-case ASCII letters all have the 0x20 bit set. - // (Two positions right of 0x80, the highest bit.) - // Unsetting that bit produces the same letter, in upper-case. - // - // Therefore: - *self & - !( - ( - self.wrapping_add(0x1f) & - !self.wrapping_add(0x05) & - 0x80 - ) >> 2 - ) + // Unset the fith bit if this is a lowercase letter + *self & !((self.is_ascii_lowercase() as u8) << 5) } /// Makes a copy of the value in its ASCII lower case equivalent. @@ -3848,15 +3817,8 @@ impl u8 { #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> u8 { - // See comments in to_ascii_uppercase above. - *self | - ( - ( - self.wrapping_add(0x3f) & - !self.wrapping_add(0x25) & - 0x80 - ) >> 2 - ) + // Set the fith bit if this is an uppercase letter + *self | ((self.is_ascii_uppercase() as u8) << 5) } /// Checks that two values are an ASCII case-insensitive match. From c1ec29ace000ce4c733cde6948c87f9bc2370691 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Tue, 19 Mar 2019 08:32:15 +0100 Subject: [PATCH 10/11] ASCII uppercase: add "subtract shifted bool" benchmark --- src/libcore/benches/ascii.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libcore/benches/ascii.rs b/src/libcore/benches/ascii.rs index 89e67cca4b754..968d28a995897 100644 --- a/src/libcore/benches/ascii.rs +++ b/src/libcore/benches/ascii.rs @@ -212,6 +212,18 @@ benches! { } } + fn case13_subtract_shifted_bool_match_range(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + match b { + b'a'...b'z' => true, + _ => false + } + } + for byte in bytes { + *byte -= (is_ascii_lowercase(*byte) as u8) << 5 + } + } + @iter is_ascii, From 7fad370fe9dc000f6f6f497a1939de6196e2c4fc Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Tue, 19 Mar 2019 13:41:59 +0100 Subject: [PATCH 11/11] ASCII uppercase: add "subtract multiplied bool" benchmark --- src/libcore/benches/ascii.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libcore/benches/ascii.rs b/src/libcore/benches/ascii.rs index 968d28a995897..635537e312158 100644 --- a/src/libcore/benches/ascii.rs +++ b/src/libcore/benches/ascii.rs @@ -224,6 +224,18 @@ benches! { } } + fn case14_subtract_multiplied_bool_match_range(bytes: &mut [u8]) { + fn is_ascii_lowercase(b: u8) -> bool { + match b { + b'a'...b'z' => true, + _ => false + } + } + for byte in bytes { + *byte -= (b'a' - b'A') * is_ascii_lowercase(*byte) as u8 + } + } + @iter is_ascii,