diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index e63fb22829a3f..924bb803b368f 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -19,7 +19,7 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; -use rustc_target::abi::{self, HasDataLayout, Primitive}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; @@ -857,28 +857,39 @@ fn generic_simd_intrinsic( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - let in_ty = arg_tys[0]; - let m_len = match in_ty.kind() { - // Note that this `.unwrap()` crashes for isize/usize, that's sort - // of intentional as there's not currently a use case for that. - ty::Int(i) => i.bit_width().unwrap(), - ty::Uint(i) => i.bit_width().unwrap(), - _ => return_error!("`{}` is not an integral type", in_ty), - }; require_simd!(arg_tys[1], "argument"); - let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!( - // Allow masks for vectors with fewer than 8 elements to be - // represented with a u8 or i8. - m_len == v_len || (m_len == 8 && v_len < 8), - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len - ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + + let expected_int_bits = (len.max(8) - 1).next_power_of_two(); + let expected_bytes = len / 8 + ((len % 8 > 0) as u64); + + let mask_ty = arg_tys[0]; + let mask = match mask_ty.kind() { + ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + let place = PlaceRef::alloca(bx, args[0].layout); + args[0].val.store(bx, place); + let int_ty = bx.type_ix(expected_bytes * 8); + let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); + bx.load(int_ty, ptr, Align::ONE) + } + _ => return_error!( + "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + mask_ty, + expected_int_bits, + expected_bytes + ), + }; + let i1 = bx.type_i1(); - let im = bx.type_ix(v_len); - let i1xn = bx.type_vector(i1, v_len); - let m_im = bx.trunc(args[0].immediate(), im); + let im = bx.type_ix(len); + let i1xn = bx.type_vector(i1, len); + let m_im = bx.trunc(mask, im); let m_i1s = bx.bitcast(m_im, i1xn); return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } @@ -1056,16 +1067,16 @@ fn generic_simd_intrinsic( if name == sym::simd_bitmask { // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a - // vector mask and returns an unsigned integer containing the most - // significant bit (MSB) of each lane. - - // If the vector has less than 8 lanes, a u8 is returned with zeroed - // trailing bits. + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. let expected_int_bits = in_len.max(8); - match ret_ty.kind() { - ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (), - _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits), - } + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); // Integer vector : let (i_xn, in_elem_bitwidth) = match in_elem.kind() { @@ -1095,8 +1106,34 @@ fn generic_simd_intrinsic( let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); // Bitcast to iN: let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); - // Zero-extend iN to the bitmask type: - return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + + match ret_ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { + // Zero-extend iN to the bitmask type: + return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + } + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + // Zero-extend iN to the array lengh: + let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); + + // Convert the integer to a byte array + let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + bx.store(ze, ptr, Align::ONE); + let array_ty = bx.type_array(bx.type_i8(), expected_bytes); + let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); + return Ok(bx.load(array_ty, ptr, Align::ONE)); + } + _ => return_error!( + "cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret_ty, + expected_int_bits, + expected_bytes + ), + } } fn simd_simple_float_intrinsic( diff --git a/src/test/ui/simd/intrinsic/generic-bitmask.rs b/src/test/ui/simd/intrinsic/generic-bitmask.rs index 92c4e67dfdd19..9a23dae77b96e 100644 --- a/src/test/ui/simd/intrinsic/generic-bitmask.rs +++ b/src/test/ui/simd/intrinsic/generic-bitmask.rs @@ -51,19 +51,19 @@ fn main() { let _: u64 = simd_bitmask(m64); let _: u16 = simd_bitmask(m2); - //~^ ERROR bitmask `u16`, expected `u8` + //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic let _: u16 = simd_bitmask(m8); - //~^ ERROR bitmask `u16`, expected `u8` + //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic let _: u32 = simd_bitmask(m16); - //~^ ERROR bitmask `u32`, expected `u16` + //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic let _: u64 = simd_bitmask(m32); - //~^ ERROR bitmask `u64`, expected `u32` + //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic let _: u128 = simd_bitmask(m64); - //~^ ERROR bitmask `u128`, expected `u64` + //~^ ERROR invalid monomorphization of `simd_bitmask` intrinsic } } diff --git a/src/test/ui/simd/intrinsic/generic-bitmask.stderr b/src/test/ui/simd/intrinsic/generic-bitmask.stderr index 5aaae68cafb34..0de3f8eead86d 100644 --- a/src/test/ui/simd/intrinsic/generic-bitmask.stderr +++ b/src/test/ui/simd/intrinsic/generic-bitmask.stderr @@ -1,28 +1,28 @@ -error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u16`, expected `u8` +error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u16`, expected `u8` or `[u8; 1]` --> $DIR/generic-bitmask.rs:53:22 | LL | let _: u16 = simd_bitmask(m2); | ^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u16`, expected `u8` +error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u16`, expected `u8` or `[u8; 1]` --> $DIR/generic-bitmask.rs:56:22 | LL | let _: u16 = simd_bitmask(m8); | ^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u32`, expected `u16` +error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u32`, expected `u16` or `[u8; 2]` --> $DIR/generic-bitmask.rs:59:22 | LL | let _: u32 = simd_bitmask(m16); | ^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u64`, expected `u32` +error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u64`, expected `u32` or `[u8; 4]` --> $DIR/generic-bitmask.rs:62:22 | LL | let _: u64 = simd_bitmask(m32); | ^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: bitmask `u128`, expected `u64` +error[E0511]: invalid monomorphization of `simd_bitmask` intrinsic: cannot return `u128`, expected `u64` or `[u8; 8]` --> $DIR/generic-bitmask.rs:65:23 | LL | let _: u128 = simd_bitmask(m64); diff --git a/src/test/ui/simd/intrinsic/generic-select.rs b/src/test/ui/simd/intrinsic/generic-select.rs index 7d68af49e2868..248e82ea21cfc 100644 --- a/src/test/ui/simd/intrinsic/generic-select.rs +++ b/src/test/ui/simd/intrinsic/generic-select.rs @@ -20,8 +20,7 @@ struct b8x4(pub i8, pub i8, pub i8, pub i8); #[repr(simd)] #[derive(Copy, Clone, PartialEq)] -struct b8x8(pub i8, pub i8, pub i8, pub i8, - pub i8, pub i8, pub i8, pub i8); +struct b8x8(pub i8, pub i8, pub i8, pub i8, pub i8, pub i8, pub i8, pub i8); extern "platform-intrinsic" { fn simd_select(x: T, a: U, b: U) -> U; @@ -50,15 +49,15 @@ fn main() { //~^ ERROR found non-SIMD `u32` simd_select_bitmask(0u16, x, x); - //~^ ERROR mask length `16` != other vector length `4` - // + //~^ ERROR invalid bitmask `u16`, expected `u8` or `[u8; 1]` + simd_select_bitmask(0u8, 1u32, 2u32); //~^ ERROR found non-SIMD `u32` simd_select_bitmask(0.0f32, x, x); - //~^ ERROR `f32` is not an integral type + //~^ ERROR invalid bitmask `f32`, expected `u8` or `[u8; 1]` simd_select_bitmask("x", x, x); - //~^ ERROR `&str` is not an integral type + //~^ ERROR invalid bitmask `&str`, expected `u8` or `[u8; 1]` } } diff --git a/src/test/ui/simd/intrinsic/generic-select.stderr b/src/test/ui/simd/intrinsic/generic-select.stderr index c53d581745a7c..d576f1bc77473 100644 --- a/src/test/ui/simd/intrinsic/generic-select.stderr +++ b/src/test/ui/simd/intrinsic/generic-select.stderr @@ -1,47 +1,47 @@ error[E0511]: invalid monomorphization of `simd_select` intrinsic: mismatched lengths: mask length `8` != other vector length `4` - --> $DIR/generic-select.rs:40:9 + --> $DIR/generic-select.rs:39:9 | LL | simd_select(m8, x, x); | ^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_select` intrinsic: mask element type is `u32`, expected `i_` - --> $DIR/generic-select.rs:43:9 + --> $DIR/generic-select.rs:42:9 | LL | simd_select(x, x, x); | ^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_select` intrinsic: mask element type is `f32`, expected `i_` - --> $DIR/generic-select.rs:46:9 + --> $DIR/generic-select.rs:45:9 | LL | simd_select(z, z, z); | ^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_select` intrinsic: expected SIMD argument type, found non-SIMD `u32` - --> $DIR/generic-select.rs:49:9 + --> $DIR/generic-select.rs:48:9 | LL | simd_select(m4, 0u32, 1u32); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: mismatched lengths: mask length `16` != other vector length `4` - --> $DIR/generic-select.rs:52:9 +error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `u16`, expected `u8` or `[u8; 1]` + --> $DIR/generic-select.rs:51:9 | LL | simd_select_bitmask(0u16, x, x); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: expected SIMD argument type, found non-SIMD `u32` - --> $DIR/generic-select.rs:55:9 + --> $DIR/generic-select.rs:54:9 | LL | simd_select_bitmask(0u8, 1u32, 2u32); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: `f32` is not an integral type - --> $DIR/generic-select.rs:58:9 +error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `f32`, expected `u8` or `[u8; 1]` + --> $DIR/generic-select.rs:57:9 | LL | simd_select_bitmask(0.0f32, x, x); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: `&str` is not an integral type - --> $DIR/generic-select.rs:61:9 +error[E0511]: invalid monomorphization of `simd_select_bitmask` intrinsic: invalid bitmask `&str`, expected `u8` or `[u8; 1]` + --> $DIR/generic-select.rs:60:9 | LL | simd_select_bitmask("x", x, x); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/test/ui/simd/simd-bitmask.rs b/src/test/ui/simd/simd-bitmask.rs new file mode 100644 index 0000000000000..14ee2e741bdfd --- /dev/null +++ b/src/test/ui/simd/simd-bitmask.rs @@ -0,0 +1,52 @@ +//run-pass +//ignore-endian-big behavior of simd_select_bitmask is endian-specific +#![feature(repr_simd, platform_intrinsics)] + +extern "platform-intrinsic" { + fn simd_bitmask(v: T) -> U; + fn simd_select_bitmask(m: T, a: U, b: U) -> U; +} + +#[derive(Copy, Clone)] +#[repr(simd)] +struct Simd([T; N]); + +fn main() { + unsafe { + let v = Simd::([-1, 0, -1, 0]); + let i: u8 = simd_bitmask(v); + let a: [u8; 1] = simd_bitmask(v); + + assert_eq!(i, 0b0101); + assert_eq!(a, [0b0101]); + + let v = Simd::([0, 0, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, -1, 0]); + let i: u16 = simd_bitmask(v); + let a: [u8; 2] = simd_bitmask(v); + + assert_eq!(i, 0b0101000000001100); + assert_eq!(a, [0b1100, 0b01010000]); + } + + unsafe { + let a = Simd::([0, 1, 2, 3, 4, 5, 6, 7]); + let b = Simd::([8, 9, 10, 11, 12, 13, 14, 15]); + let e = [0, 9, 2, 11, 12, 13, 14, 15]; + + let r = simd_select_bitmask(0b0101u8, a, b); + assert_eq!(r.0, e); + + let r = simd_select_bitmask([0b0101u8], a, b); + assert_eq!(r.0, e); + + let a = Simd::([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = Simd::([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let e = [16, 17, 2, 3, 20, 21, 22, 23, 24, 25, 26, 27, 12, 29, 14, 31]; + + let r = simd_select_bitmask(0b0101000000001100u16, a, b); + assert_eq!(r.0, e); + + let r = simd_select_bitmask([0b1100u8, 0b01010000u8], a, b); + assert_eq!(r.0, e); + } +}