From 13da30b1554326ea0c732e944c6c9e9fa980c394 Mon Sep 17 00:00:00 2001 From: Joel Falcou Date: Thu, 24 Nov 2022 14:27:29 +0100 Subject: [PATCH] Implement some SVE functions --- cmake/toolchain/specific.json | 9 +++- include/eve/module/core/regular/all.hpp | 4 ++ include/eve/module/core/regular/any.hpp | 4 ++ .../eve/module/core/regular/bit_andnot.hpp | 4 ++ .../eve/module/core/regular/count_true.hpp | 4 ++ .../module/core/regular/impl/bit_andnot.hpp | 14 +++--- .../core/regular/impl/simd/arm/sve/all.hpp | 28 ++++++++++++ .../core/regular/impl/simd/arm/sve/any.hpp | 27 ++++++++++++ .../regular/impl/simd/arm/sve/bit_andnot.hpp | 32 ++++++++++++++ .../regular/impl/simd/arm/sve/count_true.hpp | 35 +++++++++++++++ .../core/regular/impl/simd/arm/sve/sqrt.hpp | 43 +++++++++++++++++++ include/eve/module/core/regular/impl/sqrt.hpp | 6 +-- include/eve/module/core/regular/sqrt.hpp | 4 ++ include/eve/module/core/saturated/add.hpp | 4 ++ .../core/saturated/impl/simd/arm/sve/add.hpp | 25 +++++++++++ 15 files changed, 232 insertions(+), 11 deletions(-) create mode 100644 include/eve/module/core/regular/impl/simd/arm/sve/all.hpp create mode 100644 include/eve/module/core/regular/impl/simd/arm/sve/any.hpp create mode 100644 include/eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp create mode 100644 include/eve/module/core/regular/impl/simd/arm/sve/count_true.hpp create mode 100644 include/eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp create mode 100644 include/eve/module/core/saturated/impl/simd/arm/sve/add.hpp diff --git a/cmake/toolchain/specific.json b/cmake/toolchain/specific.json index 2101d3d92d..c2bc39cdd5 100644 --- a/cmake/toolchain/specific.json +++ b/cmake/toolchain/specific.json @@ -22,9 +22,16 @@ "unit.api.regular.swizzle.zero.exe": "^unit.api.regular.swizzle.zero.exe", "unit.api.regular.wide.exe": "^unit.api.regular.wide.exe", + "unit.core.add.exe": "^unit.core.add.exe", + "unit.core.all.exe": "^unit.core.all.exe", + "unit.core.any.exe": "^unit.core.any.exe", + "unit.core.bit_andnot.exe": "^unit.core.bit_andnot.exe", "unit.core.bit_cast.exe": "^unit.core.bit_cast.exe", "unit.core.convert.exe": "^unit.core.convert.*.exe", + "unit.core.count_true.exe": "^unit.core.count_true.exe", + "unit.core.if_else.exe": "^unit.core.min.exe", "unit.core.max.exe": "^unit.core.max.exe", "unit.core.min.exe": "^unit.core.min.exe", - "unit.core.if_else.exe": "^unit.core.min.exe" + "unit.core.none.exe": "^unit.core.none.exe", + "unit.core.sqrt.exe": "^unit.core.sqrt.exe" } diff --git a/include/eve/module/core/regular/all.hpp b/include/eve/module/core/regular/all.hpp index 9a19b572f5..d151505cfa 100644 --- a/include/eve/module/core/regular/all.hpp +++ b/include/eve/module/core/regular/all.hpp @@ -79,3 +79,7 @@ EVE_MAKE_CALLABLE(all_, all); #if defined(EVE_INCLUDE_ARM_HEADER) # include #endif + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/regular/any.hpp b/include/eve/module/core/regular/any.hpp index 11b4c35e2d..77540d6225 100644 --- a/include/eve/module/core/regular/any.hpp +++ b/include/eve/module/core/regular/any.hpp @@ -80,3 +80,7 @@ EVE_MAKE_CALLABLE(any_, any); #if defined(EVE_INCLUDE_ARM_HEADER) # include #endif + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/regular/bit_andnot.hpp b/include/eve/module/core/regular/bit_andnot.hpp index 31222fc344..4bcfb9cd74 100644 --- a/include/eve/module/core/regular/bit_andnot.hpp +++ b/include/eve/module/core/regular/bit_andnot.hpp @@ -79,3 +79,7 @@ EVE_MAKE_CALLABLE(bit_andnot_, bit_andnot); #if defined(EVE_INCLUDE_ARM_HEADER) # include #endif + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/regular/count_true.hpp b/include/eve/module/core/regular/count_true.hpp index a3f9121f52..d7cf96e215 100644 --- a/include/eve/module/core/regular/count_true.hpp +++ b/include/eve/module/core/regular/count_true.hpp @@ -67,3 +67,7 @@ EVE_MAKE_CALLABLE(count_true_, count_true); } #include + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/regular/impl/bit_andnot.hpp b/include/eve/module/core/regular/impl/bit_andnot.hpp index 2dc031e9f4..c91ec7618c 100644 --- a/include/eve/module/core/regular/impl/bit_andnot.hpp +++ b/include/eve/module/core/regular/impl/bit_andnot.hpp @@ -21,7 +21,7 @@ namespace eve::detail { -template +template EVE_FORCEINLINE auto bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, @@ -30,7 +30,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_), return bit_call(bit_andnot, a, b); } -template +template EVE_FORCEINLINE auto bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept { @@ -42,8 +42,8 @@ bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept else return T(a & ~b); } -template -EVE_FORCEINLINE auto +template +EVE_FORCEINLINE T bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept { return apply_over(bit_andnot, a, b); // fallback never taken if proper intrinsics are at hand @@ -51,7 +51,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept // ----------------------------------------------------------------------------------------------- // Masked case -template +template EVE_FORCEINLINE auto bit_andnot_(EVE_SUPPORTS(cpu_), C const& cond, @@ -64,7 +64,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_), //================================================================================================ // N parameters //================================================================================================ -template +template auto bit_andnot_(EVE_SUPPORTS(cpu_), T0 a0, T1 a1, Ts... args) requires bit_compatible_values &&(bit_compatible_values&&...) @@ -73,7 +73,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_), T0 a0, T1 a1, Ts... args) requires return bit_andnot(a0, that); } -template +template auto bit_andnot_(EVE_SUPPORTS(cpu_), C const& cond, T0 a0, T1 a1, Ts... args) requires bit_compatible_values &&(bit_compatible_values&&...) diff --git a/include/eve/module/core/regular/impl/simd/arm/sve/all.hpp b/include/eve/module/core/regular/impl/simd/arm/sve/all.hpp new file mode 100644 index 0000000000..e5a071b623 --- /dev/null +++ b/include/eve/module/core/regular/impl/simd/arm/sve/all.hpp @@ -0,0 +1,28 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE bool +all_(EVE_SUPPORTS(sve_), C const& cond, logical> const& v) noexcept +requires sve_abi> +{ + if constexpr( C::is_complete ) + { + if constexpr( C::is_inverted ) return count_true(v) == N::value; + else return true; + } + else return count_true(cond, v) == cond.count(as>()); +} +} diff --git a/include/eve/module/core/regular/impl/simd/arm/sve/any.hpp b/include/eve/module/core/regular/impl/simd/arm/sve/any.hpp new file mode 100644 index 0000000000..1e113736b4 --- /dev/null +++ b/include/eve/module/core/regular/impl/simd/arm/sve/any.hpp @@ -0,0 +1,27 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE bool +any_(EVE_SUPPORTS(sve_), C const& cond, logical> const& v) noexcept +requires sve_abi> +{ + if constexpr( C::is_complete ) + { + if constexpr( C::is_inverted ) return svptest_any(sve_true(), v); + else return false; + } + else return svptest_any(cond.mask(as>{}),v); +} +} diff --git a/include/eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp b/include/eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp new file mode 100644 index 0000000000..d55b03b224 --- /dev/null +++ b/include/eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp @@ -0,0 +1,32 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE auto +bit_andnot_(EVE_SUPPORTS(sve_), wide const& v0, wide const& v1) noexcept -> wide +requires sve_abi> +{ + using i_t = typename wide::template rebind,N>; + constexpr auto tgt = as(); + return bit_cast ( i_t ( svbic_x ( sve_true() + , bit_cast(v0,tgt) + , bit_cast(v1,tgt) + ) + ) + , as>{} + ); +} +} diff --git a/include/eve/module/core/regular/impl/simd/arm/sve/count_true.hpp b/include/eve/module/core/regular/impl/simd/arm/sve/count_true.hpp new file mode 100644 index 0000000000..abf52b21cf --- /dev/null +++ b/include/eve/module/core/regular/impl/simd/arm/sve/count_true.hpp @@ -0,0 +1,35 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE std::ptrdiff_t + count_true_(EVE_SUPPORTS(cpu_), C cond, logical> v) noexcept +requires sve_abi> +{ + auto const m = cond.mask(as>{}); + if constexpr(sizeof(T) == 1) return svcntp_b8(m,v); + else if constexpr(sizeof(T) == 2) return svcntp_b16(m,v); + else if constexpr(sizeof(T) == 4) return svcntp_b32(m,v); + else if constexpr(sizeof(T) == 8) return svcntp_b64(m,v); +} + +template +EVE_FORCEINLINE std::ptrdiff_t + count_true_(EVE_SUPPORTS(sve_), logical> v) noexcept +requires sve_abi> +{ + return count_true[ignore_none](v); +} +} diff --git a/include/eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp b/include/eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp new file mode 100644 index 0000000000..170b499efa --- /dev/null +++ b/include/eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp @@ -0,0 +1,43 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE auto +sqrt_(EVE_SUPPORTS(sve_), C const& cond, V const& v) noexcept -> V +requires sve_abi +{ + if constexpr( C::is_complete ) + { + if constexpr( C::is_inverted ) return svsqrt_x(sve_true>(), v); + else return alternative(cond, v, as {}); + } + else + { + auto src = alternative(cond, v, as {}); + auto m = expand_mask(cond, as {}); + return svsqrt_m(src, m, v); + } +} + +template +EVE_FORCEINLINE auto +sqrt_(EVE_SUPPORTS(sve_), wide const& v) noexcept -> wide +requires sve_abi> +{ + return sqrt[ignore_none](v); +} + +} diff --git a/include/eve/module/core/regular/impl/sqrt.hpp b/include/eve/module/core/regular/impl/sqrt.hpp index 00f0007f41..2b1058e332 100644 --- a/include/eve/module/core/regular/impl/sqrt.hpp +++ b/include/eve/module/core/regular/impl/sqrt.hpp @@ -17,14 +17,14 @@ namespace eve::detail { -template +template EVE_FORCEINLINE constexpr T sqrt_(EVE_SUPPORTS(cpu_), raw_type const&, T const& a) noexcept { return eve::sqrt(a); } -template +template EVE_FORCEINLINE constexpr T sqrt_(EVE_SUPPORTS(cpu_), T const& a0) noexcept { @@ -35,7 +35,7 @@ sqrt_(EVE_SUPPORTS(cpu_), T const& a0) noexcept // ----------------------------------------------------------------------------------------------- // Masked case -template +template EVE_FORCEINLINE auto sqrt_(EVE_SUPPORTS(cpu_), C const& cond, U const& t) noexcept { diff --git a/include/eve/module/core/regular/sqrt.hpp b/include/eve/module/core/regular/sqrt.hpp index d7d9eff420..7973778d36 100644 --- a/include/eve/module/core/regular/sqrt.hpp +++ b/include/eve/module/core/regular/sqrt.hpp @@ -109,3 +109,7 @@ EVE_MAKE_CALLABLE(sqrt_, sqrt); #if defined(EVE_INCLUDE_ARM_HEADER) # include #endif + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/saturated/add.hpp b/include/eve/module/core/saturated/add.hpp index 383553c9b3..8b9948d8da 100644 --- a/include/eve/module/core/saturated/add.hpp +++ b/include/eve/module/core/saturated/add.hpp @@ -13,3 +13,7 @@ #if defined(EVE_INCLUDE_X86_HEADER) # include #endif + +#if defined(EVE_INCLUDE_SVE_HEADER) +# include +#endif diff --git a/include/eve/module/core/saturated/impl/simd/arm/sve/add.hpp b/include/eve/module/core/saturated/impl/simd/arm/sve/add.hpp new file mode 100644 index 0000000000..2eb218639c --- /dev/null +++ b/include/eve/module/core/saturated/impl/simd/arm/sve/add.hpp @@ -0,0 +1,25 @@ +//================================================================================================== +/* + EVE - Expressive Vector Engine + Copyright : EVE Project Contributors + SPDX-License-Identifier: BSL-1.0 +*/ +//================================================================================================== +#pragma once + +#include +#include +#include +#include + +namespace eve::detail +{ +template +EVE_FORCEINLINE wide +add_(EVE_SUPPORTS(sve_), saturated_type const&, wide const& a, wide const& b) noexcept +requires sve_abi> +{ + if constexpr( std::integral ) return svqadd(a, b); + else return add(a, b); +} +}