Implement some SVE functions

cmake/toolchain/specific.json

@@ -22,9 +22,16 @@
   "unit.api.regular.swizzle.zero.exe":  "^unit.api.regular.swizzle.zero.exe",
   "unit.api.regular.wide.exe":  "^unit.api.regular.wide.exe",
 
+  "unit.core.add.exe":  "^unit.core.add.exe",
+  "unit.core.all.exe":  "^unit.core.all.exe",
+  "unit.core.any.exe":  "^unit.core.any.exe",
+  "unit.core.bit_andnot.exe":  "^unit.core.bit_andnot.exe",
   "unit.core.bit_cast.exe":  "^unit.core.bit_cast.exe",
   "unit.core.convert.exe":  "^unit.core.convert.*.exe",
+  "unit.core.count_true.exe":  "^unit.core.count_true.exe",
+  "unit.core.if_else.exe":  "^unit.core.min.exe",
   "unit.core.max.exe":  "^unit.core.max.exe",
   "unit.core.min.exe":  "^unit.core.min.exe",
-  "unit.core.if_else.exe":  "^unit.core.min.exe"
+  "unit.core.none.exe":  "^unit.core.none.exe",
+  "unit.core.sqrt.exe":  "^unit.core.sqrt.exe"
 }

include/eve/module/core/regular/all.hpp

@@ -79,3 +79,7 @@ EVE_MAKE_CALLABLE(all_, all);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/all.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/all.hpp>
+#endif

include/eve/module/core/regular/any.hpp

@@ -80,3 +80,7 @@ EVE_MAKE_CALLABLE(any_, any);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/any.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/any.hpp>
+#endif

include/eve/module/core/regular/bit_andnot.hpp

@@ -79,3 +79,7 @@ EVE_MAKE_CALLABLE(bit_andnot_, bit_andnot);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/bit_andnot.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp>
+#endif

include/eve/module/core/regular/count_true.hpp

@@ -67,3 +67,7 @@ EVE_MAKE_CALLABLE(count_true_, count_true);
 }
 
 #include <eve/module/core/regular/impl/count_true.hpp>
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/count_true.hpp>
+#endif

include/eve/module/core/regular/impl/bit_andnot.hpp

@@ -21,7 +21,7 @@
 
 namespace eve::detail
 {
-template<real_value T, real_value U>
+template<value T, value U>
 EVE_FORCEINLINE auto
 bit_andnot_(EVE_SUPPORTS(cpu_),
             T const& a,
@@ -30,7 +30,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_),
   return bit_call(bit_andnot, a, b);
 }
 
-template<real_scalar_value T>
+template<scalar_value T>
 EVE_FORCEINLINE auto
 bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept
 {
@@ -42,16 +42,16 @@ bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept
   else return T(a & ~b);
 }
 
-template<real_simd_value T>
-EVE_FORCEINLINE auto
+template<simd_value T>
+EVE_FORCEINLINE T
 bit_andnot_(EVE_SUPPORTS(cpu_), T const& a, T const& b) noexcept
 {
   return apply_over(bit_andnot, a, b); // fallback never taken if proper intrinsics are at hand
 }
 
 // -----------------------------------------------------------------------------------------------
 // Masked case
-template<conditional_expr C, real_value U, real_value V>
+template<conditional_expr C, value U, value V>
 EVE_FORCEINLINE auto
 bit_andnot_(EVE_SUPPORTS(cpu_),
             C const& cond,
@@ -64,7 +64,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_),
 //================================================================================================
 // N parameters
 //================================================================================================
-template<real_value T0, real_value T1, real_value... Ts>
+template<value T0, value T1, value... Ts>
 auto
 bit_andnot_(EVE_SUPPORTS(cpu_), T0 a0, T1 a1, Ts... args) requires
     bit_compatible_values<T0, T1> &&(bit_compatible_values<T1, Ts>&&...)
@@ -73,7 +73,7 @@ bit_andnot_(EVE_SUPPORTS(cpu_), T0 a0, T1 a1, Ts... args) requires
   return bit_andnot(a0, that);
 }
 
-template<conditional_expr C, real_value T0, real_value T1, real_value... Ts>
+template<conditional_expr C, value T0, value T1, value... Ts>
 auto
 bit_andnot_(EVE_SUPPORTS(cpu_), C const& cond, T0 a0, T1 a1, Ts... args) requires
     bit_compatible_values<T0, T1> &&(bit_compatible_values<T1, Ts>&&...)

include/eve/module/core/regular/impl/simd/arm/sve/all.hpp

@@ -0,0 +1,32 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/detail/has_abi.hpp>
+#include <eve/detail/implementation.hpp>
+#include <eve/module/core/regular/count_true.hpp>
+
+namespace eve::detail
+{
+template<scalar_value T, typename N, relative_conditional_expr C>
+EVE_FORCEINLINE bool
+all_(EVE_SUPPORTS(sve_), C const& cond, logical<wide<T,N>> const& v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && !C::is_inverted ) return true;
+  else  return count_true(cond, v) == cond.count(as<wide<T,N>>());
+}
+
+template<scalar_value T, typename N>
+EVE_FORCEINLINE bool
+all_(EVE_SUPPORTS(sve_), logical<wide<T,N>> v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  return count_true(v) == N::value;
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/any.hpp

@@ -0,0 +1,31 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/detail/has_abi.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<scalar_value T, typename N, relative_conditional_expr C>
+EVE_FORCEINLINE bool
+any_(EVE_SUPPORTS(sve_), C const& cond, logical<wide<T,N>> const& v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && !C::is_inverted ) return false;
+  else return svptest_any(cond.mask(as<wide<T,N>>{}),v);
+}
+
+template<scalar_value T, typename N>
+EVE_FORCEINLINE bool
+any_(EVE_SUPPORTS(sve_), logical<wide<T,N>> v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  return svptest_any(detail::sve_true<T>(),v);
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/bit_andnot.hpp

@@ -0,0 +1,28 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/category.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<scalar_value T, typename N>
+EVE_FORCEINLINE auto
+bit_andnot_(EVE_SUPPORTS(sve_), wide<T, N> const& v0, wide<T, N> const& v1) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  using i_t = typename wide<T, N>::template rebind<as_integer_t<T>,N>;
+  constexpr auto tgt = as<i_t>();
+  return bit_cast ( i_t(svbic_x( sve_true<T>(), bit_cast(v0,tgt), bit_cast(v1,tgt)))
+                  , as<wide<T,N>>{}
+                  );
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/count_true.hpp

@@ -0,0 +1,45 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/conditional.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<scalar_value T, typename N>
+EVE_FORCEINLINE std::ptrdiff_t
+                count_true_(EVE_SUPPORTS(sve_), logical<wide<T,N>> v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  auto mask = []()
+  {
+    if constexpr(N::value == expected_cardinal_v<T>) return sve_true<T>();
+    else return logical<wide<T>>([](auto i, auto) { return i < N::value;});
+  }();
+
+  if constexpr(sizeof(T) == 1) return svcntp_b8(mask,v);
+  else if constexpr(sizeof(T) == 2) return svcntp_b16(mask,v);
+  else if constexpr(sizeof(T) == 4) return svcntp_b32(mask,v);
+  else if constexpr(sizeof(T) == 8) return svcntp_b64(mask,v);
+}
+
+template<scalar_value T, typename N, relative_conditional_expr C>
+EVE_FORCEINLINE std::ptrdiff_t
+                count_true_(EVE_SUPPORTS(cpu_), C cond, logical<wide<T,N>> v) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  auto const m = cond.mask(as<wide<T,N>>{});
+  if constexpr(sizeof(T) == 1) return svcntp_b8(m,v);
+  else if constexpr(sizeof(T) == 2) return svcntp_b16(m,v);
+  else if constexpr(sizeof(T) == 4) return svcntp_b32(m,v);
+  else if constexpr(sizeof(T) == 8) return svcntp_b64(m,v);
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp

@@ -0,0 +1,43 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/category.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<floating_real_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+sqrt_(EVE_SUPPORTS(sve_), wide<T, N> const& v) noexcept ->  wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return svsqrt_x(sve_true<T>(), v);
+}
+
+// -----------------------------------------------------------------------------------------------
+// Masked case
+template<conditional_expr C, value V>
+EVE_FORCEINLINE auto
+sqrt_(EVE_SUPPORTS(sve_), C const& cond, V const& v) noexcept -> V
+requires sve_abi<typename V::abi_type>
+{
+  if constexpr(C::is_complete)
+  {
+    return sqrt_(EVE_RETARGET(cpu_), cond, v);
+  }
+  else
+  {
+    auto src = alternative(cond, v, as<V> {});
+    auto m   = expand_mask(cond, as<V> {});
+    return svsqrt_m(src, m, v);
+  }
+}
+}

include/eve/module/core/regular/impl/sqrt.hpp

@@ -17,14 +17,14 @@
 
 namespace eve::detail
 {
-template<real_value T>
+template<value T>
 EVE_FORCEINLINE constexpr T
 sqrt_(EVE_SUPPORTS(cpu_), raw_type const&, T const& a) noexcept
 {
   return eve::sqrt(a);
 }
 
-template<real_value T>
+template<value T>
 EVE_FORCEINLINE constexpr T
 sqrt_(EVE_SUPPORTS(cpu_), T const& a0) noexcept
 {
@@ -35,7 +35,7 @@ sqrt_(EVE_SUPPORTS(cpu_), T const& a0) noexcept
 
 // -----------------------------------------------------------------------------------------------
 // Masked case
-template<conditional_expr C, real_value U>
+template<conditional_expr C, value U>
 EVE_FORCEINLINE auto
 sqrt_(EVE_SUPPORTS(cpu_), C const& cond, U const& t) noexcept
 {

include/eve/module/core/regular/sqrt.hpp

@@ -109,3 +109,7 @@ EVE_MAKE_CALLABLE(sqrt_, sqrt);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/sqrt.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/sqrt.hpp>
+#endif

include/eve/module/core/saturated/add.hpp

@@ -13,3 +13,7 @@
 #if defined(EVE_INCLUDE_X86_HEADER)
 #  include <eve/module/core/saturated/impl/simd/x86/add.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/saturated/impl/simd/arm/sve/add.hpp>
+#endif

include/eve/module/core/saturated/impl/simd/arm/sve/add.hpp

@@ -0,0 +1,25 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/category.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<scalar_value T, typename N>
+EVE_FORCEINLINE wide<T, N>
+add_(EVE_SUPPORTS(sve_), saturated_type const&, wide<T, N> const& a, wide<T, N> const& b) noexcept
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( std::integral<T> ) return svqadd(a, b);
+  else return add(a, b);
+}
+}