Add memory space information to BP4 Variables

bindings/CXX11/adios2/cxx11/Variable.cpp

@@ -32,6 +32,12 @@ namespace adios2
     }                                                                          \
                                                                                \
     template <>                                                                \
+    void Variable<T>::SetMemorySpace(const MemorySpace mem)                    \
+    {                                                                          \
+        m_Variable->SetMemorySpace(mem);                                       \
+    }                                                                          \
+                                                                               \
+    template <>                                                                \
     void Variable<T>::SetShape(const Dims &shape)                              \
     {                                                                          \
         helper::CheckForNullptr(m_Variable,                                    \

bindings/CXX11/adios2/cxx11/Variable.h

@@ -147,6 +147,11 @@ class Variable
     /** Checks if object is valid, e.g. if( variable ) { //..valid } */
     explicit operator bool() const noexcept;
 
+    /**
+     * Sets the memory step for all following Puts
+     */
+    void SetMemorySpace(const MemorySpace mem);
+
     /**
      * Set new shape, care must be taken when reading back the variable for
      * different steps. Only applies to Global arrays.

source/adios2/common/ADIOSTypes.h

@@ -32,6 +32,14 @@
 namespace adios2
 {
 
+/** Memory space for the buffers received with Put */
+enum class MemorySpace
+{
+    Detect, ///< Detect the memory space automatically
+    Host,   ///< Host memory space (default)
+    CUDA    ///< GPU memory spaces
+};
+
 /** Variable shape type identifier, assigned automatically from the signature of
  *  DefineVariable */
 enum class ShapeID

source/adios2/core/Variable.cpp

@@ -49,6 +49,7 @@ namespace core
         info.StepsCount = stepsCount;                                          \
         info.Data = const_cast<T *>(data);                                     \
         info.Operations = m_Operations;                                        \
+        info.IsGPU = IsCUDAPointer((void *)data);                              \
         m_BlocksInfo.push_back(info);                                          \
         return m_BlocksInfo.back();                                            \
     }                                                                          \

source/adios2/core/Variable.h

@@ -112,6 +112,7 @@ class Variable : public VariableBase
         SelectionType Selection = SelectionType::BoundingBox;
         bool IsValue = false;
         bool IsReverseDims = false;
+        bool IsGPU = false;
     };
 
     /** use for multiblock info */

source/adios2/core/VariableBase.cpp

@@ -38,11 +38,32 @@ VariableBase::VariableBase(const std::string &name, const DataType type,
     InitShapeType();
 }
 
+bool VariableBase::IsCUDAPointer(void *ptr)
+{
+    if (m_MemorySpace == MemorySpace::CUDA)
+        return true;
+    if (m_MemorySpace == MemorySpace::Host)
+        return false;
+
+#ifdef ADIOS2_HAVE_CUDA
+    cudaPointerAttributes attr;
+    cudaPointerGetAttributes(&attr, ptr);
+    return attr.type == cudaMemoryTypeDevice;
+#endif
+
+    return false;
+}
+
 size_t VariableBase::TotalSize() const noexcept
 {
     return helper::GetTotalSize(m_Count);
 }
 
+void VariableBase::SetMemorySpace(const MemorySpace mem)
+{
+    m_MemorySpace = mem;
+}
+
 void VariableBase::SetShape(const adios2::Dims &shape)
 {
     if (m_Type == helper::GetDataType<std::string>())

source/adios2/core/VariableBase.h

@@ -46,6 +46,7 @@ class VariableBase
     /** Variable -> sizeof(T),
      *  VariableCompound -> from constructor sizeof(struct) */
     const size_t m_ElementSize;
+    MemorySpace m_MemorySpace = MemorySpace::Host;
 
     ShapeID m_ShapeID = ShapeID::Unknown; ///< see shape types in ADIOSTypes.h
     size_t m_BlockID = 0; ///< current block ID for local variables, global = 0
@@ -124,6 +125,18 @@ class VariableBase
      */
     size_t TotalSize() const noexcept;
 
+    /**
+     * Check if buffer is allocated on CUDA space
+     * @param pointer to the user data
+     */
+    bool IsCUDAPointer(void *ptr);
+
+    /**
+     * Set the memory space
+     * @param the memory space where the expected buffers were allocated
+     */
+    void SetMemorySpace(const MemorySpace mem);
+
     /**
      * Set new shape
      * @param shape input shape to be applied to this variable