Merge branch 'dev' into topic-streaming

.clang-tidy

@@ -2,5 +2,5 @@
 # FIXME: all performance-* reports
 # FIXME: all cert-* reports
 # FIXME: all bugprone-* reports
-Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays
+Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
 HeaderFilterRegex: '((^(?!\/share\/openPMD\/).*)*include\/openPMD\/.+\.hpp|src\/^(?!binding).+\.cpp$)'

docs/source/details/adios2.json

@@ -8,10 +8,13 @@
       }
     },
     "dataset": {
-      "operators": [ 
+      "operators": [
         {
-          "type": "bzip2",
-          "parameters": {}
+          "type": "blosc",
+          "parameters": {
+              "clevel": "1",
+              "doshuffle": "BLOSC_BITSHUFFLE"
+          }
         }
       ]
     }

docs/source/details/backendconfig.rst

@@ -19,7 +19,6 @@ Generally, keys of the configuration are *lower case*.
 Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions.
 
 The configuration string may refer to the complete ``openPMD::Series`` or may additionally be specified per ``openPMD::Dataset``, passed in the respective constructors.
-*A configuration per dataset is currently not yet implemented.*
 This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression).
 
 For a consistent user interface, backends shall follow the following rules:
@@ -50,12 +49,15 @@ Explanation of the single keys:
 * ``adios2.engine.parameters``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``.
   Please refer to the official ADIOS2 documentation for the allowable engine parameters.
 * ``adios2.engine.usesteps``: Described more closely in the documentation for the :ref:`ADIOS2 backend<backends-adios2>`.
-* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators <https://adios2.readthedocs.io/en/latest/components/components.html#operator>`_, used to enable compression or dataset transformations.
-  Each object in the list has three keys:
+* ``adios2.dataset.operators``: This key contains a list of ADIOS2 `operators <https://adios2.readthedocs.io/en/latest/components/components.html#operator>`_, used to enable compression or dataset transformations.
+  Each object in the list has two keys:
 
   * ``type`` supported ADIOS operator type, e.g. zfp, sz
   * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels)
 
+Any setting specified under ``adios2.dataset`` is applicable globally as well as on a per-dataset level.
+Any setting under ``adios2.engine`` is applicable globally only.
+
 Other backends
 ^^^^^^^^^^^^^^
 

include/openPMD/Dataset.hpp

@@ -38,7 +38,7 @@ class Dataset
     friend class RecordComponent;
 
 public:
-    Dataset(Datatype, Extent);
+    Dataset(Datatype, Extent, std::string options = "{}");
 
     Dataset& extend(Extent newExtent);
     Dataset& setChunkSize(Extent const&);
@@ -51,5 +51,6 @@ class Dataset
     Extent chunkSize;
     std::string compression;
     std::string transform;
+    std::string options = "{}"; //!< backend-dependent JSON configuration
 };
-} // openPMD
+} // namespace openPMD

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -199,8 +199,8 @@ class ADIOS2IOHandlerImpl
 
     struct ParameterizedOperator
     {
-        adios2::Operator const op;
-        adios2::Params const params;
+        adios2::Operator op;
+        adios2::Params params;
     };
 
     std::vector< ParameterizedOperator > defaultOperators;
@@ -239,11 +239,11 @@ class ADIOS2IOHandlerImpl
      * @return first parameter: the operators, second parameters: whether
      * operators have been configured
      */
-    std::pair< std::vector< ParameterizedOperator >, bool >
+    auxiliary::Option< std::vector< ParameterizedOperator > >
     getOperators( auxiliary::TracingJSON config );
 
     // use m_config
-    std::pair< std::vector< ParameterizedOperator >, bool >
+    auxiliary::Option< std::vector< ParameterizedOperator > >
     getOperators();
 
     std::string

include/openPMD/IO/IOTask.hpp

@@ -259,7 +259,7 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_DATASET > : public Abstrac
     Parameter(Parameter const & p) : AbstractParameter(),
         name(p.name), extent(p.extent), dtype(p.dtype),
         chunkSize(p.chunkSize), compression(p.compression),
-        transform(p.transform) {}
+        transform(p.transform), options(p.options) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -274,6 +274,7 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_DATASET > : public Abstrac
     Extent chunkSize = {};
     std::string compression = "";
     std::string transform = "";
+    std::string options = "{}";
 };
 
 template<>

src/Dataset.cpp

@@ -26,11 +26,12 @@
 
 namespace openPMD
 {
-Dataset::Dataset(Datatype d, Extent e)
+Dataset::Dataset(Datatype d, Extent e, std::string options_in)
         : extent{e},
           dtype{d},
           rank{static_cast<uint8_t>(e.size())},
-          chunkSize{e}
+          chunkSize{e},
+          options{std::move(options_in)}
 { }
 
 Dataset&

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -127,7 +127,6 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg )
         return;
     }
     m_config = std::move( cfg[ "adios2" ] );
-    defaultOperators = getOperators().first;
     auto engineConfig = config( ADIOS2Defaults::str_engine );
     if( !engineConfig.json().is_null() )
     {
@@ -144,20 +143,26 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg )
                 []( unsigned char c ) { return std::tolower( c ); } );
         }
     }
+    auto operators = getOperators();
+    if( operators )
+    {
+        defaultOperators = std::move( operators.get() );
+    }
 }
 
-std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool >
+auxiliary::Option< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > >
 ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg )
 {
+    using ret_t = auxiliary::Option< std::vector< ParameterizedOperator > >;
     std::vector< ParameterizedOperator > res;
     if( !cfg.json().contains( "dataset" ) )
     {
-        return std::make_pair( res, false );
+        return ret_t();
     }
     auto datasetConfig = cfg[ "dataset" ];
     if( !datasetConfig.json().contains( "operators" ) )
     {
-        return std::make_pair( res, false );
+        return ret_t();
     }
     auto _operators = datasetConfig[ "operators" ];
     nlohmann::json const & operators = _operators.json();
@@ -188,10 +193,10 @@ ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg )
         }
     }
     _operators.declareFullyRead();
-    return std::make_pair( res, true );
+    return auxiliary::makeOption( std::move( res ) );
 }
 
-std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool >
+auxiliary::Option< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > >
 ADIOS2IOHandlerImpl::getOperators()
 {
     return getOperators( m_config );
@@ -331,7 +336,30 @@ void ADIOS2IOHandlerImpl::createDataset(
         filePos->gd = ADIOS2FilePosition::GD::DATASET;
         auto const varName = filePositionToString( filePos );
 
-        auto operators = defaultOperators;
+        std::vector< ParameterizedOperator > operators;
+        nlohmann::json options = nlohmann::json::parse( parameters.options );
+        if( options.contains( "adios2" ) )
+        {
+            auxiliary::TracingJSON datasetConfig( options[ "adios2" ] );
+            auto datasetOperators = getOperators( datasetConfig );
+
+            operators = datasetOperators ? std::move( datasetOperators.get() )
+                                         : defaultOperators;
+
+            auto shadow = datasetConfig.invertShadow();
+            if( shadow.size() > 0 )
+            {
+                std::cerr << "Warning: parts of the JSON configuration for "
+                             "ADIOS2 dataset '"
+                          << varName << "' remain unused:\n"
+                          << shadow << std::endl;
+            }
+        }
+        else
+        {
+            operators = defaultOperators;
+        }
+
         if( !parameters.compression.empty() )
         {
             auxiliary::Option< adios2::Operator > adiosOperator =

src/RecordComponent.cpp

@@ -169,6 +169,7 @@ RecordComponent::flush(std::string const& name)
                 dCreate.chunkSize = m_dataset->chunkSize;
                 dCreate.compression = m_dataset->compression;
                 dCreate.transform = m_dataset->transform;
+                dCreate.options = m_dataset->options;
                 IOHandler->enqueue(IOTask(this, dCreate));
             }
         }

src/backend/PatchRecordComponent.cpp

@@ -88,6 +88,7 @@ PatchRecordComponent::flush(std::string const& name)
             dCreate.chunkSize = getExtent();
             dCreate.compression = m_dataset->compression;
             dCreate.transform = m_dataset->transform;
+            dCreate.options = m_dataset->options;
             IOHandler->enqueue(IOTask(this, dCreate));
         }
 

src/binding/python/Dataset.cpp

@@ -42,6 +42,15 @@ void init_Dataset(py::module &m) {
         }),
             py::arg("dtype"), py::arg("extent")
         )
+        .def(py::init<Datatype, Extent, std::string>(),
+            py::arg("dtype"), py::arg("extent"), py::arg("options")
+        )
+        .def(py::init( [](py::dtype dt, Extent e, std::string options) {
+            auto const d = dtype_from_numpy( dt );
+            return new Dataset{d, e, std::move(options)};
+        }),
+            py::arg("dtype"), py::arg("extent"), py::arg("options")
+        )
 
         .def("__repr__",
             [](const Dataset &d) {
@@ -61,6 +70,7 @@ void init_Dataset(py::module &m) {
         .def_property_readonly("dtype", [](const Dataset &d) {
             return dtype_to_numpy( d.dtype );
         })
+        .def_readwrite("options", &Dataset::options)
     ;
 }
 

test/SerialIOTest.cpp

@@ -2876,14 +2876,40 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" )
   }
 }
 )END";
-    auto const write = []( std::string const & filename,
-                     std::string const & config ) {
+    std::string datasetConfig = R"END(
+{
+  "adios2": {
+    "unused": "dataset parameter",
+    "dataset": {
+      "unused": "too",
+      "operators": [
+        {
+          "type": "blosc",
+          "parameters": {
+              "clevel": "3",
+              "doshuffle": "BLOSC_BITSHUFFLE"
+          }
+        }
+      ]
+    }
+  }
+}
+)END";
+    auto const write = [ &datasetConfig ](
+                           std::string const & filename,
+                           std::string const & config ) {
         openPMD::Series series( filename, openPMD::Access::CREATE, config );
         auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ];
-        E_x.resetDataset(
-            openPMD::Dataset( openPMD::Datatype::INT, { 1000 } ) );
+        openPMD::Dataset ds( openPMD::Datatype::INT, { 1000 } );
+        E_x.resetDataset( ds );
         std::vector< int > data( 1000, 0 );
         E_x.storeChunk( data, { 0 }, { 1000 } );
+
+        auto E_y = series.iterations[ 0 ].meshes[ "E" ][ "y" ];
+        // let's override the global compression settings
+        ds.options = datasetConfig;
+        E_y.resetDataset( ds );
+        E_y.storeChunk( data, { 0 }, { 1000 } );
         series.flush();
     };
     write( "../samples/jsonConfiguredBP4.bp", writeConfigBP4 );
@@ -2915,7 +2941,8 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" )
   }
 }
 )END";
-    auto const read = []( std::string const & filename, std::string const & config ) {
+    auto const read = []( std::string const & filename,
+                          std::string const & config ) {
         openPMD::Series series(
             filename, openPMD::Access::READ_ONLY, config );
         auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ];
@@ -2927,6 +2954,16 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" )
         {
             REQUIRE( chunk.get()[ i ] == 0 );
         }
+
+        auto E_y = series.iterations[ 0 ].meshes[ "E" ][ "x" ];
+        REQUIRE( E_y.getDimensionality() == 1 );
+        REQUIRE( E_y.getExtent()[ 0 ] == 1000 );
+        chunk = E_y.loadChunk< int >( { 0 }, { 1000 } );
+        series.flush();
+        for( size_t i = 0; i < 1000; ++i )
+        {
+            REQUIRE( chunk.get()[ i ] == 0 );
+        }
     };
     read( "../samples/jsonConfiguredBP3.bp", readConfigBP3 );
     read( "../samples/jsonConfiguredBP4.bp", readConfigBP4 );