diff --git a/CMakeLists.txt b/CMakeLists.txt index 99fd96e2f..9dcd45ce7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,11 +191,21 @@ add_subdirectory(src) SET(podio_PYTHON_DIR ${PROJECT_SOURCE_DIR}/python CACHE PATH "Path to the podio python directory") if(BUILD_TESTING) + include(ExternalData) + list(APPEND ExternalData_URL_TEMPLATES + "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" + ) include(cmake/podioTest.cmake) add_subdirectory(tests) endif() add_subdirectory(tools) add_subdirectory(python) + +if(BUILD_TESTING) + # Make sure to fetch all data, after all legacy test cases have been added + ExternalData_Add_Target(legacy_test_cases) + message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") +endif() #--- add CMake infrastructure -------------------------------------------------- include(cmake/podioCreateConfig.cmake) diff --git a/cmake/podioTest.cmake b/cmake/podioTest.cmake index fb92ddce7..804e0822b 100644 --- a/cmake/podioTest.cmake +++ b/cmake/podioTest.cmake @@ -3,9 +3,8 @@ function(PODIO_SET_TEST_ENV test) # We need to convert this into a list of arguments that can be used as environment variable list(JOIN PODIO_IO_HANDLERS " " IO_HANDLERS) - set_property(TEST ${test} - PROPERTY ENVIRONMENT - LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$:$<$:$>:$ENV{LD_LIBRARY_PATH} + set(test_environment + LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$:$<$:$>:$ENV{LD_LIBRARY_PATH} PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH} PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests ROOT_INCLUDE_PATH=${PROJECT_BINARY_DIR}/tests/datamodel:${PROJECT_SOURCE_DIR}/include @@ -14,6 +13,10 @@ function(PODIO_SET_TEST_ENV test) PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT} PODIO_BASE=${PROJECT_SOURCE_DIR} ENABLE_SIO=${ENABLE_SIO} + PODIO_BUILD_BASE=${PROJECT_BINARY_DIR} + ) + set_property(TEST ${test} + PROPERTY ENVIRONMENT "${test_environment}" ) endfunction() diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index c2235f565..ad8e23118 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -32,26 +32,14 @@ Before writing out a collection, the data need to be put into the proper structu ### Reading Back-End -There are two possibilities to implement a reading-back end. In case one uses the `podio::EventStore`, one simply has to implement the `IReader` interface. - -If not taking advantage of this implementation, the data reader or the event store have to implement the `ICollectionProvider` interface. Reading of a collection happens then similar to: - -```cpp - // ... - // your creation of the collection and reading of the PODs from disk - // ... - collection->setBuffer(buffer); - auto refCollections = collection->referenceCollections(); - // ... - // your filling of refCollections from disk - // ... - collection->setID( ); - collection->prepareAfterRead(); - // ... - collection->setReferences( &collectionProvider ); -``` - -The strong assumption here is that all references are being followed up directly and no later on-demand reading is done. +The main requirement for a reading backend is its capability of reading back all +the necessary data from which a collection can be constructed in the form of +`podio::CollectionReadBuffers`. From these buffers collections can then be +constructed. Each instance has to contain the (type erased) POD buffers (as a +`std::vector`), the (possibly empty) vectors of `podio::ObjectID`s that contain +the relation information as well the (possibly empty) vectors for the vector +member buffers, which are currently stored as pairs of the type (as a +`std::string`) and (type erased) data buffers in the form of `std::vector`s. ### Dumping JSON @@ -94,12 +82,9 @@ As explained in the section about mutability of data, thread-safety is only guar During the calls of `prepareForWriting` and `prepareAfterReading` on collections other operations like object creation or addition will lead to an inconsistent state. ### Not-thread-safe components -The example event store provided with PODIO is as of writing not thread-safe. Neither is the chosen serialization. - -## Implementing a transient Event Class - -PODIO contains one example `podio::EventStore` class. -To implement your own transient event store, the only requirement is to set the collectionID of each collection to a unique ID on creation. +The Readers and Writers that ship with podio are assumed to run on a single +thread only (more precisely we assume that each Reader or Writer doesn't have to +synchronize with any other for file operations). ## Running pre-commit diff --git a/doc/examples.md b/doc/examples.md index 49f91b856..568563e0b 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -102,28 +102,30 @@ Passing in a size argument is optional; If no argument is passed all elements wi if an argument is passed only as many elements as requested will be returned. If the collection holds less elements than are requested, only as elements as are available will be returned. -### EventStore functionality +### `podio::Frame` container -The event store contained in the package is for *educational* purposes and kept very minimal. It has two main methods: +The `podio::Frame` is the main container for containing and grouping collections +together. It has two main methods: ```cpp - /// create a new collection + /// Store a collection template - T& create(const std::string& name); + const T& put(T&& coll, const std::string& name); - /// access a collection. + /// access a collection template - const T& get(const std::string& name); + const& T get(const std::string& name); ``` -Please note that a `put` method for collections is not foreseen. +Note that for `put`ting collections into the Frame an explicit `std::move` is +necessary to highlight the change of ownership that happens in this case. ### Object Retrieval Collections can be retrieved explicitly: ```cpp - auto& hits = store.get("hits"); + auto& hits = frame.get("hits"); if (hits.isValid()) { ... } ``` @@ -135,51 +137,19 @@ Or implicitly when following an object reference. In both cases the access to da Sometimes it is necessary or useful to store additional data that is not directly foreseen in the EDM. This could be configuration parameters of simulation jobs, or parameter descriptions like cell-ID encoding etc. PODIO currently allows to store such meta data in terms of a `GenericParameters` class that holds an arbitrary number of named parameters of type `int, float, string` or vectors if these. -Meta data can be stored and retrieved from the `EventStore` for runs, collections and events via -the three methods: -```cpp -virtual GenericParameters& EventStore::getRunMetaData(int runID); -virtual GenericParameters& EventStore::getEventMetaData(); -virtual GenericParameters& EventStore::getCollectionMetaData(int colID); -``` - -- example for writing event data: -```cpp -auto& evtMD = store.getEventMetaData() ; -evtMD.setValue( "UserEventWeight" , (float) 100.*i ) ; -``` -- example for reading event data: -```cpp -auto& evtMD = store.getEventMetaData() ; -float evtWeight = evtMD.getFloatVal( "UserEventWeight" ) ; - -``` - -- example for writing collection meta data: - -```cpp -auto& hits = store.create("hits"); -// ... -auto& colMD = store.getCollectionMetaData( hits.getID() ); -colMD.setValue("CellIDEncodingString","system:8,barrel:3,layer:6,slice:5,x:-16,y:-16"); -``` - -- example for reading collection meta data - -```cpp -auto colMD = store.getCollectionMetaData( hits.getID() ); -std::string es = colMD.getStringVal("CellIDEncodingString") ; -``` - +Meta data can be stored and retrieved from the `Frame` via the templated `putParameter` and `getParameter` methods. #### Python Interface -The class `EventStore` provides all the necessary (read) access to event files. It can be used as follows: +The `Reader` and `Writer` classes in the `root_io` and `sio_io` submodules +provide all the necessary functionality to read and write event files. An +example of reading files looks like this: + ```python - from EventStore import EventStore - store = EventStore() - for event in store: + from podio.root_io import Reader + reader = Reader("one or many input files") + for event in reader.get("events"): hits = store.get("hits") for hit in hits: # ... diff --git a/doc/userdata.md b/doc/userdata.md index 544c79c75..b2df04568 100644 --- a/doc/userdata.md +++ b/doc/userdata.md @@ -6,17 +6,18 @@ data* via the `podio::UserDataCollection`. It gives the user access to a the data stored in the EDM classes for each event. ## Example usage -Creating or getting a `UserDataCollection` via the `EventStore` works the same -as with any other collection of the EDM via the `create` or `get` functions: +Creating or getting a `UserDataCollection` via the `Frame` works the same +as with any other collection of the EDM via the `put` or `get` functions: ```cpp #include "podio/UserDataCollection.h" -// Create a collection -auto& userFloats = store.create>("userFloats"); +// Create a collection and put it into a Frame +userFloats = podio::UserDataCollection(); +frame.put(std::move(userFloats), "userFloats"); // get a collection -const auto& userData = store.get>("userFloats"); +const auto& userData = frame.get>("userFloats"); ``` The interface of the `UserDataCollection` is similar to a basic version of the diff --git a/include/podio/ASCIIWriter.h b/include/podio/ASCIIWriter.h deleted file mode 100644 index e941bb6aa..000000000 --- a/include/podio/ASCIIWriter.h +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef PODIO_ASCIIWRITER_H -#define PODIO_ASCIIWRITER_H - -#include "podio/EventStore.h" -#include "podio/SchemaEvolution.h" -#include "podio/utilities/Deprecated.h" - -#include -#include -#include -#include -#include -#include - -namespace podio { - -class CollectionBase; - -struct ColWriterBase { - virtual void writeCollection(CollectionBase*, std::ostream&) = 0; - virtual ~ColWriterBase() = default; -}; - -template -struct ColWriter : public ColWriterBase { - void writeCollection(CollectionBase* c, std::ostream& o) override { - T* col = static_cast(c); - o << col->size() << std::endl; - o << *col << std::endl; - } -}; - -typedef std::map FunMap; - -class DEPR_EVTSTORE ASCIIWriter { - -public: - ASCIIWriter(const std::string& filename, EventStore* store); - ~ASCIIWriter(); - - // non-copyable - ASCIIWriter(const ASCIIWriter&) = delete; - ASCIIWriter& operator=(const ASCIIWriter&) = delete; - - template - bool registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - template - void writeCollection(const std::string& name); - // members - std::string m_filename; - EventStore* m_store; - - std::ofstream* m_file; - - std::vector m_storedCollections{}; - std::vector m_collectionNames{}; - FunMap m_map{}; -}; - -template -bool ASCIIWriter::registerForWrite(const std::string& name) { - const T* tmp_coll(nullptr); - if (!m_store->get(name, tmp_coll)) { - std::cerr << "no such collection to write, throw exception." << std::endl; - return false; - } - T* coll = const_cast(tmp_coll); - - m_storedCollections.emplace_back(coll); - m_collectionNames.emplace_back(name); - m_map[name] = new ColWriter; - return true; -} - -} // namespace podio -#endif diff --git a/include/podio/EventStore.h b/include/podio/EventStore.h deleted file mode 100644 index 05ae58b38..000000000 --- a/include/podio/EventStore.h +++ /dev/null @@ -1,160 +0,0 @@ -#ifndef PODIO_EVENTSTORE_H -#define PODIO_EVENTSTORE_H - -#include -#include -#include -#include -#include -#include - -// podio specific includes -#include "podio/CollectionIDTable.h" -#include "podio/GenericParameters.h" -#include "podio/ICollectionProvider.h" -#include "podio/IMetaDataProvider.h" -#include "podio/utilities/Deprecated.h" - -/** -This is an *example* event store - -The event store holds the object collections. - -It is used to create new collections, and to access existing ones. -When accessing a collection that is not yet in the event store, -the event store makes use of a Reader to read the collection. - -**/ - -namespace podio { - -class CollectionBase; -class IReader; - -typedef std::map RunMDMap; -typedef std::map ColMDMap; - -class DEPR_EVTSTORE EventStore : public ICollectionProvider, public IMetaDataProvider { -public: - /// Make non-copyable - EventStore(const EventStore&) = delete; - EventStore& operator=(const EventStore&) = delete; - - /// Collection entry. Each collection is identified by a name - typedef std::pair CollPair; - typedef std::vector CollContainer; - - EventStore(); - ~EventStore(); - - /// create a new collection - template - T& create(const std::string& name); - - /// register an existing collection - void registerCollection(const std::string& name, podio::CollectionBase* coll); - - /// access a collection by name. returns true if successful - template - bool get(const std::string& name, const T*& collection); - - /// access a collection by ID. returns true if successful - bool get(uint32_t id, CollectionBase*& coll) const final; - - /// access a collection by name - /// returns a collection w/ setting isValid to true if successful - template - const T& get(const std::string& name); - - /// empties collections. - void clearCollections(); - - /// clears itself; deletes collections (use at end of event processing) - void clear(); - - /// Clears only the cache containers (use at end of event if ownership of read objects is transferred) - void clearCaches(); - - /// set the reader - void setReader(IReader* reader); - - CollectionIDTable* getCollectionIDTable() const { - return m_table.get(); - } - - virtual bool isValid() const final; - - /// return the event meta data for the current event - GenericParameters& getEventMetaData() override; - - /// return the run meta data for the given runID - GenericParameters& getRunMetaData(int runID) override; - - /// return the collection meta data for the given colID - GenericParameters& getCollectionMetaData(uint32_t colID) override; - - RunMDMap* getRunMetaDataMap() { - return &m_runMDMap; - } - ColMDMap* getColMetaDataMap() { - return &m_colMDMap; - } - GenericParameters* eventMetaDataPtr() { - return &m_evtMD; - } - -private: - /// get the collection of given name; returns true if successfull - bool doGet(const std::string& name, CollectionBase*& collection, bool setReferences = true) const; - /// check if a collection of given name already exists - bool collectionRegistered(const std::string& name) const; - void setCollectionIDTable(std::shared_ptr table) { - m_table = std::move(table); - } - - // members - mutable std::set m_retrievedIDs{}; - mutable CollContainer m_collections{}; - IReader* m_reader{nullptr}; - std::shared_ptr m_table; - - GenericParameters m_evtMD{}; - RunMDMap m_runMDMap{}; - ColMDMap m_colMDMap{}; -}; - -template -T& EventStore::create(const std::string& name) { - static_assert(std::is_base_of::value, - "DataStore only accepts types inheriting from CollectionBase"); - // TODO: add check for existence - T* coll = new T(); - registerCollection(name, coll); - return *coll; -} - -template -bool EventStore::get(const std::string& name, const T*& collection) { - // static_assert(std::is_base_of::value, - // "DataStore only contains types inheriting from CollectionBase"); - CollectionBase* tmp{nullptr}; - doGet(name, tmp); - collection = static_cast(tmp); - if (collection != nullptr) { - return true; - } - return false; -} - -template -const T& EventStore::get(const std::string& name) { - const T* tmp(0); - auto success = this->get(name, tmp); - if (!success) { - throw std::runtime_error("No collection \'" + name + "\' is present in the EventStore"); - } - return *tmp; -} - -} // namespace podio -#endif diff --git a/include/podio/GenericParameters.h b/include/podio/GenericParameters.h index eeee3696f..8466de9cb 100644 --- a/include/podio/GenericParameters.h +++ b/include/podio/GenericParameters.h @@ -18,10 +18,12 @@ class write_device; using version_type = uint32_t; // from sio/definitions } // namespace sio +#if PODIO_ENABLE_RNTUPLE namespace podio { class ROOTNTupleReader; class ROOTNTupleWriter; } // namespace podio +#endif #define DEPR_NON_TEMPLATE \ [[deprecated("Non-templated access will be removed. Switch to templated access functionality")]] @@ -150,8 +152,11 @@ class GenericParameters { friend void writeGenericParameters(sio::write_device& device, const GenericParameters& parameters); friend void readGenericParameters(sio::read_device& device, GenericParameters& parameters, sio::version_type version); - friend ROOTNTupleReader; + +#if PODIO_ENABLE_RNTUPLE friend ROOTNTupleWriter; + friend ROOTNTupleReader; +#endif /// Get a reference to the internal map for a given type template diff --git a/include/podio/IMetaDataProvider.h b/include/podio/IMetaDataProvider.h deleted file mode 100644 index b8c662277..000000000 --- a/include/podio/IMetaDataProvider.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef PODIO_IMETADATAPROVIDER_H -#define PODIO_IMETADATAPROVIDER_H - -#include "podio/GenericParameters.h" -#include "podio/utilities/Deprecated.h" - -namespace podio { - -/** Inteface to access meta data for runs, events and collections. - * @author F. Gaede, DESY - * @date Apr 2020 - */ -class DEPR_EVTSTORE IMetaDataProvider { - -public: - /// destructor - virtual ~IMetaDataProvider() = default; - - /// return the event meta data for the current event - virtual GenericParameters& getEventMetaData() = 0; - - /// return the run meta data for the given runID - virtual GenericParameters& getRunMetaData(int runID) = 0; - - /// return the collection meta data for the given colID - virtual GenericParameters& getCollectionMetaData(uint32_t colID) = 0; -}; - -} // namespace podio - -#endif diff --git a/include/podio/IReader.h b/include/podio/IReader.h deleted file mode 100644 index 13553bf87..000000000 --- a/include/podio/IReader.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef PODIO_IREADER_H -#define PODIO_IREADER_H - -#include "podio/podioVersion.h" -#include "podio/utilities/Deprecated.h" - -#include -#include -#include -#include -#include - -/* - -This class has the function to read available data from disk -and to prepare collections and buffers. - -*/ - -namespace podio { - -class CollectionBase; -class CollectionIDTable; -class GenericParameters; - -class DEPR_EVTSTORE IReader { -public: - virtual ~IReader() = default; - /// Read Collection of given name - /// Does not set references yet. - virtual CollectionBase* readCollection(const std::string& name) = 0; - /// Get CollectionIDTable of read-in data - virtual std::shared_ptr getCollectionIDTable() = 0; - /// read event meta data from file - virtual GenericParameters* readEventMetaData() = 0; - virtual std::map* readCollectionMetaData() = 0; - virtual std::map* readRunMetaData() = 0; - /// get the number of events available from this reader - virtual unsigned getEntries() const = 0; - /// Prepare the reader to read the next event - virtual void endOfEvent() = 0; - // TODO: decide on smart-pointers for passing of objects - /// Check if reader is valid - virtual bool isValid() const = 0; - - virtual void openFile(const std::string& filename) = 0; - virtual void closeFile() = 0; - - virtual void readEvent() = 0; - virtual void goToEvent(unsigned iEvent) = 0; - - /// Get the podio version with which the current file has been written - virtual podio::version::Version currentFileVersion() const = 0; -}; - -} // namespace podio - -#endif diff --git a/include/podio/PythonEventStore.h b/include/podio/PythonEventStore.h deleted file mode 100644 index 8ae4d5140..000000000 --- a/include/podio/PythonEventStore.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef PODIO_PYTHONEVENTSTORE_H -#define PODIO_PYTHONEVENTSTORE_H - -#include "podio/EventStore.h" -#include "podio/GenericParameters.h" -#include "podio/IReader.h" -#include "podio/utilities/Deprecated.h" - -#include - -namespace podio { - -class DEPR_EVTSTORE PythonEventStore { -public: - /// constructor from filename - PythonEventStore(const char* filename); - - /// access a collection. - const podio::CollectionBase* get(const char* name); - - /// signify end of event - void endOfEvent(); - - /// go to a given event - void goToEvent(unsigned ievent); - - /// get number of entries in the tree - unsigned getEntries() const; - - /// is the input file accessible? - bool isZombie() const { - return m_isZombie; - } - - bool isValid() const { - return m_reader && m_reader->isValid(); - } - void close() { - m_reader->closeFile(); - } - - /// list available collections - const std::vector& getCollectionNames() const; - - const podio::GenericParameters& getEventMetaData() { - return m_store.getEventMetaData(); - } - -private: - std::unique_ptr m_reader; - podio::EventStore m_store; - - /// set to true if input root file accessible, false otherwise - bool m_isZombie{true}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index c152c3772..b7a8d1c46 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -32,7 +32,6 @@ namespace detail { } // namespace detail -class EventStore; class CollectionBase; class CollectionIDTable; class GenericParameters; diff --git a/include/podio/ROOTLegacyReader.h b/include/podio/ROOTLegacyReader.h index 2a2a8b621..71894debd 100644 --- a/include/podio/ROOTLegacyReader.h +++ b/include/podio/ROOTLegacyReader.h @@ -29,7 +29,6 @@ namespace detail { using CollectionInfo = std::tuple; } // namespace detail -class EventStore; class CollectionBase; class CollectionIDTable; class GenericParameters; diff --git a/include/podio/ROOTReader.h b/include/podio/ROOTReader.h deleted file mode 100644 index 757791cf0..000000000 --- a/include/podio/ROOTReader.h +++ /dev/null @@ -1,124 +0,0 @@ -#ifndef PODIO_ROOTREADER_H -#define PODIO_ROOTREADER_H - -#include "podio/CollectionBranches.h" -#include "podio/IReader.h" - -#include -#include -#include -#include -#include -#include -#include - -// forward declarations -class TClass; -class TFile; -class TTree; -class TChain; - -namespace podio { - -class EventStore; -class CollectionBase; -class Registry; -class CollectionIDTable; -class GenericParameters; - -/** -This class has the function to read available data from disk -and to prepare collections and buffers. -**/ -class ROOTReader : public IReader { - friend EventStore; - -public: - ROOTReader() = default; - // todo: see https://github.com/AIDASoft/podio/issues/290 - ~ROOTReader(); // NOLINT(modernize-use-equals-default) - - // non-copyable - ROOTReader(const ROOTReader&) = delete; - ROOTReader& operator=(const ROOTReader&) = delete; - - void openFile(const std::string& filename) override; - void openFiles(const std::vector& filenames); - void closeFile() override; - void closeFiles(); - - /// Read all collections requested - void readEvent() override; - - /// Read CollectionIDTable from ROOT file - std::shared_ptr getCollectionIDTable() override { - return m_table; - } - - /// Returns number of entries in the TTree - unsigned getEntries() const override; - - /// Preparing to read next event - void endOfEvent() override; - - /// Preparing to read a given event - void goToEvent(unsigned evnum) override; - - podio::version::Version currentFileVersion() const override { - return m_fileVersion; - } - - /// Check if TFile is valid - bool isValid() const override; - -private: - /// Implementation for collection reading - CollectionBase* readCollection(const std::string& name) override; - - /// read event meta data for current event - GenericParameters* readEventMetaData() override; - - /// read the collection meta data - std::map* readCollectionMetaData() override; - - /// read the run meta data - std::map* readRunMetaData() override; - -private: - void createCollectionBranches(const std::vector>& collInfo); - - std::pair getLocalTreeAndEntry(const std::string& treename); - // Information about the data vector as wall as the collection class type - // and the index in the collection branches cache vector - using CollectionInfo = std::tuple; - - CollectionBase* getCollection(const std::pair& collInfo); - CollectionBase* readCollectionData(const root_utils::CollectionBranches& branches, CollectionBase* collection, - Long64_t entry, const std::string& name); - - // cache collections that have been read already in a given event - typedef std::pair Input; - std::vector m_inputs{}; - - // cache the necessary information to more quickly construct and read each - // collection after it has been read the very first time - std::map m_storedClasses{}; - - std::shared_ptr m_table{nullptr}; - TChain* m_chain{nullptr}; - unsigned m_eventNumber{0}; - - // Similar to writing we cache the branches that belong to each collection - // in order to not having to look them up every event. However, for the - // reader we cannot guarantee a fixed order of collections as they are read - // on demand. Hence, we give each collection an index the first time it is - // read and we start caching the branches. - size_t m_collectionIndex = 0; - std::vector m_collectionBranches{}; - - podio::version::Version m_fileVersion{0, 0, 0}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/ROOTWriter.h b/include/podio/ROOTWriter.h deleted file mode 100644 index 6751809ae..000000000 --- a/include/podio/ROOTWriter.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef PODIO_ROOTWRITER_H -#define PODIO_ROOTWRITER_H - -#include "podio/CollectionBase.h" -#include "podio/CollectionBranches.h" -#include "podio/EventStore.h" -#include "podio/utilities/Deprecated.h" - -#include "TBranch.h" - -#include -#include -#include -#include -#include - -// forward declarations -class TFile; -class TTree; - -namespace podio { -class DEPR_EVTSTORE ROOTWriter { - -public: - ROOTWriter(const std::string& filename, EventStore* store); - ~ROOTWriter(); - - // non-copyable - ROOTWriter(const ROOTWriter&) = delete; - ROOTWriter& operator=(const ROOTWriter&) = delete; - - bool registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - using StoreCollection = std::pair; - void createBranches(const std::vector& collections); - void setBranches(const std::vector& collections); - - // members - std::string m_filename; - EventStore* m_store; - TFile* m_file; - TTree* m_datatree; - TTree* m_metadatatree; - TTree* m_runMDtree; - TTree* m_evtMDtree; - TTree* m_colMDtree; - std::vector m_collectionsToWrite{}; - // In order to avoid having to look up the branches from the datatree for - // every event, we cache them in this vector, that is populated the first - // time we write an event. Since the collections and their order do not - // change between events, the assocation between the collections to write - // and their branches is simply index based - std::vector m_collectionBranches{}; - - bool m_firstEvent{true}; -}; - -} // namespace podio -#endif diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index aa5e58c71..93b0c0e71 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -102,8 +101,6 @@ class SIOCollectionIDTableBlock : public sio::block { SIOCollectionIDTableBlock() : sio::block("CollectionIDs", sio::version::encode_version(0, 4)) { } - SIOCollectionIDTableBlock(podio::EventStore* store); - SIOCollectionIDTableBlock(std::vector&& names, std::vector&& ids, std::vector&& types, std::vector&& isSubsetColl) : sio::block("CollectionIDs", sio::version::encode_version(0, 3)), diff --git a/include/podio/SIOReader.h b/include/podio/SIOReader.h deleted file mode 100644 index 284dc13df..000000000 --- a/include/podio/SIOReader.h +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef PODIO_SIOREADER_H -#define PODIO_SIOREADER_H - -#include -#include -#include -#include -#include - -#include "podio/EventStore.h" -#include "podio/ICollectionProvider.h" -#include "podio/IReader.h" -#include "podio/SIOBlock.h" - -// -- sio headers -#include -#include -#include -#include - -namespace podio { - -class CollectionBase; -class CollectionIDTable; - -/** - This class has the function to read available data from disk - and to prepare collections and buffers. -**/ -class SIOReader : public IReader { - friend EventStore; - -public: - SIOReader(); - ~SIOReader() = default; - - // make non-copyable - SIOReader(const SIOReader&) = delete; - SIOReader& operator=(const SIOReader&) = delete; - - void openFile(const std::string& filename) override; - void closeFile() override; - - /// Read all collections requested - void readEvent() override; - - void goToEvent(unsigned iEvent) override; - - /// Read CollectionIDTable from SIO file - std::shared_ptr getCollectionIDTable() override { - return m_table; - } - - unsigned getEntries() const override { - return m_tocRecord.getNRecords("event_record"); - } - - /// Check if file is valid - bool isValid() const override; - - podio::version::Version currentFileVersion() const override { - return m_fileVersion; - } - - void endOfEvent() override; - -private: - /// Implementation for collection reading - CollectionBase* readCollection(const std::string& name) override; - - /// read event meta data for current event - GenericParameters* readEventMetaData() override; - - /// read the collection meta data - std::map* readCollectionMetaData() override; - - /// read the run meta data - std::map* readRunMetaData() override; - - /// read the TOC record - bool readFileTOCRecord(); - - /// reconstruct the TOC record from the file contents - void reconstructFileTOCRecord(); - -private: - void readCollectionIDTable(); - void readMetaDataRecord(const std::shared_ptr& mdBlock); - void createBlocks(); - - typedef std::pair Input; - std::vector m_inputs{}; - std::shared_ptr m_table{nullptr}; // Co-owned by the EventStore - int m_eventNumber{0}; - int m_lastEventRead{-1}; - std::vector m_typeNames{}; - std::vector m_subsetCollectionBits{}; - - std::shared_ptr m_eventMetaData{}; - std::shared_ptr m_runMetaData{}; - std::shared_ptr m_collectionMetaData{}; - - sio::ifstream m_stream{}; - sio::record_info m_rec_info{}; - sio::buffer m_info_buffer{sio::max_record_info_len}; - sio::buffer m_rec_buffer{sio::mbyte}; - sio::buffer m_unc_buffer{sio::mbyte}; - sio::block_list m_blocks{}; - - SIOFileTOCRecord m_tocRecord{}; - - podio::version::Version m_fileVersion{0}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/SIOWriter.h b/include/podio/SIOWriter.h deleted file mode 100644 index 7594ae74a..000000000 --- a/include/podio/SIOWriter.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef PODIO_SIOWRITER_H -#define PODIO_SIOWRITER_H - -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" -#include "podio/utilities/Deprecated.h" - -// SIO specific includes -#include - -#include -#include -#include - -// forward declarations - -namespace podio { -class DEPR_EVTSTORE SIOWriter { - -public: - SIOWriter(const std::string& filename, EventStore* store); - ~SIOWriter(); - - // non-copyable - SIOWriter(const SIOWriter&) = delete; - SIOWriter& operator=(const SIOWriter&) = delete; - - void registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - void writeCollectionIDTable(); - sio::block_list createBlocks() const; - - // members - std::string m_filename{}; - EventStore* m_store = nullptr; - - std::shared_ptr m_eventMetaData; - sio::ofstream m_stream{}; - bool m_firstEvent{true}; - - std::shared_ptr m_runMetaData; - std::shared_ptr m_collectionMetaData; - SIOFileTOCRecord m_tocRecord{}; - std::vector m_collectionsToWrite{}; - - bool m_finished{false}; -}; - -} // namespace podio -#endif diff --git a/include/podio/TimedReader.h b/include/podio/TimedReader.h deleted file mode 100644 index 288258da3..000000000 --- a/include/podio/TimedReader.h +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef PODIO_TIMEDREADER_H -#define PODIO_TIMEDREADER_H - -#include "podio/BenchmarkRecorder.h" -#include "podio/BenchmarkUtil.h" - -#include "podio/GenericParameters.h" -#include "podio/IReader.h" - -#include - -namespace podio { - -template -class TimedReader : public IReader { - using ClockT = benchmark::ClockT; - -public: - template - TimedReader(benchmark::BenchmarkRecorder& recorder, Args&&... args) : - m_start(ClockT::now()), - m_reader(WrappedReader(std::forward(args)...)), - m_end(ClockT::now()), - m_recorder(recorder), - m_perEventTree(m_recorder.addTree( - "event_times", - {"read_collections", "read_ev_md", "read_run_md", "read_coll_md", "end_of_event", "read_event"})) { - m_recorder.addTree("setup_times", {"constructor", "open_file", "close_file", "read_collection_ids", "get_entries"}); - m_recorder.recordTime("setup_times", "constructor", m_end - m_start); - } - - ~TimedReader() { - // Timing deconstructors is not straight forward when wrapping a value. - // Since nothing is usually happening in them in any case, we simply don't - // do it. We still have to fill the setup_times tree here though. - m_recorder.Fill("setup_times"); - } - - /// Read Collection of given name - /// Does not set references yet. - CollectionBase* readCollection(const std::string& name) override { - const auto [result, duration] = benchmark::run_member_timed(m_reader, &IReader::readCollection, name); - // since we cannot in general know how many collections there will be read - // we simply sum up all the requests in an event and record that - m_totalCollectionReadTime += duration; - return result; - } - - /// Get CollectionIDTable of read-in data - std::shared_ptr getCollectionIDTable() override { - return runTimed(false, "read_collection_ids", &IReader::getCollectionIDTable); - } - - /// read event meta data from file - GenericParameters* readEventMetaData() override { - return runTimed(true, "read_ev_md", &IReader::readEventMetaData); - } - - std::map* readCollectionMetaData() override { - return runTimed(true, "read_coll_md", &IReader::readCollectionMetaData); - } - - std::map* readRunMetaData() override { - return runTimed(true, "read_run_md", &IReader::readRunMetaData); - } - - /// get the number of events available from this reader - unsigned getEntries() const override { - return runTimed(false, "get_entries", &IReader::getEntries); - } - - /// Prepare the reader to read the next event - void endOfEvent() override { - runVoidTimed(true, "end_of_event", &IReader::endOfEvent); - - m_perEventTree.recordTime("read_collections", m_totalCollectionReadTime); - m_perEventTree.Fill(); - m_totalCollectionReadTime = std::chrono::nanoseconds{0}; - } - - // not benchmarking this one - bool isValid() const override { - return m_reader.isValid(); - } - - void openFile(const std::string& filename) override { - runVoidTimed(false, "open_file", &IReader::openFile, filename); - } - - void closeFile() override { - runVoidTimed(false, "close_file", &IReader::closeFile); - } - - void readEvent() override { - runVoidTimed(true, "read_event", &IReader::readEvent); - } - - void goToEvent(unsigned ev) override { - // TODO: Do we need to time this? Not really used at the moment - m_reader.goToEvent(ev); - } - - podio::version::Version currentFileVersion() const override { - // no need to time this as it is really just a very simple get - return m_reader.currentFileVersion(); - } - -private: - void recordTime(bool perEvent, const std::string& step, ClockT::duration duration) const { - if (perEvent) { - m_perEventTree.recordTime(step, duration); - } else { - m_recorder.recordTime("setup_times", step, duration); - } - } - - template - inline std::invoke_result_t runTimed(bool perEvent, const std::string& step, - FuncT func, Args&&... args) { - const auto [result, duration] = benchmark::run_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - - return result; - } - - template - inline std::invoke_result_t runTimed(bool perEvent, const std::string& step, - FuncT func, Args&&... args) const { - const auto [result, duration] = benchmark::run_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - - return result; - } - - template - inline void runVoidTimed(bool perEvent, const std::string& step, FuncT func, Args&&... args) { - const auto duration = benchmark::run_void_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - } - - // NOTE: c++ initializes its class members in the order they are defined not - // in the order in which they appear in the initializer list! - ClockT::time_point m_start; // to time the construction - WrappedReader m_reader; // the decorated reader that does the actual work - ClockT::time_point m_end; // to time the constructor call - - benchmark::BenchmarkRecorder& m_recorder; - // Keep a reference to this one around, to save the look-up in each event - benchmark::BenchmarkRecorderTree& m_perEventTree; - ClockT::duration m_totalCollectionReadTime{std::chrono::nanoseconds{0}}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/TimedWriter.h b/include/podio/TimedWriter.h deleted file mode 100644 index 90e154273..000000000 --- a/include/podio/TimedWriter.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef PODIO_TIMEDWRITER_H -#define PODIO_TIMEDWRITER_H - -#include "podio/BenchmarkRecorder.h" -#include "podio/BenchmarkUtil.h" -#include "podio/utilities/Deprecated.h" - -#include -#include - -namespace podio { - -template -class DEPR_EVTSTORE TimedWriter { - using ClockT = benchmark::ClockT; - -public: - template - TimedWriter(benchmark::BenchmarkRecorder& recorder, Args&&... args) : - m_start(ClockT::now()), - m_writer(WrappedWriter(std::forward(args)...)), - m_end(ClockT::now()), - m_recorder(recorder), - m_perEventTree(m_recorder.addTree("event_times", {"write_event"})) { - m_recorder.addTree("setup_times", {"constructor", "finish", "register_for_write"}); - m_recorder.recordTime("setup_times", "constructor", m_end - m_start); - } - - ~TimedWriter() { - m_recorder.recordTime("setup_times", "register_for_write", m_registerTime); - m_recorder.Fill("setup_times"); - } - - void registerForWrite(const std::string& name) { - // summing up the times it takes for all the collections to be registered - // here, since we do not know in advance how many collections there will be - // in the end - const auto duration = benchmark::run_void_member_timed(m_writer, &WrappedWriter::registerForWrite, name); - m_registerTime += duration; - } - - void writeEvent() { - m_perEventTree.recordTime("write_event", benchmark::run_void_member_timed(m_writer, &WrappedWriter::writeEvent)); - m_perEventTree.Fill(); - } - - void finish() { - m_recorder.recordTime("setup_times", "finish", benchmark::run_void_member_timed(m_writer, &WrappedWriter::finish)); - } - -private: - ClockT::time_point m_start; - WrappedWriter m_writer; - ClockT::time_point m_end; - benchmark::BenchmarkRecorder& m_recorder; - benchmark::BenchmarkRecorderTree& m_perEventTree; - ClockT::duration m_registerTime{std::chrono::nanoseconds{0}}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/utilities/Deprecated.h b/include/podio/utilities/Deprecated.h deleted file mode 100644 index 566bca814..000000000 --- a/include/podio/utilities/Deprecated.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef PODIO_UTILITIES_DEPRECATED_H -#define PODIO_UTILITIES_DEPRECATED_H - -#define DEPR_EVTSTORE \ - [[deprecated("The EventStore based I/O model is deprecated and will be removed. Switch to the Frame based model.")]] - -#endif // PODIO_UTILITIES_DEPRECATED_H diff --git a/include/podio/utilities/IOHelpers.h b/include/podio/utilities/IOHelpers.h deleted file mode 100644 index b11ed76bc..000000000 --- a/include/podio/utilities/IOHelpers.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef PODIO_UTILITIES_IOHELPERS_H -#define PODIO_UTILITIES_IOHELPERS_H - -#ifndef PODIO_ENABLE_SIO - #define PODIO_ENABLE_SIO 0 -#endif - -#include "podio/IReader.h" - -#include -#include - -namespace podio { -std::unique_ptr createReader(const std::string& filename); -} - -#endif diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 273ecd4a4..99f16dc8b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,7 +4,7 @@ SET(podio_PYTHON_INSTALLDIR ${podio_PYTHON_INSTALLDIR} PARENT_SCOPE) set(to_install podio_class_generator.py podio_schema_evolution.py - EventStore.py) + ) install(FILES ${to_install} DESTINATION ${podio_PYTHON_INSTALLDIR}) diff --git a/python/EventStore.py b/python/EventStore.py deleted file mode 100644 index 5607f2da0..000000000 --- a/python/EventStore.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Legacy import wrapper for EventStore.""" - -import warnings -warnings.warn("You are using the legacy EventStore import. Switch to 'from podio import EventStore'", FutureWarning) - -from podio import EventStore # noqa: F401 # pylint: disable=wrong-import-position, unused-import diff --git a/python/podio/EventStore.py b/python/podio/EventStore.py deleted file mode 100644 index 068027aa4..000000000 --- a/python/podio/EventStore.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Python EventStore for reading files with podio generated datamodels""" - -import warnings -warnings.warn("The EventStore based I/O model is deprecated and will be removed. Switch to the Frame based model.", - FutureWarning) - -from ROOT import gSystem # pylint: disable=wrong-import-position -gSystem.Load("libpodioPythonStore") # noqa: E402 -from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position - - -def size(self): - """Override size function that can be attached as __len__ method to - collections""" - return self.size() - - -def getitem(self, key): - """Override getitem function that can be attached as __getitem__ method to - collections (see below why this is necessary sometimes)""" - return self.at(key) - - -class EventStore: - '''Interface to events in an podio root file. - Example of use: - events = EventStore(["example.root", "example1.root"]) - for iev, store in islice(enumerate(events), 0, 2): - particles = store.get("GenParticle"); - for i, p in islice(enumerate(particles), 0, 5): - print "particle ", i, p.ID(), p.P4().Pt - ''' - - def __init__(self, filenames): - '''Create an event list from the podio root file. - Parameters: - filenames: list of root files - you can of course provide a list containing a single - root file. you could use the glob module to get all - files matching a wildcard pattern. - ''' - if isinstance(filenames, str): - filenames = (filenames,) - self.files = filenames - self.stores = [] - self.current_store = None - for fname in self.files: - store = podio.PythonEventStore(fname) - if store.isZombie(): - raise ValueError(fname + ' does not exist.') - store.name = fname - if self.current_store is None: - self.current_store = store - self.stores.append((store.getEntries(), store)) - - def __str__(self): - result = "Content:" - result += "\n\t".join(n for n in self.current_store.getCollectionNames()) - return result - - def get(self, name): - '''Returns a collection. - Parameters: - name: name of the collection in the podio root file. - ''' - coll = self.current_store.get(name) - # adding length function - coll.__len__ = size - # enabling the use of [] notation on the collection - # cppyy defines the __getitem__ method if the underlying c++ class has an operator[] - # method. For some reason they do not conform to the usual signature and only - # pass one argument to the function they call. Here we simply check if we have to - # define the __getitem__ for the collection. - if not hasattr(coll, '__getitem__'): - coll.__getitem__ = getitem - return coll - - def collections(self): - """Return list of all collection names.""" - return [str(c) for c in self.current_store.getCollectionNames()] - - def metadata(self): - """Get the metadata of the current event as GenericParameters""" - return self.current_store.getEventMetaData() - - def isValid(self): - """Check if the EventStore is in a valid state""" - return self.current_store is not None and self.current_store.isValid() - - # def __getattr__(self, name): - # '''missing attributes are taken from self.current_store''' - # if name != 'current_store': - # return getattr(self.current_store, name) - # else: - # return None - - def current_filename(self): - '''Returns the name of the current file.''' - if self.current_store is None: - return None - return self.current_store.fname - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_val, trace): - for store in self.stores: - store[1].close() - - def __iter__(self): - '''iterate on events in the tree. - ''' - for _, store in self.stores: - self.current_store = store - for _ in range(store.getEntries()): - yield store - store.endOfEvent() - - def __getitem__(self, evnum): - '''Get event number evnum''' - current_store = None - rel_evnum = evnum - for nev, store in self.stores: - if rel_evnum < nev: - current_store = store - break - rel_evnum -= nev - if current_store is None: - raise ValueError('event number too large: ' + str(evnum)) - self.current_store = current_store - self.current_store.goToEvent(rel_evnum) - return self - - def __len__(self): - '''Returns the total number of events in all files.''' - nevts_all_files = 0 - for nev, _ in self.stores: - nevts_all_files += nev - return nevts_all_files diff --git a/python/podio/__init__.py b/python/podio/__init__.py index fd5fa7209..4c68b7a58 100644 --- a/python/podio/__init__.py +++ b/python/podio/__init__.py @@ -6,7 +6,7 @@ try: from ROOT import podio # noqa: F401 except ImportError: - print('Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH') + print("Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH") raise from .frame import Frame @@ -20,14 +20,10 @@ except ImportError: pass -from . import EventStore - - __all__ = [ "__version__", "Frame", "root_io", "sio_io", "reading", - "EventStore" ] diff --git a/python/podio/test_EventStore.py b/python/podio/test_EventStore.py deleted file mode 100644 index a8e4fb965..000000000 --- a/python/podio/test_EventStore.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Unit tests for the EventStore class""" - -from podio.EventStore import EventStore - - -class EventStoreBaseTestCaseMixin: - """EventStore unit tests - - These define some tests that should work regardless of the backend that is - used. In order to not have to duplicate this functionality for each backend, - this base class defines the common tests and inheriting classes define a - corresponding setUp method that sets up the correct EventStore and potentially - additional backend specific functionality - """ - def test_eventloop(self): - self.assertTrue(len(self.store) >= 0) - self.assertEqual(self.store.current_store.getEntries(), - len(self.store)) - for iev, event in enumerate(self.store): - self.assertTrue(event is not None) - if iev > 5: - break - - def test_navigation(self): - event0 = self.store[0] - self.assertEqual(event0.__class__, self.store.__class__) - - def test_collections(self): - evinfo = self.store.get("info") - self.assertTrue(len(evinfo) > 0) - particles = self.store.get("CollectionNotThere") - self.assertFalse(particles) - - def test_read_only(self): - hits = self.store.get("hits") - # testing that one can't modify attributes in - # read-only pods - self.assertEqual(hits[0].energy(), 23.) - hits[0].energy(10) - self.assertEqual(hits[0].energy(), 10) # oops - - def test_one_to_many(self): - clusters = self.store.get("clusters") - ref_hits = [] - # testing that cluster hits can be accessed and make sense - for cluster in clusters: - sume = 0 - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - ref_hits.append(hit) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - hits = self.store.get("hits") - # testing that the hits stored as a one to many relation - # in the cluster can be found in the hit collection - for hit in ref_hits: - self.assertTrue(hit in hits) - - def test_relation_range(self): - """Test that the RelationRange functionality is also accessible in python""" - clusters = self.store.get("clusters") - hits = self.store.get("hits") - - for cluster in clusters: - sume = 0 - for hit in cluster.Hits(): - self.assertTrue(hit in hits) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - - def test_hash(self): - clusters = self.store.get("clusters") - ref_hits = [] - # testing that cluster hits can be accessed and make sense - for cluster in clusters: - sume = 0 - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - ref_hits.append(hit) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - hits = self.store.get("hits") - if hits[0] == ref_hits[0]: - self.assertEqual(hits[0].getObjectID().index, - ref_hits[0].getObjectID().index) - self.assertEqual(hits[0].getObjectID().collectionID, - ref_hits[0].getObjectID().collectionID) - self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID()) - # testing that the hits stored as a one to many relation - # import pdb; pdb.set_trace() - - def test_context_managers(self): - with EventStore([self.filename]) as store: - self.assertTrue(len(store) >= 0) - self.assertTrue(store.isValid()) diff --git a/python/podio/test_EventStoreRoot.py b/python/podio/test_EventStoreRoot.py deleted file mode 100644 index 522e389d7..000000000 --- a/python/podio/test_EventStoreRoot.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Python unit tests for the ROOT backend""" - -import unittest -import os - -from ROOT import TFile - -from test_EventStore import EventStoreBaseTestCaseMixin # pylint: disable=import-error - -from podio.EventStore import EventStore - - -class EventStoreRootTestCase(EventStoreBaseTestCaseMixin, unittest.TestCase): - """Test cases for root input files""" - def setUp(self): - """Setup an EventStore reading from a ROOT file""" - self.filename = 'root_io/example.root' - self.assertTrue(os.path.isfile(self.filename)) - self.store = EventStore([self.filename]) - - def test_chain(self): - self.store = EventStore([self.filename, - self.filename]) - rootfile = TFile(self.filename) - events = rootfile.Get(str('events')) - numbers = [] - for iev, _ in enumerate(self.store): - evinfo = self.store.get("info") - numbers.append(evinfo[0].Number()) - self.assertEqual(iev + 1, 2 * events.GetEntries()) # pylint: disable=undefined-loop-variable - # testing that numbers is [0, .. 1999, 0, .. 1999] - self.assertEqual(numbers, list(range(events.GetEntries())) * 2) - # trying to go to an event beyond the last one - self.assertRaises(ValueError, self.store.__getitem__, 4001) - # this is in the first event in the second file, - # so its event number should be 0. - self.assertEqual(self.store[2000].get("info")[0].Number(), 0) - - def test_no_file(self): - '''Test that non-accessible files are gracefully handled.''' - with self.assertRaises(ValueError): - self.store = EventStore('foo.root') - - -if __name__ == '__main__': - # NOTE: These tests are really not intended to be run directly as they depend - # on quite some environment setup as well as externally produced inputs. - # See the CMakeLists.txt file in the tests folder for the specifics of that - # environment and the inputs - unittest.main() diff --git a/python/podio/test_EventStoreSio.py b/python/podio/test_EventStoreSio.py deleted file mode 100644 index c8d19b852..000000000 --- a/python/podio/test_EventStoreSio.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Python unit tests for the SIO backend""" - -import unittest -import os - -from test_utils import SKIP_SIO_TESTS # pylint: disable=import-error -from test_EventStore import EventStoreBaseTestCaseMixin # pylint: disable=import-error - -from podio.EventStore import EventStore - - -@unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") -class EventStoreSioTestCase(EventStoreBaseTestCaseMixin, unittest.TestCase): - """Test cases for root input files""" - def setUp(self): - """setup an EventStore reading an SIO file""" - self.filename = 'sio_io/example.sio' - self.assertTrue(os.path.isfile(self.filename)) - self.store = EventStore([self.filename]) - - def test_no_file(self): - '''Test that non-accessible files are gracefully handled.''' - with self.assertRaises(ValueError): - self.store = EventStore('foo.sio') - - -if __name__ == '__main__': - # NOTE: These tests are really not intended to be run directly as they depend - # on quite some environment setup as well as externally produced inputs. - # See the CMakeLists.txt file in the tests folder for the specifics of that - # environment and the inputs - unittest.main() diff --git a/python/podio/test_ReaderRoot.py b/python/podio/test_ReaderRoot.py index a7bdf98f8..eeeb2256f 100644 --- a/python/podio/test_ReaderRoot.py +++ b/python/podio/test_ReaderRoot.py @@ -4,6 +4,7 @@ import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error +from podio.test_utils import get_legacy_input from podio.root_io import Reader, LegacyReader @@ -19,4 +20,4 @@ class RootLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): """Setup a reader, reading from the example files""" - self.reader = LegacyReader('root_io/example.root') + self.reader = LegacyReader(get_legacy_input("v00-16-06-example.root")) diff --git a/python/podio/test_ReaderSio.py b/python/podio/test_ReaderSio.py index ef7b86b6b..5db31a0da 100644 --- a/python/podio/test_ReaderSio.py +++ b/python/podio/test_ReaderSio.py @@ -4,7 +4,7 @@ import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error -from test_utils import SKIP_SIO_TESTS # pylint: disable=import-error +from podio.test_utils import SKIP_SIO_TESTS, get_legacy_input @unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") @@ -22,4 +22,4 @@ class SIOLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): def setUp(self): """Setup a reader, reading from the example files""" from podio.sio_io import LegacyReader # pylint: disable=import-outside-toplevel - self.reader = LegacyReader('sio_io/example.sio') + self.reader = LegacyReader(get_legacy_input("v00-16-06-example.sio")) diff --git a/python/podio/test_utils.py b/python/podio/test_utils.py index 948571293..c69de90a6 100644 --- a/python/podio/test_utils.py +++ b/python/podio/test_utils.py @@ -4,3 +4,18 @@ import os SKIP_SIO_TESTS = os.environ.get("SKIP_SIO_TESTS", "1") == "1" + + +def get_legacy_input(filename): + """Try to get a legacy input file by name from the ExternalData that is + fetched by CMake. + + Returns either the absolute path to the actual file or an empty string. + """ + try: + datafile = os.path.join(os.environ["PODIO_BUILD_BASE"], "tests", "input_files", filename) + if os.path.isfile(datafile): + return os.path.abspath(datafile) + except KeyError: + pass + return "" diff --git a/src/ASCIIWriter.cc b/src/ASCIIWriter.cc deleted file mode 100644 index f47eec8d6..000000000 --- a/src/ASCIIWriter.cc +++ /dev/null @@ -1,40 +0,0 @@ -// podio specific includes -#include "podio/ASCIIWriter.h" -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" - -namespace podio { - -ASCIIWriter::ASCIIWriter(const std::string& filename, EventStore* store) : - m_filename(filename), m_store(store), m_file(new std::ofstream) { - - m_file->open(filename, std::ofstream::binary); -} - -ASCIIWriter::~ASCIIWriter() { - delete m_file; -} - -void ASCIIWriter::writeEvent() { - - unsigned i = 0; - for (auto& coll : m_storedCollections) { - coll->prepareForWrite(); - - const std::string& name = m_collectionNames[i++]; - std::cout << " writing collection " << name << std::endl; - - *m_file << name << " "; - - ColWriterBase* wrt = m_map[name]; - - wrt->writeCollection(coll, *m_file); - } -} - -void ASCIIWriter::finish() { - - m_file->close(); -} - -} // namespace podio diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a68b94bfb..b284abbd8 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,8 +50,6 @@ ENDFUNCTION() SET(core_sources CollectionIDTable.cc GenericParameters.cc - ASCIIWriter.cc - EventStore.cc DatamodelRegistry.cc DatamodelRegistryIOHelpers.cc UserDataCollection.cc @@ -63,14 +61,13 @@ SET(core_sources SET(core_headers ${PROJECT_SOURCE_DIR}/include/podio/CollectionBase.h ${PROJECT_SOURCE_DIR}/include/podio/CollectionIDTable.h - ${PROJECT_SOURCE_DIR}/include/podio/EventStore.h ${PROJECT_SOURCE_DIR}/include/podio/ICollectionProvider.h - ${PROJECT_SOURCE_DIR}/include/podio/IReader.h ${PROJECT_SOURCE_DIR}/include/podio/ObjectID.h ${PROJECT_SOURCE_DIR}/include/podio/UserDataCollection.h ${PROJECT_SOURCE_DIR}/include/podio/podioVersion.h ${PROJECT_SOURCE_DIR}/include/podio/DatamodelRegistry.h ${PROJECT_SOURCE_DIR}/include/podio/utilities/DatamodelRegistryIOHelpers.h + ${PROJECT_SOURCE_DIR}/include/podio/GenericParameters.h ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) @@ -80,8 +77,6 @@ target_compile_options(podio PRIVATE -pthread) # --- Root I/O functionality and corresponding dictionary SET(root_sources rootUtils.h - ROOTWriter.cc - ROOTReader.cc ROOTFrameWriter.cc ROOTFrameReader.cc ROOTLegacyReader.cc @@ -115,24 +110,9 @@ else() endif() -# --- Python EventStore for enabling (legacy) python bindings -SET(python_sources - IOHelpers.cc - PythonEventStore.cc - ) - -SET(python_headers - ${PROJECT_SOURCE_DIR}/include/podio/PythonEventStore.h -) -PODIO_ADD_LIB_AND_DICT(podioPythonStore "${python_headers}" "${python_sources}" python_selection.xml) -target_link_libraries(podioPythonStore PUBLIC podio::podio) -target_link_libraries(podioPythonStore PRIVATE podio::podioRootIO) - # --- SIO I/O functionality and corresponding dictionary if(ENABLE_SIO) SET(sio_sources - SIOReader.cc - SIOWriter.cc SIOBlockUserData.cc SIOBlock.cc SIOFrameWriter.cc @@ -152,14 +132,11 @@ if(ENABLE_SIO) target_link_libraries(podioSioIO PUBLIC podio::podio SIO::sio ${CMAKE_DL_LIBS} ${PODIO_FS_LIBS}) target_compile_definitions(podioSioIO PUBLIC PODIO_ENABLE_SIO=1) - # Make sure the legacy python bindings know about the SIO backend - target_link_libraries(podioPythonStore PRIVATE podioSioIO) - LIST(APPEND INSTALL_LIBRARIES podioSioIO podioSioIODict) endif() # --- Install everything -install(TARGETS podio podioDict podioPythonStore podioPythonStoreDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} +install(TARGETS podio podioDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} EXPORT podioTargets DESTINATION "${CMAKE_INSTALL_LIBDIR}") @@ -176,8 +153,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libpodioDict_rdict.pcm ${CMAKE_CURRENT_BINARY_DIR}/podioRootIODictDict.rootmap ${CMAKE_CURRENT_BINARY_DIR}/libpodioRootIODict_rdict.pcm - ${CMAKE_CURRENT_BINARY_DIR}/podioPythonStoreDictDict.rootmap - ${CMAKE_CURRENT_BINARY_DIR}/libpodioPythonStoreDict_rdict.pcm DESTINATION "${CMAKE_INSTALL_LIBDIR}") if (ENABLE_SIO) diff --git a/src/EventStore.cc b/src/EventStore.cc deleted file mode 100644 index dde8961f8..000000000 --- a/src/EventStore.cc +++ /dev/null @@ -1,140 +0,0 @@ - -// podio specific includes -#include "podio/EventStore.h" -#include "podio/CollectionBase.h" -#include "podio/IReader.h" - -namespace podio { - -EventStore::EventStore() : m_table(new CollectionIDTable()) { -} - -EventStore::~EventStore() { - for (auto& coll : m_collections) { - delete coll.second; - } -} - -bool EventStore::get(uint32_t id, CollectionBase*& collection) const { - auto val = m_retrievedIDs.insert(id); - bool success = false; - if (val.second == true) { - // collection not yet retrieved in recursive-call - auto name = m_table->name(id).value(); - success = doGet(name, collection, true); - } else { - // collection already requested in recursive call - // do not set the references to break collection dependency-cycle - auto name = m_table->name(id).value(); - success = doGet(name, collection, false); - } - // fg: the set should only be cleared at the end of event (in clear() ) ... - // m_retrievedIDs.erase(id); - return success; -} - -void EventStore::registerCollection(const std::string& name, podio::CollectionBase* coll) { - m_collections.emplace_back(name, coll); - auto id = m_table->add(name); - coll->setID(id); -} - -bool EventStore::isValid() const { - return m_reader->isValid(); -} - -bool EventStore::doGet(const std::string& name, CollectionBase*& collection, bool setReferences) const { - auto result = std::find_if(begin(m_collections), end(m_collections), - [&name](const CollPair& item) -> bool { return name == item.first; }); - if (result != end(m_collections)) { - auto tmp = result->second; - if (tmp != nullptr) { - collection = tmp; - return true; - } - } else if (m_reader != nullptr) { - auto tmp = m_reader->readCollection(name); - if (setReferences == true) { - if (tmp != nullptr) { - tmp->setReferences(this); - // check again whether collection exists - // it may have been created on-demand already - if (collectionRegistered(name) == false) { - m_collections.emplace_back(std::make_pair(name, tmp)); - } - } - } - collection = tmp; - if (tmp != nullptr) { - return true; - } - } else { - return false; - } - return false; -} - -GenericParameters& EventStore::getEventMetaData() { - - if (m_evtMD.empty() && m_reader != nullptr) { - GenericParameters* tmp = m_reader->readEventMetaData(); - m_evtMD = std::move(*tmp); - delete tmp; - } - return m_evtMD; -} - -GenericParameters& EventStore::getRunMetaData(int runID) { - - if (m_runMDMap.empty() && m_reader != nullptr) { - RunMDMap* tmp = m_reader->readRunMetaData(); - m_runMDMap = std::move(*tmp); - delete tmp; - } - return m_runMDMap[runID]; -} - -GenericParameters& EventStore::getCollectionMetaData(uint32_t colID) { - - if (m_colMDMap.empty() && m_reader != nullptr) { - ColMDMap* tmp = m_reader->readCollectionMetaData(); - m_colMDMap = std::move(*tmp); - delete tmp; - } - return m_colMDMap[colID]; -} - -void EventStore::clearCollections() { - for (auto& coll : m_collections) { - coll.second->clear(); - } - m_evtMD.clear(); -} - -void EventStore::clear() { - for (auto& coll : m_collections) { - coll.second->clear(); - delete coll.second; - } - - m_evtMD.clear(); - clearCaches(); -} - -void EventStore::clearCaches() { - m_collections.clear(); - m_retrievedIDs.clear(); -} - -bool EventStore::collectionRegistered(const std::string& name) const { - auto result = std::find_if(begin(m_collections), end(m_collections), - [&name](const CollPair& item) -> bool { return name == item.first; }); - return (result != end(m_collections)); -} - -void EventStore::setReader(IReader* reader) { - m_reader = reader; - setCollectionIDTable(reader->getCollectionIDTable()); -} - -} // namespace podio diff --git a/src/IOHelpers.cc b/src/IOHelpers.cc deleted file mode 100644 index 7dd764083..000000000 --- a/src/IOHelpers.cc +++ /dev/null @@ -1,39 +0,0 @@ -#include "podio/utilities/IOHelpers.h" - -#include "podio/ROOTReader.h" - -#if PODIO_ENABLE_SIO - #include "podio/SIOReader.h" -#endif - -namespace podio { -std::unique_ptr createReader(const std::string& filename) { - const auto fileEnding = [&filename]() -> std::string { - const auto n = filename.rfind('.'); - if (n != std::string::npos) { - return filename.substr(n); - } - return ""; - }(); - - if (fileEnding.empty()) { - return nullptr; - } - - if (fileEnding == ".root") { - return std::make_unique(); - } else if (fileEnding == ".sio") { -#if PODIO_ENABLE_SIO - return std::make_unique(); -#else - std::cerr << "PODIO: You are trying to open a .sio file but podio has not been built with SIO support\nMake sure " - "to build PODIO with SIO support to be able to read .sio files" - << std::endl; - return nullptr; -#endif - } else { - return nullptr; - } -} - -} // namespace podio diff --git a/src/PythonEventStore.cc b/src/PythonEventStore.cc deleted file mode 100644 index 979bfb261..000000000 --- a/src/PythonEventStore.cc +++ /dev/null @@ -1,54 +0,0 @@ -#include "podio/PythonEventStore.h" - -#include "podio/ROOTReader.h" -#include "podio/utilities/IOHelpers.h" - -#include -#include -#include - -podio::PythonEventStore::PythonEventStore(const char* name) : m_reader(podio::createReader(name)), m_store() { - std::ifstream inputfile(name); - m_isZombie = inputfile.good() ? false : true; - - if (m_reader) { - if (m_isZombie && dynamic_cast(m_reader.get())) { - // the file could be a remote file that we cannot access but root - // knows how to handle via the xrootd protocol. - // if that is the case we ignore m_isZombie. - if (!std::string("root:/").compare(0, 6, name, 6)) { - m_isZombie = false; - } - } - } - - if (!m_isZombie) { - // at this point we have a combination of file and reader that should work - m_reader->openFile(name); - m_store.setReader(m_reader.get()); - } -} - -const podio::CollectionBase* podio::PythonEventStore::get(const char* name) { - const podio::CollectionBase* coll(nullptr); - m_store.get(name, coll); - return coll; -} - -void podio::PythonEventStore::endOfEvent() { - m_store.clear(); - m_reader->endOfEvent(); -} - -void podio::PythonEventStore::goToEvent(unsigned ievent) { - m_store.clear(); - m_reader->goToEvent(ievent); -} - -unsigned podio::PythonEventStore::getEntries() const { - return m_reader->getEntries(); -} - -const std::vector& podio::PythonEventStore::getCollectionNames() const { - return m_store.getCollectionIDTable()->names(); -} diff --git a/src/ROOTReader.cc b/src/ROOTReader.cc deleted file mode 100644 index 5b91378fe..000000000 --- a/src/ROOTReader.cc +++ /dev/null @@ -1,295 +0,0 @@ -#include "rootUtils.h" - -// podio specific includes -#include "podio/CollectionBase.h" -#include "podio/CollectionIDTable.h" -#include "podio/GenericParameters.h" -#include "podio/ROOTReader.h" - -// ROOT specific includes -#include "TChain.h" -#include "TClass.h" -#include "TFile.h" -#include "TTree.h" -#include "TTreeCache.h" -#include -#include - -namespace podio { -// todo: see https://github.com/AIDASoft/podio/issues/290 -ROOTReader::~ROOTReader() { // NOLINT(modernize-use-equals-default) -} - -std::pair ROOTReader::getLocalTreeAndEntry(const std::string& treename) { - auto localEntry = m_chain->LoadTree(m_eventNumber); - auto* tree = static_cast(m_chain->GetFile()->Get(treename.c_str())); - return {tree, localEntry}; -} - -GenericParameters* ROOTReader::readEventMetaData() { - auto* emd = new GenericParameters(); - auto [evt_metadatatree, entry] = getLocalTreeAndEntry("evt_metadata"); - auto* branch = root_utils::getBranch(evt_metadatatree, "evtMD"); - branch->SetAddress(&emd); - evt_metadatatree->GetEntry(entry); - return emd; -} -std::map* ROOTReader::readCollectionMetaData() { - auto* emd = new std::map; - auto* col_metadatatree = getLocalTreeAndEntry("col_metadata").first; - auto* branch = root_utils::getBranch(col_metadatatree, "colMD"); - branch->SetAddress(&emd); - col_metadatatree->GetEntry(0); - return emd; -} -std::map* ROOTReader::readRunMetaData() { - auto* emd = new std::map; - auto* run_metadatatree = getLocalTreeAndEntry("run_metadata").first; - auto* branch = root_utils::getBranch(run_metadatatree, "runMD"); - branch->SetAddress(&emd); - run_metadatatree->GetEntry(0); - return emd; -} - -CollectionBase* ROOTReader::readCollection(const std::string& name) { - // has the collection already been constructed? - auto p = - std::find_if(begin(m_inputs), end(m_inputs), [&name](const ROOTReader::Input& t) { return t.second == name; }); - if (p != end(m_inputs)) { - return p->first; - } - - // Do we know about this collection? If so, read it - if (const auto& info = m_storedClasses.find(name); info != m_storedClasses.end()) { - return getCollection(*info); - } - - // At this point this collection is definitely not in this file, because we - // have no information on how to construct it in the first place - return nullptr; -} - -CollectionBase* ROOTReader::getCollection(const std::pair& collInfo) { - const auto& name = collInfo.first; - const auto& [theClass, collectionClass, index] = collInfo.second; - auto& branches = m_collectionBranches[index]; - - auto* collection = static_cast(collectionClass->New()); - auto collBuffers = collection->getBuffers(); - // If we have a valid data buffer class we know that have to read data, - // otherwise we are handling a subset collection - if (theClass) { - collBuffers.data = theClass->New(); - } else { - collection->setSubsetCollection(); - } - - const auto localEntry = m_chain->LoadTree(m_eventNumber); - // After switching trees in the chain, branch pointers get invalidated so - // they need to be reassigned. - // NOTE: root 6.22/06 requires that we get completely new branches here, - // with 6.20/04 we could just re-set them - if (localEntry == 0) { - branches.data = root_utils::getBranch(m_chain, name.c_str()); - - // reference collections - if (auto* refCollections = collBuffers.references) { - for (size_t i = 0; i < refCollections->size(); ++i) { - const auto brName = root_utils::refBranch(name, i); - branches.refs[i] = root_utils::getBranch(m_chain, brName.c_str()); - } - } - - // vector members - if (auto* vecMembers = collBuffers.vectorMembers) { - for (size_t i = 0; i < vecMembers->size(); ++i) { - const auto brName = root_utils::vecBranch(name, i); - branches.vecs[i] = root_utils::getBranch(m_chain, brName.c_str()); - } - } - } - - // set the addresses - root_utils::setCollectionAddresses(collection->getBuffers(), branches); - - return readCollectionData(branches, collection, localEntry, name); -} - -CollectionBase* ROOTReader::readCollectionData(const root_utils::CollectionBranches& branches, - CollectionBase* collection, Long64_t entry, const std::string& name) { - // Read all data - if (branches.data) { - branches.data->GetEntry(entry); - } - for (auto* br : branches.refs) { - br->GetEntry(entry); - } - for (auto* br : branches.vecs) { - br->GetEntry(entry); - } - - // do the unpacking - const auto id = m_table->collectionID(name).value(); - collection->setID(id); - collection->prepareAfterRead(); - - m_inputs.emplace_back(collection, name); - return collection; -} - -void ROOTReader::openFile(const std::string& filename) { - openFiles({filename}); -} - -void ROOTReader::openFiles(const std::vector& filenames) { - m_chain = new TChain("events"); - for (const auto& filename : filenames) { - //-1 forces the headers to be read so that - // the validity of the files can be checked - if (!m_chain->Add(filename.c_str(), -1)) { - delete m_chain; - throw std::runtime_error("File " + filename + " couldn't be found"); - } - } - - // read the meta data and build the collectionBranches cache - // NOTE: This is a small pessimization, if we do not read all collections - // afterwards, but it makes the handling much easier in general - auto metadatatree = static_cast(m_chain->GetFile()->Get("metadata")); - m_table = std::make_shared(); - auto* table = m_table.get(); - metadatatree->SetBranchAddress("CollectionIDs", &table); - - podio::version::Version* versionPtr{nullptr}; - if (auto* versionBranch = root_utils::getBranch(metadatatree, "PodioVersion")) { - versionBranch->SetAddress(&versionPtr); - metadatatree->GetEntry(0); - } - m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; - - // Read the collection type info - // For versions <0.13.1 it does not exist and has to be rebuilt from scratch - if (m_fileVersion < podio::version::Version{0, 13, 1}) { - - std::cout << "PODIO: Reconstructing CollectionTypeInfo branch from other sources in file: \'" - << m_chain->GetFile()->GetName() << "\'" << std::endl; - metadatatree->GetEntry(0); - const auto collectionInfo = root_utils::reconstructCollectionInfo(m_chain, *m_table); - createCollectionBranches(collectionInfo); - - } else if (m_fileVersion < podio::version::Version{0, 16, 4}) { - - auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); - auto collectionInfoWithoutSchema = new std::vector; - auto collectionInfo = new std::vector; - collInfoBranch->SetAddress(&collectionInfoWithoutSchema); - metadatatree->GetEntry(0); - for (const auto& [collID, collType, isSubsetColl] : *collectionInfoWithoutSchema) { - collectionInfo->emplace_back(collID, collType, isSubsetColl, 0); - } - createCollectionBranches(*collectionInfo); - delete collectionInfoWithoutSchema; - delete collectionInfo; - - } else { - - auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); - - auto collectionInfo = new std::vector; - collInfoBranch->SetAddress(&collectionInfo); - metadatatree->GetEntry(0); - createCollectionBranches(*collectionInfo); - delete collectionInfo; - } - - delete versionPtr; -} - -void ROOTReader::closeFile() { - closeFiles(); -} - -void ROOTReader::closeFiles() { - delete m_chain; -} - -void ROOTReader::readEvent() { - m_chain->GetEntry(m_eventNumber); - // first prepare all collections in memory... - for (auto inputs : m_inputs) { - inputs.first->prepareAfterRead(); - } - // ...then clean-up the references between them - // for(auto inputs : m_inputs){ - // inputs.first->setReferences(m_registry); - - // } -} -bool ROOTReader::isValid() const { - return m_chain->GetFile()->IsOpen() && !m_chain->GetFile()->IsZombie(); -} - -void ROOTReader::endOfEvent() { - ++m_eventNumber; - m_inputs.clear(); -} - -unsigned ROOTReader::getEntries() const { - return m_chain->GetEntries(); -} - -void ROOTReader::goToEvent(unsigned eventNumber) { - m_eventNumber = eventNumber; - m_inputs.clear(); -} - -void ROOTReader::createCollectionBranches(const std::vector& collInfo) { - size_t collectionIndex{0}; - - for (const auto& [collID, collType, isSubsetColl, collSchemaVersion] : collInfo) { - // We only write collections that are in the collectionIDTable, so no need - // to check here - const auto name = m_table->name(collID).value(); - - root_utils::CollectionBranches branches{}; - const auto collectionClass = TClass::GetClass(collType.c_str()); - - // Make sure that ROOT actually knows about this datatype before running - // into a potentially cryptic segmentation fault by accessing the nullptr - if (!collectionClass) { - std::cerr << "PODIO: Cannot create the collection type \'" << collType << "\' stored in branch \'" << name - << "\'. Contents of this branch cannot be read." << std::endl; - continue; - } - // Need the collection here to setup all the branches. Have to manage the - // temporary collection ourselves - auto collection = - std::unique_ptr(static_cast(collectionClass->New())); - collection->setSubsetCollection(isSubsetColl); - - if (!isSubsetColl) { - // This branch is guaranteed to exist since only collections that are - // also written to file are in the info metadata that we work with here - branches.data = root_utils::getBranch(m_chain, name.c_str()); - } - - const auto buffers = collection->getBuffers(); - for (size_t i = 0; i < buffers.references->size(); ++i) { - const auto brName = root_utils::refBranch(name, i); - branches.refs.push_back(root_utils::getBranch(m_chain, brName.c_str())); - } - - for (size_t i = 0; i < buffers.vectorMembers->size(); ++i) { - const auto brName = root_utils::vecBranch(name, i); - branches.vecs.push_back(root_utils::getBranch(m_chain, brName.c_str())); - } - - const auto bufferClassName = "std::vector<" + std::string(collection->getDataTypeName()) + ">"; - const auto bufferClass = isSubsetColl ? nullptr : TClass::GetClass(bufferClassName.c_str()); - - m_storedClasses.emplace(name, std::make_tuple(bufferClass, collectionClass, collectionIndex++)); - m_collectionBranches.push_back(branches); - } -} - -} // namespace podio diff --git a/src/ROOTWriter.cc b/src/ROOTWriter.cc deleted file mode 100644 index 73e9f8e1f..000000000 --- a/src/ROOTWriter.cc +++ /dev/null @@ -1,145 +0,0 @@ -#include "rootUtils.h" - -// podio specific includes -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/ROOTWriter.h" -#include "podio/podioVersion.h" - -// ROOT specifc includes -#include "TFile.h" -#include "TTree.h" - -namespace podio { -ROOTWriter::ROOTWriter(const std::string& filename, EventStore* store) : - m_filename(filename), - m_store(store), - m_file(new TFile(filename.c_str(), "RECREATE", "data file")), - m_datatree(new TTree("events", "Events tree")), - m_metadatatree(new TTree("metadata", "Metadata tree")), - m_runMDtree(new TTree("run_metadata", "Run metadata tree")), - m_evtMDtree(new TTree("evt_metadata", "Event metadata tree")), - m_colMDtree(new TTree("col_metadata", "Collection metadata tree")) { - - m_evtMDtree->Branch("evtMD", "GenericParameters", m_store->eventMetaDataPtr()); -} - -ROOTWriter::~ROOTWriter() { - delete m_file; -} - -void ROOTWriter::writeEvent() { - std::vector collections; - collections.reserve(m_collectionsToWrite.size()); - for (const auto& name : m_collectionsToWrite) { - const podio::CollectionBase* coll; - m_store->get(name, coll); - collections.emplace_back(name, const_cast(coll)); - collections.back().second->prepareForWrite(); - } - - if (m_firstEvent) { - createBranches(collections); - m_firstEvent = false; - } else { - setBranches(collections); - } - - m_datatree->Fill(); - m_evtMDtree->Fill(); -} - -void ROOTWriter::createBranches(const std::vector& collections) { - for (auto& [name, coll] : collections) { - root_utils::CollectionBranches branches; - const auto collBuffers = coll->getBuffers(); - if (collBuffers.data) { - // only create the data buffer branch if necessary - - const auto collClassName = "vector<" + std::string(coll->getDataTypeName()) + ">"; - - branches.data = m_datatree->Branch(name.c_str(), collClassName.c_str(), collBuffers.data); - } - - // reference collections - if (auto refColls = collBuffers.references) { - int i = 0; - for (auto& c : (*refColls)) { - const auto brName = root_utils::refBranch(name, i); - branches.refs.push_back(m_datatree->Branch(brName.c_str(), c.get())); - ++i; - } - } - - // vector members - if (auto vminfo = collBuffers.vectorMembers) { - int i = 0; - for (auto& [type, vec] : (*vminfo)) { - const auto typeName = "vector<" + type + ">"; - const auto brName = root_utils::vecBranch(name, i); - branches.vecs.push_back(m_datatree->Branch(brName.c_str(), typeName.c_str(), vec)); - ++i; - } - } - - m_collectionBranches.push_back(branches); - } -} - -void ROOTWriter::setBranches(const std::vector& collections) { - size_t iCollection = 0; - for (auto& coll : collections) { - const auto& branches = m_collectionBranches[iCollection]; - root_utils::setCollectionAddresses(coll.second->getBuffers(), branches); - - iCollection++; - } -} - -void ROOTWriter::finish() { - // now we want to safe the metadata. This includes info about the - // collections - const auto collIDTable = m_store->getCollectionIDTable(); - m_metadatatree->Branch("CollectionIDs", collIDTable); - - // collectionID, collection type, subset collection - std::vector collectionInfo; - collectionInfo.reserve(m_collectionsToWrite.size()); - for (const auto& name : m_collectionsToWrite) { - const auto collID = collIDTable->collectionID(name).value(); - const podio::CollectionBase* coll{nullptr}; - // No check necessary, only registered collections possible - m_store->get(name, coll); - const auto collType = coll->getTypeName(); - // const auto collType = "std::vector<" + coll->getDataTypeName() + ">"; - collectionInfo.emplace_back(collID, std::move(collType), coll->isSubsetCollection(), coll->getSchemaVersion()); - } - - m_metadatatree->Branch("CollectionTypeInfo", &collectionInfo); - - podio::version::Version podioVersion = podio::version::build_version; - m_metadatatree->Branch("PodioVersion", &podioVersion); - - m_metadatatree->Fill(); - - m_colMDtree->Branch("colMD", "std::map", m_store->getColMetaDataMap()); - m_colMDtree->Fill(); - m_runMDtree->Branch("runMD", "std::map", m_store->getRunMetaDataMap()); - m_runMDtree->Fill(); - - m_file->Write(); - m_file->Close(); -} - -bool ROOTWriter::registerForWrite(const std::string& name) { - const podio::CollectionBase* tmp_coll(nullptr); - if (!m_store->get(name, tmp_coll)) { - std::cerr << "no such collection to write, throw exception." << std::endl; - return false; - } - - m_collectionsToWrite.push_back(name); - return true; -} - -} // namespace podio diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index efb26c867..4645d2a74 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -12,25 +12,6 @@ #endif namespace podio { -SIOCollectionIDTableBlock::SIOCollectionIDTableBlock(podio::EventStore* store) : - sio::block("CollectionIDs", sio::version::encode_version(0, 3)) { - const auto table = store->getCollectionIDTable(); - _names = table->names(); - _ids = table->ids(); - _types.reserve(_names.size()); - _isSubsetColl.reserve(_names.size()); - for (const int id : _ids) { - CollectionBase* tmp; - if (!store->get(id, tmp)) { - std::cerr - << "PODIO-ERROR cannot construct CollectionIDTableBlock because a collection is missing from the store (id: " - << id << ", name: " << table->name(id).value_or("") << ")" << std::endl; - } - - _types.emplace_back(tmp->getValueTypeName()); - _isSubsetColl.push_back(tmp->isSubsetCollection()); - } -} void SIOCollectionIDTableBlock::read(sio::read_device& device, sio::version_type version) { device.data(_names); diff --git a/src/SIOReader.cc b/src/SIOReader.cc deleted file mode 100644 index 0ad0d9e15..000000000 --- a/src/SIOReader.cc +++ /dev/null @@ -1,241 +0,0 @@ -// podio specific includes -#include "podio/SIOReader.h" - -#include "podio/CollectionBase.h" -#include "podio/CollectionIDTable.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" -#include "sio/definitions.h" - -#include -#include - -namespace podio { - -SIOReader::SIOReader() : - m_eventNumber(0), - m_eventMetaData(std::make_shared()), - m_runMetaData(std::make_shared("RunMetaData")), - m_collectionMetaData(std::make_shared("CollectionMetaData")) { - auto& libLoader [[maybe_unused]] = SIOBlockLibraryLoader::instance(); -} - -CollectionBase* SIOReader::readCollection(const std::string& name) { - if (m_lastEventRead != m_eventNumber) { - readEvent(); - } - - // Have we unpacked this already? - auto p = - std::find_if(begin(m_inputs), end(m_inputs), [&name](const SIOReader::Input& t) { return t.second == name; }); - - if (p != end(m_inputs)) { - p->first->setID(m_table->collectionID(name).value()); - p->first->prepareAfterRead(); - return p->first; - } - - return nullptr; -} - -std::map* SIOReader::readCollectionMetaData() { - // Always read a new map, because the EventStore takes ownership - m_collectionMetaData->data = new ColMDMap(); - readMetaDataRecord(m_collectionMetaData); - return m_collectionMetaData->data; -} - -std::map* SIOReader::readRunMetaData() { - // Always read a new map, because the EventStore takes ownership - m_runMetaData->data = new RunMDMap(); - readMetaDataRecord(m_runMetaData); - return m_runMetaData->data; -} - -podio::GenericParameters* SIOReader::readEventMetaData() { - if (m_lastEventRead != m_eventNumber) { - readEvent(); - } - return m_eventMetaData->metadata; -} - -void SIOReader::openFile(const std::string& filename) { - m_stream.open(filename, std::ios::binary); - if (!this->isValid()) { - throw std::runtime_error("File " + filename + " couldn't be found"); - } - readCollectionIDTable(); - - if (!readFileTOCRecord()) { - reconstructFileTOCRecord(); - } -} - -void SIOReader::closeFile() { - m_stream.close(); -} - -void SIOReader::readEvent() { - // recreate the blocks, since the contents are owned and managed by the - // EventStore - createBlocks(); - - // skip possible intermediate records that are not event data - sio::api::go_to_record(m_stream, "event_record"); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - sio::api::read_blocks(m_unc_buffer.span(), m_blocks); - - for (size_t i = 1; i < m_blocks.size(); ++i) { - auto* blk = static_cast(m_blocks[i].get()); - m_inputs.emplace_back(blk->getCollection(), m_table->names()[i - 1]); - } - - m_lastEventRead = m_eventNumber; -} - -bool SIOReader::isValid() const { - return m_stream.good(); -} - -void SIOReader::endOfEvent() { - ++m_eventNumber; - m_blocks.clear(); - m_inputs.clear(); -} - -void SIOReader::goToEvent(unsigned eventNumber) { - // If we are already past the desired event number, rewind to the start first - if (eventNumber < (unsigned)m_eventNumber) { - m_stream.clear(); - m_stream.seekg(0); - m_eventNumber = 0; - } - - sio::api::go_to_record(m_stream, "event_record"); - if ((eventNumber - m_eventNumber) > 0) { - sio::api::skip_n_records(m_stream, eventNumber - m_eventNumber); - } - m_eventNumber = eventNumber; - - m_inputs.clear(); - m_blocks.clear(); -} - -void SIOReader::createBlocks() { - // make sure that the first block is EventMetaData as it is also the first - // during wrting - m_eventMetaData->metadata = new GenericParameters(); // will be managed by EventStore (?) - m_blocks.push_back(m_eventMetaData); - - for (size_t i = 0; i < m_typeNames.size(); ++i) { - const bool subsetColl = !m_subsetCollectionBits.empty() && m_subsetCollectionBits[i]; - auto blk = podio::SIOBlockFactory::instance().createBlock(m_typeNames[i], m_table->names()[i], subsetColl); - m_blocks.push_back(blk); - } -} - -void SIOReader::readCollectionIDTable() { - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks; - blocks.emplace_back(std::make_shared()); - blocks.emplace_back(std::make_shared()); - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - auto* idTableBlock = static_cast(blocks[0].get()); - m_table = std::make_shared(); - m_table.reset(idTableBlock->getTable()); - m_typeNames = idTableBlock->getTypeNames(); - m_subsetCollectionBits = idTableBlock->getSubsetCollectionBits(); - m_fileVersion = static_cast(blocks[1].get())->version; -} - -void SIOReader::readMetaDataRecord(const std::shared_ptr& mdBlock) { - const auto currPos = m_stream.tellg(); - sio::api::go_to_record(m_stream, mdBlock->name()); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks{}; - blocks.push_back(mdBlock); - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - m_stream.seekg(currPos); -} - -void SIOReader::reconstructFileTOCRecord() { - try { - // use a simple unary predicate that always returns true, and hence skips - // over all records, but as a sideffect populates the tocRecord - sio::api::skip_records(m_stream, [&](const sio::record_info& rec_info) { - m_tocRecord.addRecord(rec_info._name, rec_info._file_start); - return true; - }); - } catch (sio::exception& e) { - if (e.code() != sio::error_code::eof) { - SIO_RETHROW(e, e.code(), e.what()); - } - } - - // rewind to the start of the file - m_stream.clear(); - m_stream.seekg(0); -} - -bool SIOReader::readFileTOCRecord() { - // Check if there is a dedicated marker at the end of the file that tells us - // where the TOC actually starts - m_stream.seekg(-sio_helpers::SIOTocInfoSize, std::ios_base::end); - uint64_t firstWords{0}; - m_stream.read(reinterpret_cast(&firstWords), sizeof(firstWords)); - - const uint32_t marker = (firstWords >> 32) & 0xffffffff; - if (marker == sio_helpers::SIOTocMarker) { - const uint32_t position = firstWords & 0xffffffff; - m_stream.seekg(position); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks; - auto tocBlock = std::make_shared(); - tocBlock->record = &m_tocRecord; - blocks.push_back(tocBlock); - - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - m_unc_buffer.clear(); - m_rec_buffer.clear(); - m_stream.seekg(0); - return true; - } - - m_stream.clear(); - m_stream.seekg(0); - return false; -} -} // namespace podio diff --git a/src/SIOWriter.cc b/src/SIOWriter.cc deleted file mode 100644 index 1c1221c13..000000000 --- a/src/SIOWriter.cc +++ /dev/null @@ -1,124 +0,0 @@ -// podio specific includes -#include "podio/SIOWriter.h" -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" - -#include "sioUtils.h" - -// SIO specifc includes -#include "sio/block.h" -#include "sio/compression/zlib.h" - -namespace podio { - -SIOWriter::SIOWriter(const std::string& filename, EventStore* store) : - m_filename(filename), - m_store(store), - m_eventMetaData(std::make_shared()), - m_runMetaData(std::make_shared("RunMetaData")), - m_collectionMetaData(std::make_shared("CollectionMetaData")) { - - m_stream.open(filename, std::ios::binary); - - if (not m_stream.is_open()) { - SIO_THROW(sio::error_code::not_open, "Couldn't open output stream '" + filename + "'"); - } - - // TODO: re-visit this once metadata handling is done in better defined way - m_eventMetaData->metadata = m_store->eventMetaDataPtr(); - - m_runMetaData->data = m_store->getRunMetaDataMap(); - m_collectionMetaData->data = m_store->getColMetaDataMap(); - - auto& libLoader [[maybe_unused]] = SIOBlockLibraryLoader::instance(); -} - -SIOWriter::~SIOWriter() { - if (!m_finished) { - finish(); - } -} - -void SIOWriter::writeEvent() { - if (m_firstEvent) { - // Write the collectionIDs as a separate record at the beginning of the - // file. In this way they can easily be retrieved in the SIOReader without - // having to look for this specific record. - writeCollectionIDTable(); - m_firstEvent = false; - } - - auto blocks = createBlocks(); - m_tocRecord.addRecord("event_record", sio_utils::writeRecord(blocks, "event_record", m_stream)); -} - -sio::block_list SIOWriter::createBlocks() const { - sio::block_list blocks; - blocks.emplace_back(m_eventMetaData); - - for (const auto& name : m_collectionsToWrite) { - const podio::CollectionBase* col{nullptr}; - m_store->get(name, col); - col->prepareForWrite(); - - blocks.emplace_back(podio::SIOBlockFactory::instance().createBlock(col, name)); - } - - return blocks; -} - -void SIOWriter::finish() { - sio::block_list blocks{}; - blocks.push_back(m_runMetaData); - - m_tocRecord.addRecord(m_runMetaData->name(), sio_utils::writeRecord(blocks, m_runMetaData->name(), m_stream)); - - blocks.clear(); - blocks.push_back(m_collectionMetaData); - m_tocRecord.addRecord(m_collectionMetaData->name(), - sio_utils::writeRecord(blocks, m_collectionMetaData->name(), m_stream)); - - blocks.clear(); - auto tocRecordBlock = std::make_shared(); - tocRecordBlock->record = &m_tocRecord; - blocks.push_back(tocRecordBlock); - - const auto tocStartPos = sio_utils::writeRecord(blocks, sio_helpers::SIOTocRecordName, m_stream); - // Now that we know the position of the TOC Record, put this information - // into a final marker that can be identified and interpreted when reading - // again - uint64_t finalWords = (((uint64_t)sio_helpers::SIOTocMarker) << 32) | ((uint64_t)tocStartPos & 0xffffffff); - m_stream.write(reinterpret_cast(&finalWords), sizeof(finalWords)); - - m_stream.close(); - - m_finished = true; -} - -void SIOWriter::registerForWrite(const std::string& name) { - - const podio::CollectionBase* colB(nullptr); - m_store->get(name, colB); - - if (!colB) { - throw std::runtime_error(std::string("no such collection to write: ") + name); - } - // Check if we can instantiate the blocks here so that we can skip the checks later - if (auto blk = podio::SIOBlockFactory::instance().createBlock(colB, name); !blk) { - const auto typName = std::string(colB->getValueTypeName()); - throw std::runtime_error(std::string("could not create SIOBlock for type: ") + typName); - } - - m_collectionsToWrite.push_back(name); -} - -void SIOWriter::writeCollectionIDTable() { - sio::block_list blocks; - blocks.emplace_back(std::make_shared(m_store)); - blocks.emplace_back(std::make_shared(podio::version::build_version)); - - m_tocRecord.addRecord("file_metadata", sio_utils::writeRecord(blocks, "file_metadata", m_stream)); -} - -} // namespace podio diff --git a/src/python_selection.xml b/src/python_selection.xml deleted file mode 100644 index 61b39b94c..000000000 --- a/src/python_selection.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/selection.xml b/src/selection.xml index 4daca9192..03686376f 100644 --- a/src/selection.xml +++ b/src/selection.xml @@ -10,7 +10,6 @@ - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4132ff19c..b8233c528 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,23 +36,30 @@ PODIO_ADD_ROOT_IO_DICT(ExtensionDataModelDict ExtensionDataModel "${ext_headers} PODIO_ADD_SIO_IO_BLOCKS(ExtensionDataModel "${ext_headers}" "${ext_sources}") -set(legacy_test_versions +# Add a legacy test case based on a base executable and a version for which an +# input file exists +macro(ADD_PODIO_LEGACY_TEST version base_test input_file) + ExternalData_Add_Test(legacy_test_cases + NAME ${base_test}_${version} + COMMAND ${base_test} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} + ) + PODIO_SET_TEST_ENV(${base_test}_${version}) +endmacro() + +set(root_legacy_test_versions v00-16 v00-16-02 v00-16-05 v00-16-06 ) -include(ExternalData) -list(APPEND ExternalData_URL_TEMPLATES - "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" - ) - -add_executable(check_benchmark_outputs check_benchmark_outputs.cpp) -target_link_libraries(check_benchmark_outputs PRIVATE ROOT::Tree) - add_subdirectory(root_io) if (ENABLE_SIO) + set(sio_legacy_test_versions + v00-16-05 + v00-16-06 + ) + add_subdirectory(sio_io) endif() add_subdirectory(unittests) @@ -61,7 +68,6 @@ add_subdirectory(schema_evolution) # Tests that don't fit into one of the broad categories above CREATE_PODIO_TEST(ostream_operator.cpp "") -CREATE_PODIO_TEST(write_ascii.cpp "") if (ENABLE_JULIA) message(STATUS "Julia Datamodel generation is enabled.") @@ -93,5 +99,3 @@ endif() # Customize CTest to potentially disable some of the tests with known problems configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY) - -ExternalData_Add_Target(legacy_test_cases) diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index af55f1532..396c32b78 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -11,15 +11,8 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} - write - read - read_and_write read_and_write_associated - write_timed - read_timed check_benchmark_outputs - read-multiple - read-legacy-files-root_v00-13 read_frame_legacy_root read_frame_root_multiple write_python_frame_root @@ -29,26 +22,23 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ write_frame_root read_frame_root - check_benchmark_outputs_sio write_python_frame_sio read_python_frame_sio - write_ascii - relation_range pyunittest podio-dump-help - podio-dump-root-legacy podio-dump-root podio-dump-detailed-root - podio-dump-detailed-root-legacy + podio-dump-legacy_root_v00-16-06 + podio-dump-legacy_root-detailed_v00-16-06 - podio-dump-sio-legacy podio-dump-sio podio-dump-detailed-sio - podio-dump-detailed-sio-legacy + podio-dump-legacy_sio_v00-16-06 + podio-dump-legacy_sio-detailed_v00-16-06 datamodel_def_store_roundtrip_root datamodel_def_store_roundtrip_root_extension @@ -59,12 +49,16 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ read_new_data_root ) - foreach(version in @legacy_test_versions@) - list(APPEND CTEST_CUSTOM_TESTS_IGNORE read-legacy-files-root_${version}) + foreach(version in @root_legacy_test_versions@) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_root_${version}) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_legacy_root_${version}) endforeach() + foreach(version in @sio_legacy_test_versions@) + list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_sio_${version}) + list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_legacy_sio_${version}) + endforeach() + # ostream_operator is working with Memory sanitizer (at least locally) if("@USE_SANITIZER@" MATCHES "Memory(WithOrigin)?") list(REMOVE_ITEM CTEST_CUSTOM_TESTS_IGNORE ostream_operator) diff --git a/tests/check_benchmark_outputs.cpp b/tests/check_benchmark_outputs.cpp deleted file mode 100644 index eb268f9d4..000000000 --- a/tests/check_benchmark_outputs.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "TFile.h" -#include "TTree.h" - -#include -#include -#include - -constexpr int nExpectedEvents = 2000; -using StringVec = std::vector; - -bool verifyTree(TTree* tree, int expectedEntries, const StringVec& expectedBranches) { - const std::string treeName = tree->GetName(); - if (tree->GetEntries() != expectedEntries) { - std::cerr << "Tree \'" << treeName << "\' should have " << expectedEntries << " but has " << tree->GetEntries() - << std::endl; - return false; - } - - const auto* branches = tree->GetListOfBranches(); - for (const auto& branch : expectedBranches) { - bool found = false; - for (int i = 0; i < branches->GetEntries(); ++i) { - if (branch == branches->At(i)->GetName()) { - found = true; - break; - } - } - if (!found) { - std::cerr << "Branch \'" << branch << "\' was expected to be in Tree \'" << treeName - << "\' but could not be found" << std::endl; - return false; - } - } - - if ((unsigned)branches->GetEntries() != expectedBranches.size()) { - std::cerr << "Tree \'" << treeName << "\' has additional, unexpected branches" << std::endl; - return false; - } - - return true; -} - -void verifyBMFile(const char* fileName, const StringVec& setupBranches, const StringVec& eventBranches, - int expectedEvents = nExpectedEvents) { - TFile* bmFile = TFile::Open(fileName); - if (!bmFile) { - std::cerr << "Benchmark file \'" << fileName << "\' does not exist!" << std::endl; - std::exit(1); - } - if (!verifyTree(static_cast(bmFile->Get("setup_times")), 1, setupBranches)) { - std::cerr << "In file \'" << fileName << "\' setup_times Tree does not have the expected entries" << std::endl; - bmFile->Close(); - std::exit(1); - } - if (!verifyTree(static_cast(bmFile->Get("event_times")), expectedEvents, eventBranches)) { - std::cerr << "In file \'" << fileName << "\' event_times Tree does not have the expected entries" << std::endl; - bmFile->Close(); - std::exit(1); - } - - bmFile->Close(); -} - -/** - * We can't really make any checks on the actual execution times, but we can at - * least verify that the expected timing points are here. - */ -int main(int, char* argv[]) { - const StringVec writeBMSetupBranches = {"constructor", "finish", "register_for_write"}; - const StringVec writeBMEventBranches = {"write_event"}; - verifyBMFile(argv[1], writeBMSetupBranches, writeBMEventBranches); - - const StringVec readBMSetupBranches = {"constructor", "open_file", "close_file", "get_entries", - "read_collection_ids"}; - const StringVec readBMEventBranches = {"read_collections", "read_ev_md", "read_run_md", - "read_coll_md", "end_of_event", "read_event"}; - verifyBMFile(argv[2], readBMSetupBranches, readBMEventBranches); - - return 0; -} diff --git a/tests/read.py b/tests/read.py deleted file mode 100644 index 3428b3c60..000000000 --- a/tests/read.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Integration test for python interface when reading a file with a podio -generated datamodel -""" - -from __future__ import absolute_import, unicode_literals, print_function - -from podio.EventStore import EventStore - -if __name__ == '__main__': - - FILENAME = 'example.root' - store = EventStore([FILENAME]) # pylint: disable=invalid-name # too strict before 2.5.0 - for i, event in enumerate(store): - if i % 1000 == 0: - print('reading event', i) - evinfo = store.get("info")[0] - clusters = store.get("clusters") - for cluster in clusters: - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - print(' Referenced hit has an energy of', hit.energy()) diff --git a/tests/read_test.h b/tests/read_test.h index 6382cb20a..e38be95d5 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -14,8 +14,7 @@ #include "datamodel/ExampleWithVectorMemberCollection.h" // podio specific includes -#include "podio/EventStore.h" -#include "podio/IReader.h" +#include "podio/Frame.h" #include "podio/UserDataCollection.h" #include "podio/podioVersion.h" @@ -39,19 +38,8 @@ bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std return true; } -template -static constexpr bool isEventStore = std::is_same_v; - -template -void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersion) { - - float evtWeight = -1; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - evtWeight = evtMD.template getValue("UserEventWeight"); - } else { - evtWeight = store.template getParameter("UserEventWeight"); - } +void processEvent(const podio::Frame& event, int eventNum, podio::version::Version fileVersion) { + const float evtWeight = event.getParameter("UserEventWeight"); if (evtWeight != (float)100. * eventNum) { std::cout << " read UserEventWeight: " << evtWeight << " - expected : " << (float)100. * eventNum << std::endl; throw std::runtime_error("Couldn't read event meta data parameters 'UserEventWeight'"); @@ -59,13 +47,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi std::stringstream ss; ss << " event_number_" << eventNum; - std::string evtName = ""; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - evtName = evtMD.template getValue("UserEventName"); - } else { - evtName = store.template getParameter("UserEventName"); - } + const auto& evtName = event.getParameter("UserEventName"); if (evtName != ss.str()) { std::cout << " read UserEventName: " << evtName << " - expected : " << ss.str() << std::endl; @@ -73,13 +55,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion > podio::version::Version{0, 14, 1}) { - std::vector someVectorData{}; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - someVectorData = evtMD.template getValue>("SomeVectorData"); - } else { - someVectorData = store.template getParameter>("SomeVectorData"); - } + const auto& someVectorData = event.getParameter>("SomeVectorData"); if (someVectorData.size() != 4) { throw std::runtime_error("Couldn't read event meta data parameters: 'SomeVectorData'"); } @@ -90,43 +66,18 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - if constexpr (!isEventStore) { - if (fileVersion > podio::version::Version{0, 16, 2}) { - const auto doubleParams = store.template getParameter>("SomeVectorData"); - if (doubleParams.size() != 2 || doubleParams[0] != eventNum * 1.1 || doubleParams[1] != eventNum * 2.2) { - throw std::runtime_error("Could not read event parameter: 'SomeDoubleValues' correctly"); - } - } - } - - try { - // not assigning to a variable, because it will remain unused, we just want - // the exception here - store.template get("notthere"); - } catch (const std::runtime_error& err) { - if (std::string(err.what()) != "No collection \'notthere\' is present in the EventStore") { - throw std::runtime_error("Trying to get non present collection \'notthere' should throw an exception"); + if (fileVersion > podio::version::Version{0, 16, 2}) { + const auto& doubleParams = event.getParameter>("SomeVectorData"); + if (doubleParams.size() != 2 || doubleParams[0] != eventNum * 1.1 || doubleParams[1] != eventNum * 2.2) { + throw std::runtime_error("Could not read event parameter: 'SomeDoubleValues' correctly"); } } // read collection meta data - auto& hits = store.template get("hits"); - if constexpr (isEventStore) { - const auto& colMD = store.getCollectionMetaData(hits.getID()); - const auto& es = colMD.template getValue("CellIDEncodingString"); - if (es != std::string("system:8,barrel:3,layer:6,slice:5,x:-16,y:-16")) { - std::cout << " meta data from collection 'hits' with id = " << hits.getID() - << " read CellIDEncodingString: " << es << " - expected : system:8,barrel:3,layer:6,slice:5,x:-16,y:-16" - << std::endl; - throw std::runtime_error("Couldn't read event meta data parameters 'CellIDEncodingString'"); - } - - } else { - // TODO: Integrate this into the frame workflow somehow - } + auto& hits = event.get("hits"); if (fileVersion > podio::version::Version{0, 14, 0}) { - auto& hitRefs = store.template get("hitRefs"); + auto& hitRefs = event.get("hitRefs"); if (hitRefs.size() != hits.size()) { throw std::runtime_error("hit and subset hit collection do not have the same size"); } @@ -135,7 +86,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& clusters = store.template get("clusters"); + auto& clusters = event.get("clusters"); if (clusters.isValid()) { auto cluster = clusters[0]; for (auto i = cluster.Hits_begin(), end = cluster.Hits_end(); i != end; ++i) { @@ -149,7 +100,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi // Read the mcParticleRefs before reading any of the other collections that // are referenced to make sure that all the necessary relations are handled // correctly - auto& mcpRefs = store.template get("mcParticleRefs"); + auto& mcpRefs = event.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -171,7 +122,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& mcps = store.template get("mcparticles"); + auto& mcps = event.get("mcparticles"); if (!mcps.isValid()) { throw std::runtime_error("Collection 'mcparticles' should be present"); } @@ -239,7 +190,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi // Load the subset collection first to ensure that it pulls in objects taht // have not been read yet - auto& mcpRefs = store.template get("mcParticleRefs"); + auto& mcpRefs = event.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -251,7 +202,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& moreMCs = store.template get("moreMCs"); + auto& moreMCs = event.get("moreMCs"); // First check that the two mc collections that we store are the same if (mcps.size() != moreMCs.size()) { @@ -284,7 +235,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } // std::cout << "Fetching collection 'refs'" << std::endl; - auto& refs = store.template get("refs"); + auto& refs = event.get("refs"); if (refs.isValid()) { auto ref = refs[0]; for (auto cluster : ref.Clusters()) { @@ -296,7 +247,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'refs' should be present"); } // std::cout << "Fetching collection 'OneRelation'" << std::endl; - auto& rels = store.template get("OneRelation"); + auto& rels = event.get("OneRelation"); if (rels.isValid()) { // std::cout << "Referenced object has an energy of " << (*rels)[0].cluster().energy() << std::endl; } else { @@ -304,7 +255,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } // std::cout << "Fetching collection 'WithVectorMember'" << std::endl; - auto& vecs = store.template get("WithVectorMember"); + auto& vecs = event.get("WithVectorMember"); if (vecs.isValid()) { if (vecs.size() != 2) { throw std::runtime_error("Collection 'WithVectorMember' should have two elements'"); @@ -331,14 +282,14 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'WithVectorMember' should be present"); } - auto& comps = store.template get("Component"); + auto& comps = event.get("Component"); if (comps.isValid()) { auto comp = comps[0]; int a [[maybe_unused]] = comp.component().data.x + comp.component().data.z; } - auto& arrays = store.template get("arrays"); - if (arrays.isValid() && arrays.size() != 0) { + auto& arrays = event.get("arrays"); + if (arrays.isValid() && !arrays.empty()) { auto array = arrays[0]; if (array.myArray(1) != eventNum) { throw std::runtime_error("Array not properly set."); @@ -353,8 +304,8 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'arrays' should be present"); } - auto& nmspaces = store.template get("WithNamespaceRelation"); - auto& copies = store.template get("WithNamespaceRelationCopy"); + auto& nmspaces = event.get("WithNamespaceRelation"); + auto& copies = event.get("WithNamespaceRelationCopy"); auto cpytest = ex42::ExampleWithARelationCollection{}; if (nmspaces.isValid() && copies.isValid()) { @@ -393,7 +344,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion >= podio::version::Version{0, 13, 1}) { - const auto& fixedWidthInts = store.template get("fixedWidthInts"); + const auto& fixedWidthInts = event.get("fixedWidthInts"); if (not fixedWidthInts.isValid() or fixedWidthInts.size() != 3) { throw std::runtime_error("Collection \'fixedWidthInts\' should be present and have 3 elements"); } @@ -427,7 +378,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion >= podio::version::Version{0, 13, 2}) { - auto& usrInts = store.template get>("userInts"); + auto& usrInts = event.get>("userInts"); if (usrInts.size() != (unsigned)eventNum + 1) { throw std::runtime_error("Could not read all userInts properly (expected: " + std::to_string(eventNum + 1) + @@ -449,7 +400,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& usrDbl = store.template get>("userDoubles"); + auto& usrDbl = event.get>("userDoubles"); if (usrDbl.size() != 100) { throw std::runtime_error( "Could not read all userDoubles properly (expected: 100, actual: " + std::to_string(usrDbl.size()) + ")"); @@ -463,45 +414,4 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } -void run_read_test(podio::IReader& reader) { - auto store = podio::EventStore(); - store.setReader(&reader); - - const auto nEvents = reader.getEntries(); - - // Some information in the test files dpends directly on the index. In - // read-multiple, the same file is essentially read twice, so that at the file - // change the index which is used for testing has to start at 0 again. This - // function just wraps that. The magic number of 2000 is the number of events - // that are present in each file that is written in write - const auto correctIndex = [nEvents](unsigned index) { - if (nEvents > 2000) { - return index % (nEvents / 2); - } - return index; - }; - - for (unsigned i = 0; i < nEvents; ++i) { - - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - - processEvent(store, correctIndex(i), reader.currentFileVersion()); - store.clear(); - reader.endOfEvent(); - } -} - -// Same as above but only for a specified event -void run_read_test_event(podio::IReader& reader, unsigned event) { - auto store = podio::EventStore(); - store.setReader(&reader); - - reader.goToEvent(event); - processEvent(store, event, reader.currentFileVersion()); - store.clear(); - reader.endOfEvent(); -} - #endif // PODIO_TESTS_READ_TEST_H diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index 099815022..1bc906755 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -1,17 +1,10 @@ set(root_dependent_tests - write.cpp - read.cpp - read-multiple.cpp relation_range.cpp - read_and_write.cpp read_and_write_associated.cpp - write_timed.cpp - read_timed.cpp read_frame_root.cpp write_frame_root.cpp - read_frame_legacy_root.cpp - read_frame_root_multiple.cpp read_python_frame_root.cpp + read_frame_root_multiple.cpp read_and_write_frame_root.cpp ) if(ENABLE_RNTUPLE) @@ -27,14 +20,6 @@ foreach( sourcefile ${root_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${root_libs}") endforeach() - -#--- set some dependencies between the different tests to ensure input generating ones are run first -set_property(TEST read PROPERTY DEPENDS write) -set_property(TEST read-multiple PROPERTY DEPENDS write) -set_property(TEST read_and_write PROPERTY DEPENDS write) -set_property(TEST read_frame_legacy_root PROPERTY DEPENDS write) -set_property(TEST read_timed PROPERTY DEPENDS write_timed) - set_tests_properties( read_frame_root read_frame_root_multiple @@ -48,35 +33,12 @@ if(ENABLE_RNTUPLE) set_property(TEST read_rntuple PROPERTY DEPENDS write_rntuple) endif() -add_test(NAME check_benchmark_outputs COMMAND check_benchmark_outputs write_benchmark_root.root read_benchmark_root.root) -set_property(TEST check_benchmark_outputs PROPERTY DEPENDS read_timed write_timed) - -message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") -add_executable(read-legacy-files-root read-legacy-files-root.cpp) -target_link_libraries(read-legacy-files-root PRIVATE TestDataModel TestDataModelDict podio::podioRootIO) - -# Add a legacy test case based on a base executable and a version for which an -# input file exists -macro(ADD_PODIO_LEGACY_TEST version base_test input_file) - ExternalData_Add_Test(legacy_test_cases - NAME ${base_test}_${version} - COMMAND ${base_test} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} - ) - set_property(TEST ${base_test}_${version} PROPERTY ENVIRONMENT - LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$ENV{LD_LIBRARY_PATH} - # Clear the ROOT_INCLUDE_PATH for the tests, to avoid potential conflicts - # with existing headers from other installations - ROOT_INCLUDE_PATH= - ) - -endmacro() - -ADD_PODIO_LEGACY_TEST(v00-13 read-legacy-files-root v00-13-example.root legacy_test_cases) +add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) +target_link_libraries(read_frame_legacy_root PRIVATE "${root_libs}") -foreach(version IN LISTS legacy_test_versions) - ADD_PODIO_LEGACY_TEST(${version} read-legacy-files-root ${version}-example.root legacy_test_cases) - ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root legacy_test_cases) - ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root legacy_test_cases) +foreach(version IN LISTS root_legacy_test_versions) + ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root) + ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root) endforeach() #--- Write via python and the ROOT backend and see if we can read it back in in diff --git a/tests/root_io/read-legacy-files-root.cpp b/tests/root_io/read-legacy-files-root.cpp deleted file mode 100644 index b0c7cee1c..000000000 --- a/tests/root_io/read-legacy-files-root.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -#include - -int main(int argc, char* argv[]) { - if (argc < 2) { - std::cerr << "Usage: read-legacy-files inputfile" << std::endl; - return 1; - } - - auto reader = podio::ROOTReader(); - reader.openFile(argv[1]); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read-multiple.cpp b/tests/root_io/read-multiple.cpp deleted file mode 100644 index e51743006..000000000 --- a/tests/root_io/read-multiple.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -int main() { - auto reader = podio::ROOTReader(); - reader.openFiles({"example.root", "example1.root"}); - - run_read_test(reader); - - reader.closeFiles(); - return 0; -} diff --git a/tests/root_io/read.cpp b/tests/root_io/read.cpp deleted file mode 100644 index d21ad81b0..000000000 --- a/tests/root_io/read.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -int main() { - auto reader = podio::ROOTReader(); - try { - reader.openFile("example.root"); - } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; - return 1; - } - - if (reader.currentFileVersion() != podio::version::build_version) { - return 1; - } - - run_read_test(reader); - - // jump back and forth a bit - run_read_test_event(reader, 10); - run_read_test_event(reader, 150); - run_read_test_event(reader, 120); - run_read_test_event(reader, 0); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read_and_write.cpp b/tests/root_io/read_and_write.cpp deleted file mode 100644 index 647eceadf..000000000 --- a/tests/root_io/read_and_write.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "read_test.h" -// podio specific includes -#include "podio/EventStore.h" -#include "podio/ROOTReader.h" -#include "podio/ROOTWriter.h" - -int main() { - auto reader = podio::ROOTReader(); - auto store = podio::EventStore(); - reader.openFile("example.root"); - store.setReader(&reader); - - // test writing a subset of the event to a new file - auto writer = podio::ROOTWriter("example_copy.root", &store); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - - unsigned nEvents = reader.getEntries(); - for (unsigned i = 0; i < nEvents; ++i) { - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - processEvent(store, i, reader.currentFileVersion()); - - writer.writeEvent(); - - store.clear(); - reader.endOfEvent(); - } - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read_frame_legacy_root.cpp b/tests/root_io/read_frame_legacy_root.cpp index f6c410220..6d98f79ac 100644 --- a/tests/root_io/read_frame_legacy_root.cpp +++ b/tests/root_io/read_frame_legacy_root.cpp @@ -6,25 +6,17 @@ #include int main(int argc, char* argv[]) { - std::string inputFile = "example.root"; - bool assertBuildVersion = true; - if (argc == 2) { - inputFile = argv[1]; - assertBuildVersion = false; + if (argc != 2) { + std::cerr << "usage: read_frame_legacy_frame inputfile" << std::endl; + return 1; } + const std::string inputFile = argv[1]; auto reader = podio::ROOTLegacyReader(); try { reader.openFile(inputFile); } catch (const std::runtime_error& e) { - std::cout << "File (" << inputFile << ")could not be opened, aborting." << std::endl; - return 1; - } - - if (assertBuildVersion && reader.currentFileVersion() != podio::version::build_version) { - std::cerr << "The podio build version could not be read back correctly. " - << "(expected:" << podio::version::build_version << ", actual: " << reader.currentFileVersion() << ")" - << std::endl; + std::cout << "File (" << inputFile << ") could not be opened, aborting." << std::endl; return 1; } diff --git a/tests/root_io/read_timed.cpp b/tests/root_io/read_timed.cpp deleted file mode 100644 index 30beaa885..000000000 --- a/tests/root_io/read_timed.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/ROOTReader.h" -#include "podio/TimedReader.h" -#include "read_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("read_benchmark_root.root"); - - podio::TimedReader reader(recorder); - reader.openFile("example_timed.root"); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/write.cpp b/tests/root_io/write.cpp deleted file mode 100644 index 00c99b8c5..000000000 --- a/tests/root_io/write.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "podio/EventStore.h" -#include "podio/ROOTWriter.h" -#include "write_test.h" - -int main(int, char**) { - auto store = podio::EventStore(); - podio::ROOTWriter writer("example.root", &store); - write(store, writer); - - // start from a clean slate for the second file - auto store2 = podio::EventStore(); - auto writer2 = podio::ROOTWriter("example1.root", &store2); - write(store2, writer2); -} diff --git a/tests/root_io/write_timed.cpp b/tests/root_io/write_timed.cpp deleted file mode 100644 index 8e2852c73..000000000 --- a/tests/root_io/write_timed.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/ROOTWriter.h" -#include "podio/TimedWriter.h" -#include "write_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("write_benchmark_root.root"); - podio::EventStore store; - podio::TimedWriter writer(recorder, "example_timed.root", &store); - - write(store, writer); - - return 0; -} diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index 2d24ce2f9..b3987ba5a 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -1,12 +1,6 @@ set(sio_dependent_tests - write_sio.cpp - read_sio.cpp - read_and_write_sio.cpp - write_timed_sio.cpp - read_timed_sio.cpp read_frame_sio.cpp write_frame_sio.cpp - read_frame_legacy_sio.cpp read_and_write_frame_sio.cpp read_python_frame_sio.cpp ) @@ -15,17 +9,6 @@ foreach( sourcefile ${sio_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${sio_libs}") endforeach() -# These need to be linked against TTree explicitly, since it is not done -# through another library and the TimedReader/Writer decorators are -# header-only wrappers -target_link_libraries(write_timed_sio PRIVATE ROOT::Tree) -target_link_libraries(read_timed_sio PRIVATE ROOT::Tree) - -set_property(TEST read_sio PROPERTY DEPENDS write_sio) -set_property(TEST read_and_write_sio PROPERTY DEPENDS write_sio) -set_property(TEST read_timed_sio PROPERTY DEPENDS write_timed_sio) -set_property(TEST read_frame_legacy_sio PROPERTY DEPENDS write_sio) - set_tests_properties( read_frame_sio read_and_write_frame_sio @@ -35,11 +18,16 @@ set_tests_properties( write_frame_sio ) -add_test(NAME check_benchmark_outputs_sio COMMAND check_benchmark_outputs write_benchmark_sio.root read_benchmark_sio.root) -set_property(TEST check_benchmark_outputs_sio PROPERTY DEPENDS read_timed_sio write_timed_sio) - #--- Write via python and the SIO backend and see if we can read it back in in #--- c++ add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio sio_io.Writer) PODIO_SET_TEST_ENV(write_python_frame_sio) set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio) + +add_executable(read_frame_legacy_sio read_frame_legacy_sio.cpp) +target_link_libraries(read_frame_legacy_sio PRIVATE "${sio_libs}" TestDataModel) + +foreach(version IN LISTS sio_legacy_test_versions) + ADD_PODIO_LEGACY_TEST(${version} read_frame_sio ${version}-example_frame.sio) + ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_sio ${version}-example.sio) +endforeach() diff --git a/tests/sio_io/read_and_write_sio.cpp b/tests/sio_io/read_and_write_sio.cpp deleted file mode 100644 index 69f25d270..000000000 --- a/tests/sio_io/read_and_write_sio.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "read_test.h" - -#include "podio/EventStore.h" -#include "podio/SIOReader.h" -#include "podio/SIOWriter.h" - -int main() { - podio::SIOReader reader; - reader.openFile("example.sio"); - - auto store = podio::EventStore(); - store.setReader(&reader); - - auto writer = podio::SIOWriter("example_copy.sio", &store); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - - unsigned nEvents = reader.getEntries(); - for (unsigned i = 0; i < nEvents; ++i) { - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - processEvent(store, i, reader.currentFileVersion()); - - writer.writeEvent(); - - store.clear(); - reader.endOfEvent(); - } - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/read_frame_legacy_sio.cpp b/tests/sio_io/read_frame_legacy_sio.cpp index b8fbf009f..5dbc123d3 100644 --- a/tests/sio_io/read_frame_legacy_sio.cpp +++ b/tests/sio_io/read_frame_legacy_sio.cpp @@ -5,19 +5,18 @@ #include -int main() { - auto reader = podio::SIOLegacyReader(); - try { - reader.openFile("example.sio"); - } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cerr << "usage: read_frame_legacy_sio inputfile" << std::endl; return 1; } - if (reader.currentFileVersion() != podio::version::build_version) { - std::cerr << "The podio build version could not be read back correctly. " - << "(expected:" << podio::version::build_version << ", actual: " << reader.currentFileVersion() << ")" - << std::endl; + const auto inputFile = argv[1]; + auto reader = podio::SIOLegacyReader(); + try { + reader.openFile(inputFile); + } catch (const std::runtime_error& e) { + std::cout << "File (" << inputFile << ") could not be opened, aborting." << std::endl; return 1; } diff --git a/tests/sio_io/read_frame_sio.cpp b/tests/sio_io/read_frame_sio.cpp index 5f69d4da1..625a7748e 100644 --- a/tests/sio_io/read_frame_sio.cpp +++ b/tests/sio_io/read_frame_sio.cpp @@ -3,7 +3,14 @@ #include "podio/SIOFrameReader.h" -int main() { - return read_frames("example_frame.sio") + - test_frame_aux_info("example_frame.sio"); +int main(int argc, char* argv[]) { + std::string inputFile = "example_frame.sio"; + bool assertBuildVersion = true; + if (argc == 2) { + inputFile = argv[1]; + assertBuildVersion = false; + } + + return read_frames(inputFile, assertBuildVersion) + + test_frame_aux_info(inputFile); } diff --git a/tests/sio_io/read_sio.cpp b/tests/sio_io/read_sio.cpp deleted file mode 100644 index 85ef72332..000000000 --- a/tests/sio_io/read_sio.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "podio/SIOReader.h" -#include "read_test.h" - -int main() { - // auto reader = podio::SIOReader(); - podio::SIOReader reader; // SIOReader has no copy c'tor ... - try { - reader.openFile("example.sio"); - } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; - return 1; - } - - if (reader.currentFileVersion() != podio::version::build_version) { - return 1; - } - - run_read_test(reader); - - // jump back and forth a bit - run_read_test_event(reader, 10); - run_read_test_event(reader, 150); - run_read_test_event(reader, 120); - run_read_test_event(reader, 0); - - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/read_timed_sio.cpp b/tests/sio_io/read_timed_sio.cpp deleted file mode 100644 index c109f0bee..000000000 --- a/tests/sio_io/read_timed_sio.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/SIOReader.h" -#include "podio/TimedReader.h" -#include "read_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("read_benchmark_sio.root"); - - podio::TimedReader reader(recorder); - reader.openFile("example_timed.sio"); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/write_sio.cpp b/tests/sio_io/write_sio.cpp deleted file mode 100644 index 9068f312c..000000000 --- a/tests/sio_io/write_sio.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "podio/EventStore.h" -#include "podio/SIOWriter.h" -#include "write_test.h" - -int main(int, char**) { - podio::EventStore store; - podio::SIOWriter writer("example.sio", &store); - - write(store, writer); -} diff --git a/tests/sio_io/write_timed_sio.cpp b/tests/sio_io/write_timed_sio.cpp deleted file mode 100644 index 6edb8b771..000000000 --- a/tests/sio_io/write_timed_sio.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/EventStore.h" -#include "podio/SIOWriter.h" -#include "podio/TimedWriter.h" -#include "write_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("write_benchmark_sio.root"); - podio::EventStore store; - podio::TimedWriter writer(recorder, "example_timed.sio", &store); - - write(store, writer); - - return 0; -} diff --git a/tests/unittests/frame.cpp b/tests/unittests/frame.cpp index cc6be08e1..623b65a56 100644 --- a/tests/unittests/frame.cpp +++ b/tests/unittests/frame.cpp @@ -22,6 +22,8 @@ TEST_CASE("Frame collections", "[frame][basics]") { auto& coll = event.get("clusters"); REQUIRE(coll[0].energy() == 3.14f); REQUIRE(coll[1].energy() == 42.0f); + + REQUIRE_FALSE(event.get("non-existant")); } TEST_CASE("Frame parameters", "[frame][basics]") { diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index 35c936dae..1be4b89c5 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -12,14 +12,11 @@ #include "catch2/matchers/catch_matchers_vector.hpp" // podio specific includes -#include "datamodel/MutableExampleHit.h" -#include "podio/EventStore.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/ROOTFrameReader.h" #include "podio/ROOTFrameWriter.h" #include "podio/ROOTLegacyReader.h" -#include "podio/ROOTReader.h" #include "podio/podioVersion.h" #ifndef PODIO_ENABLE_SIO @@ -28,7 +25,6 @@ #if PODIO_ENABLE_SIO #include "podio/SIOFrameReader.h" #include "podio/SIOLegacyReader.h" - #include "podio/SIOReader.h" #endif #if PODIO_ENABLE_RNTUPLE @@ -52,29 +48,14 @@ #include "podio/UserDataCollection.h" TEST_CASE("AutoDelete", "[basics][memory-management]") { - auto store = podio::EventStore(); + auto coll = EventInfoCollection(); auto hit1 = MutableEventInfo(); auto hit2 = MutableEventInfo(); auto hit3 = MutableEventInfo(); - auto& coll = store.create("info"); coll.push_back(hit1); hit3 = hit2; } -TEST_CASE("Basics", "[basics][memory-management]") { - auto store = podio::EventStore(); - // Adding - auto& collection = store.create("name"); - auto hit1 = collection.create(0xcaffeeULL, 0., 0., 0., 0.); // initialize w/ value - auto hit2 = collection.create(); // default initialize - hit2.energy(12.5); - // Retrieving - const ExampleHitCollection* coll2(nullptr); - REQUIRE(store.get("name", coll2)); - const ExampleHitCollection* coll3(nullptr); - REQUIRE_FALSE(store.get("wrongName", coll3)); -} - TEST_CASE("Assignment-operator ref count", "[basics][memory-management]") { // Make sure that the assignment operator handles the reference count // correctly. (Will trigger in an ASan build if it is not the case) @@ -101,31 +82,6 @@ TEST_CASE("ostream-operator", "[basics]") { REQUIRE(sstr.str() == "[not available]"); } -TEST_CASE("Clearing", "[UBSAN-FAIL][ASAN-FAIL][THREAD-FAIL][basics][memory-management]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); - auto& clusters = store.create("clusters"); - auto& oneRels = store.create("OneRelation"); - auto nevents = unsigned(1000); - for (unsigned i = 0; i < nevents; ++i) { - hits.clear(); - clusters.clear(); - auto hit1 = hits.create(); - auto hit2 = MutableExampleHit(); - hit1.energy(double(i)); - auto cluster = clusters.create(); - cluster.addHits(hit1); - cluster.addHits(hit2); - hits.push_back(hit2); - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - } - hits.clear(); - REQUIRE(hits.empty()); -} - TEST_CASE("Cloning", "[basics][memory-management]") { bool success = true; auto hit = MutableExampleHit(); @@ -205,11 +161,10 @@ TEST_CASE("Container lifetime", "[basics][memory-management]") { } TEST_CASE("Invalid_refs", "[basics][relations]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); auto hit1 = hits.create(0xcaffeeULL, 0., 0., 0., 0.); - auto hit2 = MutableExampleHit(); - auto& clusters = store.create("clusters"); + auto hit2 = ExampleHit(); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); cluster.addHits(hit1); cluster.addHits(hit2); @@ -217,8 +172,7 @@ TEST_CASE("Invalid_refs", "[basics][relations]") { } TEST_CASE("Looping", "[basics]") { - auto store = podio::EventStore(); - auto& coll = store.create("name"); + auto coll = ExampleHitCollection(); auto hit1 = coll.create(0xbadULL, 0., 0., 0., 0.); auto hit2 = coll.create(0xcaffeeULL, 1., 1., 1., 1.); for (auto&& i : coll) { @@ -235,7 +189,7 @@ TEST_CASE("Looping", "[basics]") { REQUIRE(coll[0].energy() == 0); REQUIRE(coll[1].energy() == 1); - auto& constColl = store.get("name"); + const auto& constColl = coll; int index = 0; for (auto hit : constColl) { auto energy = hit.energy(); @@ -244,8 +198,7 @@ TEST_CASE("Looping", "[basics]") { } TEST_CASE("Notebook", "[basics]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); for (unsigned i = 0; i < 12; ++i) { auto hit = hits.create(0xcaffeeULL, 0., 0., 0., double(i)); } @@ -285,11 +238,10 @@ TEST_CASE("Podness", "[basics][code-gen]") { } TEST_CASE("Referencing", "[basics][relations]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); auto hit1 = hits.create(0x42ULL, 0., 0., 0., 0.); auto hit2 = hits.create(0x42ULL, 1., 1., 1., 1.); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); cluster.addHits(hit1); cluster.addHits(hit2); @@ -302,8 +254,7 @@ TEST_CASE("Referencing", "[basics][relations]") { TEST_CASE("VariadicCreate", "[basics]") { // Test that objects created via the variadic create template function handle relations correctly - auto store = podio::EventStore(); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto variadic_cluster = clusters.create(3.14f); auto normal_cluster = clusters.create(); @@ -315,11 +266,10 @@ TEST_CASE("VariadicCreate", "[basics]") { } TEST_CASE("write_buffer", "[basics][io]") { - auto store = podio::EventStore(); - auto& coll = store.create("data"); + auto coll = ExampleHitCollection(); auto hit1 = coll.create(0x42ULL, 0., 0., 0., 0.); auto hit2 = coll.create(0x42ULL, 1., 1., 1., 1.); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); // add a few related objects to also exercise relation writing cluster.addHits(hit1); @@ -333,7 +283,7 @@ TEST_CASE("write_buffer", "[basics][io]") { REQUIRE_NOTHROW(clusters.prepareForWrite()); REQUIRE(clusters.getBuffers().data == buffers.data); - auto& ref_coll = store.create("onerel"); + auto ref_coll = ExampleWithOneRelationCollection(); auto withRef = ref_coll.create(); REQUIRE_NOTHROW(ref_coll.prepareForWrite()); } @@ -514,11 +464,6 @@ TEST_CASE("UserInitialization", "[basics][code-gen]") { REQUIRE(ex.comp().arr[1] == 3.4); } -TEST_CASE("NonPresentCollection", "[basics][event-store]") { - auto store = podio::EventStore(); - REQUIRE_THROWS_AS(store.get("NonPresentCollection"), std::runtime_error); -} - TEST_CASE("Collection size and empty", "[basics][collections]") { ExampleClusterCollection coll{}; REQUIRE(coll.empty()); @@ -538,11 +483,7 @@ TEST_CASE("const correct indexed access to const collections", "[const-correctne } TEST_CASE("const correct indexed access to collections", "[const-correctness]") { - auto store = podio::EventStore(); - auto& collection = store.create("irrelevant name"); - - STATIC_REQUIRE(std::is_same_v); // collection created by store should not - // be const + auto collection = ExampleHitCollection(); STATIC_REQUIRE(std::is_same_v); // non-const collections should have // indexed access to mutable objects @@ -1155,9 +1096,6 @@ TEST_CASE("GenericParameters return types", "[generic-parameters][static-checks] } TEST_CASE("Missing files (ROOT readers)", "[basics]") { - auto root_reader = podio::ROOTReader(); - REQUIRE_THROWS_AS(root_reader.openFile("NonExistentFile.root"), std::runtime_error); - auto root_legacy_reader = podio::ROOTLegacyReader(); REQUIRE_THROWS_AS(root_legacy_reader.openFile("NonExistentFile.root"), std::runtime_error); @@ -1167,9 +1105,6 @@ TEST_CASE("Missing files (ROOT readers)", "[basics]") { #if PODIO_ENABLE_SIO TEST_CASE("Missing files (SIO readers)", "[basics]") { - auto sio_reader = podio::SIOReader(); - REQUIRE_THROWS_AS(sio_reader.openFile("NonExistentFile.sio"), std::runtime_error); - auto sio_legacy_reader = podio::SIOLegacyReader(); REQUIRE_THROWS_AS(sio_legacy_reader.openFile("NonExistentFile.sio"), std::runtime_error); diff --git a/tests/write_ascii.cpp b/tests/write_ascii.cpp deleted file mode 100644 index bc9dbe9c6..000000000 --- a/tests/write_ascii.cpp +++ /dev/null @@ -1,200 +0,0 @@ -// Data model -#include "datamodel/EventInfoCollection.h" -#include "datamodel/ExampleClusterCollection.h" -#include "datamodel/ExampleHitCollection.h" -#include "datamodel/ExampleMCCollection.h" -#include "datamodel/ExampleReferencingTypeCollection.h" -#include "datamodel/ExampleWithARelationCollection.h" -#include "datamodel/ExampleWithComponentCollection.h" -#include "datamodel/ExampleWithNamespaceCollection.h" -#include "datamodel/ExampleWithOneRelationCollection.h" -#include "datamodel/ExampleWithVectorMemberCollection.h" - -// STL -#include -#include - -// podio specific includes -#include "podio/ASCIIWriter.h" -#include "podio/EventStore.h" - -int main() { - - std::cout << "start processing" << std::endl; - - auto store = podio::EventStore(); - auto writer = podio::ASCIIWriter("example.txt", &store); - - auto& info = store.create("info"); - auto& mcps = store.create("mcparticles"); - auto& hits = store.create("hits"); - auto& clusters = store.create("clusters"); - auto& refs = store.create("refs"); - auto& refs2 = store.create("refs2"); - auto& comps = store.create("Component"); - auto& oneRels = store.create("OneRelation"); - auto& vecs = store.create("WithVectorMember"); - auto& namesps = store.create("WithNamespaceMember"); - auto& namesprels = store.create("WithNamespaceRelation"); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - writer.registerForWrite("refs"); - writer.registerForWrite("refs2"); - writer.registerForWrite("Component"); - writer.registerForWrite("OneRelation"); - writer.registerForWrite("WithVectorMember"); - writer.registerForWrite("WithNamespaceMember"); - writer.registerForWrite("WithNamespaceRelation"); - - unsigned nevents = 1; // 2000; - - for (unsigned i = 0; i < nevents; ++i) { - if (i % 1000 == 0) { - std::cout << "processing event " << i << std::endl; - } - - auto item1 = MutableEventInfo(); - item1.Number(i); - info.push_back(item1); - auto hit1 = MutableExampleHit(0xbad, 0., 0., 0., 23. + i); - auto hit2 = MutableExampleHit(0xcaffee, 1., 0., 0., 12. + i); - - hits.push_back(hit1); - hits.push_back(hit2); - - // ---- add some MC particles ---- - auto mcp0 = MutableExampleMC(); - auto mcp1 = MutableExampleMC(); - auto mcp2 = MutableExampleMC(); - auto mcp3 = MutableExampleMC(); - auto mcp4 = MutableExampleMC(); - auto mcp5 = MutableExampleMC(); - auto mcp6 = MutableExampleMC(); - auto mcp7 = MutableExampleMC(); - auto mcp8 = MutableExampleMC(); - auto mcp9 = MutableExampleMC(); - - mcps.push_back(mcp0); - mcps.push_back(mcp1); - mcps.push_back(mcp2); - mcps.push_back(mcp3); - mcps.push_back(mcp4); - mcps.push_back(mcp5); - mcps.push_back(mcp6); - mcps.push_back(mcp7); - mcps.push_back(mcp8); - mcps.push_back(mcp9); - - // --- add some daughter relations - - auto p = mcps[0]; - p.adddaughters(mcps[2]); - p.adddaughters(mcps[3]); - p.adddaughters(mcps[4]); - p.adddaughters(mcps[5]); - p = mcps[1]; - p.adddaughters(mcps[2]); - p.adddaughters(mcps[3]); - p.adddaughters(mcps[4]); - p.adddaughters(mcps[5]); - p = mcps[2]; - p.adddaughters(mcps[6]); - p.adddaughters(mcps[7]); - p.adddaughters(mcps[8]); - p.adddaughters(mcps[9]); - p = mcps[3]; - p.adddaughters(mcps[6]); - p.adddaughters(mcps[7]); - p.adddaughters(mcps[8]); - p.adddaughters(mcps[9]); - - //--- now fix the parent relations - for (unsigned j = 0, N = mcps.size(); j < N; ++j) { - p = mcps[j]; - for (auto it = p.daughters_begin(), end = p.daughters_end(); it != end; ++it) { - int dIndex = it->getObjectID().index; - auto d = mcps[dIndex]; - d.addparents(p); - } - } - //-------- print relations for debugging: - for (auto _p : mcps) { - std::cout << " particle " << _p.getObjectID().index << " has daughters: "; - for (auto it = _p.daughters_begin(), end = _p.daughters_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = _p.parents_begin(), end = _p.parents_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << std::endl; - } - //------------------------------- - - auto cluster = MutableExampleCluster(); - auto clu0 = MutableExampleCluster(); - auto clu1 = MutableExampleCluster(); - - clu0.addHits(hit1); - clu0.energy(hit1.energy()); - clu1.addHits(hit2); - clu1.energy(hit2.energy()); - cluster.addHits(hit1); - cluster.addHits(hit2); - cluster.energy(hit1.energy() + hit2.energy()); - cluster.addClusters(clu0); - cluster.addClusters(clu1); - - clusters.push_back(clu0); - clusters.push_back(clu1); - clusters.push_back(cluster); - - auto ref = MutableExampleReferencingType(); - refs.push_back(ref); - - auto ref2 = MutableExampleReferencingType(); - refs2.push_back(ref2); - - ref.addClusters(cluster); - ref.addRefs(ref2); - - auto comp = MutableExampleWithComponent(); - comp.component().data.x = 0; - comp.component().data.y = 1; - comp.component().data.z = i; - comps.push_back(comp); - - auto cyclic = MutableExampleReferencingType(); - cyclic.addRefs(cyclic); - refs.push_back(cyclic); - - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - - // write non-filled relation - auto oneRelEmpty = MutableExampleWithOneRelation(); - oneRels.push_back(oneRelEmpty); - - auto vec = MutableExampleWithVectorMember(); - vec.addcount(23); - vec.addcount(24); - vecs.push_back(vec); - - auto namesp = ex42::MutableExampleWithNamespace(); - namesp.component().x = 1; - namesp.component().y = i; - namesps.push_back(namesp); - - auto rel = ex42::MutableExampleWithARelation(); - rel.ref(namesp); - namesprels.push_back(rel); - - writer.writeEvent(); - store.clearCollections(); - } - - writer.finish(); -} diff --git a/tests/write_test.h b/tests/write_test.h deleted file mode 100644 index 492fb049b..000000000 --- a/tests/write_test.h +++ /dev/null @@ -1,346 +0,0 @@ -#ifndef PODIO_TESTS_WRITE_TEST_H // NOLINT(llvm-header-guard): folder structure not suitable -#define PODIO_TESTS_WRITE_TEST_H // NOLINT(llvm-header-guard): folder structure not suitable - -// Data model -#include "datamodel/EventInfoCollection.h" -#include "datamodel/ExampleClusterCollection.h" -#include "datamodel/ExampleHitCollection.h" -#include "datamodel/ExampleMCCollection.h" -#include "datamodel/ExampleReferencingTypeCollection.h" -#include "datamodel/ExampleWithARelationCollection.h" -#include "datamodel/ExampleWithArrayCollection.h" -#include "datamodel/ExampleWithComponentCollection.h" -#include "datamodel/ExampleWithFixedWidthIntegersCollection.h" -#include "datamodel/ExampleWithNamespaceCollection.h" -#include "datamodel/ExampleWithOneRelationCollection.h" -#include "datamodel/ExampleWithVectorMemberCollection.h" - -#include "podio/EventStore.h" -#include "podio/UserDataCollection.h" - -// STL -#include -#include -#include -#include - -template -void write(podio::EventStore& store, WriterT& writer) { - std::cout << "start processing" << std::endl; - - auto& info = store.create("info"); - auto& mcps = store.create("mcparticles"); - auto& moreMCs = store.create("moreMCs"); - auto& mcpsRefs = store.create("mcParticleRefs"); - mcpsRefs.setSubsetCollection(); - auto& hits = store.create("hits"); - auto& hitRefs = store.create("hitRefs"); - hitRefs.setSubsetCollection(); - auto& clusters = store.create("clusters"); - auto& refs = store.create("refs"); - auto& refs2 = store.create("refs2"); - auto& comps = store.create("Component"); - auto& oneRels = store.create("OneRelation"); - auto& vecs = store.create("WithVectorMember"); - auto& namesps = store.create("WithNamespaceMember"); - auto& namesprels = store.create("WithNamespaceRelation"); - auto& cpytest = store.create("WithNamespaceRelationCopy"); - auto& arrays = store.create("arrays"); - auto& fixedWidthInts = store.create("fixedWidthInts"); - auto& usrInts = store.create>("userInts"); - auto& usrDoubles = store.create>("userDoubles"); - - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("moreMCs"); - writer.registerForWrite("mcParticleRefs"); - writer.registerForWrite("hits"); - writer.registerForWrite("hitRefs"); - writer.registerForWrite("clusters"); - writer.registerForWrite("refs"); - writer.registerForWrite("refs2"); - writer.registerForWrite("Component"); - writer.registerForWrite("OneRelation"); - writer.registerForWrite("WithVectorMember"); - writer.registerForWrite("WithNamespaceMember"); - writer.registerForWrite("WithNamespaceRelation"); - writer.registerForWrite("WithNamespaceRelationCopy"); - writer.registerForWrite("arrays"); - writer.registerForWrite("fixedWidthInts"); - writer.registerForWrite("userInts"); - writer.registerForWrite("userDoubles"); - - unsigned nevents = 2000; - - for (unsigned i = 0; i < nevents; ++i) { - if (i % 1000 == 0) { - std::cout << "processing event " << i << std::endl; - } - - auto item1 = MutableEventInfo(); - item1.Number(i); - info.push_back(item1); - - auto& evtMD = store.getEventMetaData(); - evtMD.setValue("UserEventWeight", (float)100. * i); - std::stringstream ss; - ss << " event_number_" << i; - evtMD.setValue("UserEventName", ss.str()); - evtMD.setValue("SomeVectorData", {1, 2, 3, 4}); - evtMD.setValue("SomeVectorData", {i * 1.1, i * 2.2}); - - auto& colMD = store.getCollectionMetaData(hits.getID()); - colMD.setValue("CellIDEncodingString", "system:8,barrel:3,layer:6,slice:5,x:-16,y:-16"); - - auto hit1 = MutableExampleHit(0xbad, 0., 0., 0., 23. + i); - auto hit2 = MutableExampleHit(0xcaffee, 1., 0., 0., 12. + i); - - hits.push_back(hit1); - hits.push_back(hit2); - - hitRefs.push_back(hit2); - hitRefs.push_back(hit1); - - // ---- add some MC particles ---- - auto mcp0 = MutableExampleMC(); - auto mcp1 = MutableExampleMC(); - auto mcp2 = MutableExampleMC(); - auto mcp3 = MutableExampleMC(); - auto mcp4 = MutableExampleMC(); - auto mcp5 = MutableExampleMC(); - auto mcp6 = MutableExampleMC(); - auto mcp7 = MutableExampleMC(); - auto mcp8 = MutableExampleMC(); - auto mcp9 = MutableExampleMC(); - - mcps.push_back(mcp0); - mcps.push_back(mcp1); - mcps.push_back(mcp2); - mcps.push_back(mcp3); - mcps.push_back(mcp4); - mcps.push_back(mcp5); - mcps.push_back(mcp6); - mcps.push_back(mcp7); - mcps.push_back(mcp8); - mcps.push_back(mcp9); - - auto mcp = mcps[0]; - mcp.adddaughters(mcps[2]); - mcp.adddaughters(mcps[3]); - mcp.adddaughters(mcps[4]); - mcp.adddaughters(mcps[5]); - mcp = mcps[1]; - mcp.adddaughters(mcps[2]); - mcp.adddaughters(mcps[3]); - mcp.adddaughters(mcps[4]); - mcp.adddaughters(mcps[5]); - mcp = mcps[2]; - mcp.adddaughters(mcps[6]); - mcp.adddaughters(mcps[7]); - mcp.adddaughters(mcps[8]); - mcp.adddaughters(mcps[9]); - mcp = mcps[3]; - mcp.adddaughters(mcps[6]); - mcp.adddaughters(mcps[7]); - mcp.adddaughters(mcps[8]); - mcp.adddaughters(mcps[9]); - - //--- now fix the parent relations - // use a range-based for loop here to see if we get mutable objects from the - // begin/end iterators - for (auto mc : mcps) { - for (auto p : mc.daughters()) { - int dIndex = p.getObjectID().index; - auto d = mcps[dIndex]; - d.addparents(p); - } - } - //-------- print relations for debugging: - for (auto p : mcps) { - std::cout << " particle " << p.getObjectID().index << " has daughters: "; - for (auto it = p.daughters_begin(), end = p.daughters_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = p.parents_begin(), end = p.parents_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << std::endl; - - // make sure that this does not crash when we do it on an immutable object - ExampleMC constP{p}; - std::cout << "The const particle still has the same relations: daughters: "; - for (auto it = constP.daughters_begin(); it != constP.daughters_end(); ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = constP.parents_begin(); it != constP.parents_end(); ++it) { - std::cout << " " << it->getObjectID().index; - } - } - //------------------------------- - - // ----------------- create a second MC collection ----------------- - // Can use it to test subset collections that store elements from multiple - // collections - for (const auto&& mc : mcps) { - moreMCs.push_back(mc.clone()); - } - - // ----------------- add all "odd" mc particles into a subset collection - for (auto p : mcps) { - if (p.id().index % 2) { - mcpsRefs.push_back(p); - } - } - // ----------------- add the "even" counterparts from a different collection - for (auto p : moreMCs) { - if (p.id().index % 2 == 0) { - mcpsRefs.push_back(p); - } - } - - if (mcpsRefs.size() != mcps.size()) { - throw std::runtime_error( - "The mcParticleRefs collection should now contain as many elements as the mcparticles collection"); - } - //------------------------------- - - auto cluster = MutableExampleCluster(); - auto clu0 = MutableExampleCluster(); - auto clu1 = MutableExampleCluster(); - - clu0.addHits(hit1); - clu0.energy(hit1.energy()); - clu1.addHits(hit2); - clu1.energy(hit2.energy()); - cluster.addHits(hit1); - cluster.addHits(hit2); - cluster.energy(hit1.energy() + hit2.energy()); - cluster.addClusters(clu0); - cluster.addClusters(clu1); - - clusters.push_back(clu0); - clusters.push_back(clu1); - clusters.push_back(cluster); - - auto ref = MutableExampleReferencingType(); - refs.push_back(ref); - - auto ref2 = MutableExampleReferencingType(); - refs2.push_back(ref2); - - ref.addClusters(cluster); - ref.addRefs(ref2); - - auto comp = MutableExampleWithComponent(); - comp.component().data.x = 0; - comp.component().data.y = 1; - comp.component().data.z = i; - comps.push_back(comp); - - auto cyclic = MutableExampleReferencingType(); - cyclic.addRefs(cyclic); - refs.push_back(cyclic); - - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - - // write non-filled relation - auto oneRelEmpty = MutableExampleWithOneRelation(); - oneRels.push_back(oneRelEmpty); - - auto vec = MutableExampleWithVectorMember(); - vec.addcount(i); - vec.addcount(i + 10); - vecs.push_back(vec); - auto vec1 = MutableExampleWithVectorMember(); - vec1.addcount(i + 1); - vec1.addcount(i + 11); - vecs.push_back(vec1); - - for (int j = 0; j < 5; j++) { - auto rel = ex42::MutableExampleWithARelation(); - rel.number(0.5 * j); - auto exWithNamesp = ex42::MutableExampleWithNamespace(); - exWithNamesp.component().x = i; - exWithNamesp.component().y = 1000 * i; - namesps.push_back(exWithNamesp); - if (j != 3) { // also check for empty relations - rel.ref(exWithNamesp); - for (int k = 0; k < 5; k++) { - auto namesp = ex42::MutableExampleWithNamespace(); - namesp.x(3 * k); - namesp.component().y = k; - namesps.push_back(namesp); - rel.addrefs(namesp); - } - } - namesprels.push_back(rel); - } - for (auto&& namesprel : namesprels) { - cpytest.push_back(namesprel.clone()); - } - - std::array arrayTest = {0, 0, 2, 3}; - std::array arrayTest2 = {4, 4, 2 * static_cast(i)}; - NotSoSimpleStruct a; - a.data.p = arrayTest2; - ex2::NamespaceStruct nstruct; - nstruct.x = static_cast(i); - std::array structArrayTest = {nstruct, nstruct, nstruct, nstruct}; - auto array = MutableExampleWithArray(a, arrayTest, arrayTest, arrayTest, arrayTest, structArrayTest); - array.myArray(1, i); - array.arrayStruct(a); - arrays.push_back(array); - - auto maxValues = fixedWidthInts.create(); - maxValues.fixedI16(std::numeric_limits::max()); // 2^(16 - 1) - 1 == 32767 - maxValues.fixedU32(std::numeric_limits::max()); // 2^32 - 1 == 4294967295 - maxValues.fixedU64(std::numeric_limits::max()); // 2^64 - 1 == 18446744073709551615 - auto& maxComp = maxValues.fixedWidthStruct(); - maxComp.fixedUnsigned16 = std::numeric_limits::max(); // 2^16 - 1 == 65535 - maxComp.fixedInteger64 = std::numeric_limits::max(); // 2^(64 -1) - 1 == 9223372036854775807 - maxComp.fixedInteger32 = std::numeric_limits::max(); // 2^(32 - 1) - 1 == 2147483647 - - auto minValues = fixedWidthInts.create(); - minValues.fixedI16(std::numeric_limits::min()); // -2^(16 - 1) == -32768 - minValues.fixedU32(std::numeric_limits::min()); // 0 - minValues.fixedU64(std::numeric_limits::min()); // 0 - auto& minComp = minValues.fixedWidthStruct(); - minComp.fixedUnsigned16 = std::numeric_limits::min(); // 0 - minComp.fixedInteger64 = std::numeric_limits::min(); // -2^(64 - 1) == -9223372036854775808 - minComp.fixedInteger32 = std::numeric_limits::min(); // -2^(32 - 1) == -2147483648 - - auto arbValues = fixedWidthInts.create(); - arbValues.fixedI16(-12345); - arbValues.fixedU32(1234567890); - arbValues.fixedU64(1234567890123456789); - auto& arbComp = arbValues.fixedWidthStruct(); - arbComp.fixedUnsigned16 = 12345; - arbComp.fixedInteger32 = -1234567890; - arbComp.fixedInteger64 = -1234567890123456789ll; - - // add some plain ints as user data - auto& uivec = usrInts; - uivec.resize(i + 1); - int myInt = 0; - for (auto& iu : uivec) { - iu = myInt++; - } - // and some user double values - unsigned nd = 100; - usrDoubles.resize(nd); - for (unsigned id = 0; id < nd; ++id) { - usrDoubles[id] = 42.; - } - - writer.writeEvent(); - store.clearCollections(); - } - - writer.finish(); -} - -#endif // PODIO_TESTS_WRITE_TEST_H diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 06c5118fe..4f4a277db 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -11,30 +11,52 @@ if(BUILD_TESTING) # depends_on the target name of the test that produces the required input file function(CREATE_DUMP_TEST name depends_on) add_test(NAME ${name} COMMAND ./podio-dump ${ARGN}) - - set(SIO_LD_PATH "") - if (ENABLE_SIO) - set(SIO_LD_PATH $) - endif() PODIO_SET_TEST_ENV(${name}) set_tests_properties(${name} PROPERTIES - DEPENDS ${depends_on} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} ) + if (depends_on) + set_tests_properties(${name} PROPERTIES + DEPENDS ${depends_on} + ) + endif() + endfunction() + + # Helper function for easily creating "tests" that simply execute podio-dump + # with different arguments on legacy input files. Not crashing is considered + # success. + # + # Args: + # name the name suffix of the test (the actual name will be assembled) + # version the legacy version of the input + # input_file the input file to use + function(CREATE_LEGACY_DUMP_TEST name version input_file) + set(_name podio-dump-legacy_${name}_${version}) + ExternalData_Add_Test(legacy_test_cases + NAME ${_name} + COMMAND ./podio-dump ${ARGN} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} + ) + PODIO_SET_TEST_ENV(${_name}) + + set_tests_properties(${_name} PROPERTIES + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + ) endfunction() CREATE_DUMP_TEST(podio-dump-help _dummy_target_ --help) - CREATE_DUMP_TEST(podio-dump-root-legacy "write" ${PROJECT_BINARY_DIR}/tests/root_io/example.root) CREATE_DUMP_TEST(podio-dump-root "write_frame_root" ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) CREATE_DUMP_TEST(podio-dump-detailed-root "write_frame_root" --detailed --category other_events --entries 2:3 ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) - CREATE_DUMP_TEST(podio-dump-detailed-root-legacy "write" --detailed --entries 2:3 ${PROJECT_BINARY_DIR}/tests/root_io/example.root) + + CREATE_LEGACY_DUMP_TEST("root" v00-16-06 v00-16-06-example.root) + CREATE_LEGACY_DUMP_TEST("root-detailed" v00-16-06 v00-16-06-example.root --detailed --entries 2:3) if (ENABLE_SIO) - CREATE_DUMP_TEST(podio-dump-sio-legacy "write_sio" ${PROJECT_BINARY_DIR}/tests/sio_io/example.sio) CREATE_DUMP_TEST(podio-dump-sio "write_frame_sio" --entries 4:7 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) CREATE_DUMP_TEST(podio-dump-detailed-sio "write_frame_sio" --detailed --entries 9 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) - CREATE_DUMP_TEST(podio-dump-detailed-sio-legacy "write_sio" --detailed --entries 9 ${PROJECT_BINARY_DIR}/tests/sio_io/example.sio) + + CREATE_LEGACY_DUMP_TEST("sio" v00-16-06 v00-16-06-example.sio) + CREATE_LEGACY_DUMP_TEST("sio-detailed" v00-16-06 v00-16-06-example.sio --detailed --entries 2:3) endif() if (ENABLE_RNTUPLE)