From c78fc014423585c78280f28f2c5d6dfd0810cae4 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 15 Sep 2023 16:51:23 +0200 Subject: [PATCH 01/37] Remove EventStore related files and make things compile again --- include/podio/ASCIIWriter.h | 80 --------- include/podio/EventStore.h | 160 ------------------ include/podio/IMetaDataProvider.h | 31 ---- include/podio/IReader.h | 58 ------- include/podio/PythonEventStore.h | 59 ------- include/podio/ROOTReader.h | 124 -------------- include/podio/ROOTWriter.h | 62 ------- include/podio/SIOBlock.h | 3 - include/podio/SIOReader.h | 117 ------------- include/podio/SIOWriter.h | 54 ------ include/podio/TimedReader.h | 158 ------------------ include/podio/TimedWriter.h | 62 ------- include/podio/utilities/Deprecated.h | 7 - include/podio/utilities/IOHelpers.h | 17 -- src/ASCIIWriter.cc | 40 ----- src/CMakeLists.txt | 27 +-- src/IOHelpers.cc | 39 ----- src/PythonEventStore.cc | 54 ------ src/SIOBlock.cc | 19 --- src/SIOWriter.cc | 124 -------------- src/python_selection.xml | 5 - src/selection.xml | 1 - tests/CMakeLists.txt | 4 - tests/CTestCustom.cmake | 11 -- tests/check_benchmark_outputs.cpp | 80 --------- tests/read_test.h | 47 +----- tests/root_io/CMakeLists.txt | 15 +- tests/root_io/read-legacy-files-root.cpp | 19 --- tests/root_io/read-multiple.cpp | 12 -- tests/root_io/read.cpp | 27 --- tests/root_io/read_and_write.cpp | 34 ---- tests/root_io/read_timed.cpp | 16 -- tests/root_io/write.cpp | 14 -- tests/root_io/write_timed.cpp | 14 -- tests/sio_io/CMakeLists.txt | 18 -- tests/sio_io/read_and_write_sio.cpp | 34 ---- tests/sio_io/read_sio.cpp | 28 ---- tests/sio_io/read_timed_sio.cpp | 16 -- tests/sio_io/write_sio.cpp | 10 -- tests/sio_io/write_timed_sio.cpp | 15 -- tests/unittests/CMakeLists.txt | 3 +- tests/unittests/unittest.cpp | 6 - tests/write_ascii.cpp | 200 ----------------------- 43 files changed, 7 insertions(+), 1917 deletions(-) delete mode 100644 include/podio/ASCIIWriter.h delete mode 100644 include/podio/EventStore.h delete mode 100644 include/podio/IMetaDataProvider.h delete mode 100644 include/podio/IReader.h delete mode 100644 include/podio/PythonEventStore.h delete mode 100644 include/podio/ROOTReader.h delete mode 100644 include/podio/ROOTWriter.h delete mode 100644 include/podio/SIOReader.h delete mode 100644 include/podio/SIOWriter.h delete mode 100644 include/podio/TimedReader.h delete mode 100644 include/podio/TimedWriter.h delete mode 100644 include/podio/utilities/Deprecated.h delete mode 100644 include/podio/utilities/IOHelpers.h delete mode 100644 src/ASCIIWriter.cc delete mode 100644 src/IOHelpers.cc delete mode 100644 src/PythonEventStore.cc delete mode 100644 src/SIOWriter.cc delete mode 100644 src/python_selection.xml delete mode 100644 tests/check_benchmark_outputs.cpp delete mode 100644 tests/root_io/read-legacy-files-root.cpp delete mode 100644 tests/root_io/read-multiple.cpp delete mode 100644 tests/root_io/read.cpp delete mode 100644 tests/root_io/read_and_write.cpp delete mode 100644 tests/root_io/read_timed.cpp delete mode 100644 tests/root_io/write.cpp delete mode 100644 tests/root_io/write_timed.cpp delete mode 100644 tests/sio_io/read_and_write_sio.cpp delete mode 100644 tests/sio_io/read_sio.cpp delete mode 100644 tests/sio_io/read_timed_sio.cpp delete mode 100644 tests/sio_io/write_sio.cpp delete mode 100644 tests/sio_io/write_timed_sio.cpp delete mode 100644 tests/write_ascii.cpp diff --git a/include/podio/ASCIIWriter.h b/include/podio/ASCIIWriter.h deleted file mode 100644 index e941bb6aa..000000000 --- a/include/podio/ASCIIWriter.h +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef PODIO_ASCIIWRITER_H -#define PODIO_ASCIIWRITER_H - -#include "podio/EventStore.h" -#include "podio/SchemaEvolution.h" -#include "podio/utilities/Deprecated.h" - -#include -#include -#include -#include -#include -#include - -namespace podio { - -class CollectionBase; - -struct ColWriterBase { - virtual void writeCollection(CollectionBase*, std::ostream&) = 0; - virtual ~ColWriterBase() = default; -}; - -template -struct ColWriter : public ColWriterBase { - void writeCollection(CollectionBase* c, std::ostream& o) override { - T* col = static_cast(c); - o << col->size() << std::endl; - o << *col << std::endl; - } -}; - -typedef std::map FunMap; - -class DEPR_EVTSTORE ASCIIWriter { - -public: - ASCIIWriter(const std::string& filename, EventStore* store); - ~ASCIIWriter(); - - // non-copyable - ASCIIWriter(const ASCIIWriter&) = delete; - ASCIIWriter& operator=(const ASCIIWriter&) = delete; - - template - bool registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - template - void writeCollection(const std::string& name); - // members - std::string m_filename; - EventStore* m_store; - - std::ofstream* m_file; - - std::vector m_storedCollections{}; - std::vector m_collectionNames{}; - FunMap m_map{}; -}; - -template -bool ASCIIWriter::registerForWrite(const std::string& name) { - const T* tmp_coll(nullptr); - if (!m_store->get(name, tmp_coll)) { - std::cerr << "no such collection to write, throw exception." << std::endl; - return false; - } - T* coll = const_cast(tmp_coll); - - m_storedCollections.emplace_back(coll); - m_collectionNames.emplace_back(name); - m_map[name] = new ColWriter; - return true; -} - -} // namespace podio -#endif diff --git a/include/podio/EventStore.h b/include/podio/EventStore.h deleted file mode 100644 index 05ae58b38..000000000 --- a/include/podio/EventStore.h +++ /dev/null @@ -1,160 +0,0 @@ -#ifndef PODIO_EVENTSTORE_H -#define PODIO_EVENTSTORE_H - -#include -#include -#include -#include -#include -#include - -// podio specific includes -#include "podio/CollectionIDTable.h" -#include "podio/GenericParameters.h" -#include "podio/ICollectionProvider.h" -#include "podio/IMetaDataProvider.h" -#include "podio/utilities/Deprecated.h" - -/** -This is an *example* event store - -The event store holds the object collections. - -It is used to create new collections, and to access existing ones. -When accessing a collection that is not yet in the event store, -the event store makes use of a Reader to read the collection. - -**/ - -namespace podio { - -class CollectionBase; -class IReader; - -typedef std::map RunMDMap; -typedef std::map ColMDMap; - -class DEPR_EVTSTORE EventStore : public ICollectionProvider, public IMetaDataProvider { -public: - /// Make non-copyable - EventStore(const EventStore&) = delete; - EventStore& operator=(const EventStore&) = delete; - - /// Collection entry. Each collection is identified by a name - typedef std::pair CollPair; - typedef std::vector CollContainer; - - EventStore(); - ~EventStore(); - - /// create a new collection - template - T& create(const std::string& name); - - /// register an existing collection - void registerCollection(const std::string& name, podio::CollectionBase* coll); - - /// access a collection by name. returns true if successful - template - bool get(const std::string& name, const T*& collection); - - /// access a collection by ID. returns true if successful - bool get(uint32_t id, CollectionBase*& coll) const final; - - /// access a collection by name - /// returns a collection w/ setting isValid to true if successful - template - const T& get(const std::string& name); - - /// empties collections. - void clearCollections(); - - /// clears itself; deletes collections (use at end of event processing) - void clear(); - - /// Clears only the cache containers (use at end of event if ownership of read objects is transferred) - void clearCaches(); - - /// set the reader - void setReader(IReader* reader); - - CollectionIDTable* getCollectionIDTable() const { - return m_table.get(); - } - - virtual bool isValid() const final; - - /// return the event meta data for the current event - GenericParameters& getEventMetaData() override; - - /// return the run meta data for the given runID - GenericParameters& getRunMetaData(int runID) override; - - /// return the collection meta data for the given colID - GenericParameters& getCollectionMetaData(uint32_t colID) override; - - RunMDMap* getRunMetaDataMap() { - return &m_runMDMap; - } - ColMDMap* getColMetaDataMap() { - return &m_colMDMap; - } - GenericParameters* eventMetaDataPtr() { - return &m_evtMD; - } - -private: - /// get the collection of given name; returns true if successfull - bool doGet(const std::string& name, CollectionBase*& collection, bool setReferences = true) const; - /// check if a collection of given name already exists - bool collectionRegistered(const std::string& name) const; - void setCollectionIDTable(std::shared_ptr table) { - m_table = std::move(table); - } - - // members - mutable std::set m_retrievedIDs{}; - mutable CollContainer m_collections{}; - IReader* m_reader{nullptr}; - std::shared_ptr m_table; - - GenericParameters m_evtMD{}; - RunMDMap m_runMDMap{}; - ColMDMap m_colMDMap{}; -}; - -template -T& EventStore::create(const std::string& name) { - static_assert(std::is_base_of::value, - "DataStore only accepts types inheriting from CollectionBase"); - // TODO: add check for existence - T* coll = new T(); - registerCollection(name, coll); - return *coll; -} - -template -bool EventStore::get(const std::string& name, const T*& collection) { - // static_assert(std::is_base_of::value, - // "DataStore only contains types inheriting from CollectionBase"); - CollectionBase* tmp{nullptr}; - doGet(name, tmp); - collection = static_cast(tmp); - if (collection != nullptr) { - return true; - } - return false; -} - -template -const T& EventStore::get(const std::string& name) { - const T* tmp(0); - auto success = this->get(name, tmp); - if (!success) { - throw std::runtime_error("No collection \'" + name + "\' is present in the EventStore"); - } - return *tmp; -} - -} // namespace podio -#endif diff --git a/include/podio/IMetaDataProvider.h b/include/podio/IMetaDataProvider.h deleted file mode 100644 index b8c662277..000000000 --- a/include/podio/IMetaDataProvider.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef PODIO_IMETADATAPROVIDER_H -#define PODIO_IMETADATAPROVIDER_H - -#include "podio/GenericParameters.h" -#include "podio/utilities/Deprecated.h" - -namespace podio { - -/** Inteface to access meta data for runs, events and collections. - * @author F. Gaede, DESY - * @date Apr 2020 - */ -class DEPR_EVTSTORE IMetaDataProvider { - -public: - /// destructor - virtual ~IMetaDataProvider() = default; - - /// return the event meta data for the current event - virtual GenericParameters& getEventMetaData() = 0; - - /// return the run meta data for the given runID - virtual GenericParameters& getRunMetaData(int runID) = 0; - - /// return the collection meta data for the given colID - virtual GenericParameters& getCollectionMetaData(uint32_t colID) = 0; -}; - -} // namespace podio - -#endif diff --git a/include/podio/IReader.h b/include/podio/IReader.h deleted file mode 100644 index 13553bf87..000000000 --- a/include/podio/IReader.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef PODIO_IREADER_H -#define PODIO_IREADER_H - -#include "podio/podioVersion.h" -#include "podio/utilities/Deprecated.h" - -#include -#include -#include -#include -#include - -/* - -This class has the function to read available data from disk -and to prepare collections and buffers. - -*/ - -namespace podio { - -class CollectionBase; -class CollectionIDTable; -class GenericParameters; - -class DEPR_EVTSTORE IReader { -public: - virtual ~IReader() = default; - /// Read Collection of given name - /// Does not set references yet. - virtual CollectionBase* readCollection(const std::string& name) = 0; - /// Get CollectionIDTable of read-in data - virtual std::shared_ptr getCollectionIDTable() = 0; - /// read event meta data from file - virtual GenericParameters* readEventMetaData() = 0; - virtual std::map* readCollectionMetaData() = 0; - virtual std::map* readRunMetaData() = 0; - /// get the number of events available from this reader - virtual unsigned getEntries() const = 0; - /// Prepare the reader to read the next event - virtual void endOfEvent() = 0; - // TODO: decide on smart-pointers for passing of objects - /// Check if reader is valid - virtual bool isValid() const = 0; - - virtual void openFile(const std::string& filename) = 0; - virtual void closeFile() = 0; - - virtual void readEvent() = 0; - virtual void goToEvent(unsigned iEvent) = 0; - - /// Get the podio version with which the current file has been written - virtual podio::version::Version currentFileVersion() const = 0; -}; - -} // namespace podio - -#endif diff --git a/include/podio/PythonEventStore.h b/include/podio/PythonEventStore.h deleted file mode 100644 index 8ae4d5140..000000000 --- a/include/podio/PythonEventStore.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef PODIO_PYTHONEVENTSTORE_H -#define PODIO_PYTHONEVENTSTORE_H - -#include "podio/EventStore.h" -#include "podio/GenericParameters.h" -#include "podio/IReader.h" -#include "podio/utilities/Deprecated.h" - -#include - -namespace podio { - -class DEPR_EVTSTORE PythonEventStore { -public: - /// constructor from filename - PythonEventStore(const char* filename); - - /// access a collection. - const podio::CollectionBase* get(const char* name); - - /// signify end of event - void endOfEvent(); - - /// go to a given event - void goToEvent(unsigned ievent); - - /// get number of entries in the tree - unsigned getEntries() const; - - /// is the input file accessible? - bool isZombie() const { - return m_isZombie; - } - - bool isValid() const { - return m_reader && m_reader->isValid(); - } - void close() { - m_reader->closeFile(); - } - - /// list available collections - const std::vector& getCollectionNames() const; - - const podio::GenericParameters& getEventMetaData() { - return m_store.getEventMetaData(); - } - -private: - std::unique_ptr m_reader; - podio::EventStore m_store; - - /// set to true if input root file accessible, false otherwise - bool m_isZombie{true}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/ROOTReader.h b/include/podio/ROOTReader.h deleted file mode 100644 index 757791cf0..000000000 --- a/include/podio/ROOTReader.h +++ /dev/null @@ -1,124 +0,0 @@ -#ifndef PODIO_ROOTREADER_H -#define PODIO_ROOTREADER_H - -#include "podio/CollectionBranches.h" -#include "podio/IReader.h" - -#include -#include -#include -#include -#include -#include -#include - -// forward declarations -class TClass; -class TFile; -class TTree; -class TChain; - -namespace podio { - -class EventStore; -class CollectionBase; -class Registry; -class CollectionIDTable; -class GenericParameters; - -/** -This class has the function to read available data from disk -and to prepare collections and buffers. -**/ -class ROOTReader : public IReader { - friend EventStore; - -public: - ROOTReader() = default; - // todo: see https://github.com/AIDASoft/podio/issues/290 - ~ROOTReader(); // NOLINT(modernize-use-equals-default) - - // non-copyable - ROOTReader(const ROOTReader&) = delete; - ROOTReader& operator=(const ROOTReader&) = delete; - - void openFile(const std::string& filename) override; - void openFiles(const std::vector& filenames); - void closeFile() override; - void closeFiles(); - - /// Read all collections requested - void readEvent() override; - - /// Read CollectionIDTable from ROOT file - std::shared_ptr getCollectionIDTable() override { - return m_table; - } - - /// Returns number of entries in the TTree - unsigned getEntries() const override; - - /// Preparing to read next event - void endOfEvent() override; - - /// Preparing to read a given event - void goToEvent(unsigned evnum) override; - - podio::version::Version currentFileVersion() const override { - return m_fileVersion; - } - - /// Check if TFile is valid - bool isValid() const override; - -private: - /// Implementation for collection reading - CollectionBase* readCollection(const std::string& name) override; - - /// read event meta data for current event - GenericParameters* readEventMetaData() override; - - /// read the collection meta data - std::map* readCollectionMetaData() override; - - /// read the run meta data - std::map* readRunMetaData() override; - -private: - void createCollectionBranches(const std::vector>& collInfo); - - std::pair getLocalTreeAndEntry(const std::string& treename); - // Information about the data vector as wall as the collection class type - // and the index in the collection branches cache vector - using CollectionInfo = std::tuple; - - CollectionBase* getCollection(const std::pair& collInfo); - CollectionBase* readCollectionData(const root_utils::CollectionBranches& branches, CollectionBase* collection, - Long64_t entry, const std::string& name); - - // cache collections that have been read already in a given event - typedef std::pair Input; - std::vector m_inputs{}; - - // cache the necessary information to more quickly construct and read each - // collection after it has been read the very first time - std::map m_storedClasses{}; - - std::shared_ptr m_table{nullptr}; - TChain* m_chain{nullptr}; - unsigned m_eventNumber{0}; - - // Similar to writing we cache the branches that belong to each collection - // in order to not having to look them up every event. However, for the - // reader we cannot guarantee a fixed order of collections as they are read - // on demand. Hence, we give each collection an index the first time it is - // read and we start caching the branches. - size_t m_collectionIndex = 0; - std::vector m_collectionBranches{}; - - podio::version::Version m_fileVersion{0, 0, 0}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/ROOTWriter.h b/include/podio/ROOTWriter.h deleted file mode 100644 index 6751809ae..000000000 --- a/include/podio/ROOTWriter.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef PODIO_ROOTWRITER_H -#define PODIO_ROOTWRITER_H - -#include "podio/CollectionBase.h" -#include "podio/CollectionBranches.h" -#include "podio/EventStore.h" -#include "podio/utilities/Deprecated.h" - -#include "TBranch.h" - -#include -#include -#include -#include -#include - -// forward declarations -class TFile; -class TTree; - -namespace podio { -class DEPR_EVTSTORE ROOTWriter { - -public: - ROOTWriter(const std::string& filename, EventStore* store); - ~ROOTWriter(); - - // non-copyable - ROOTWriter(const ROOTWriter&) = delete; - ROOTWriter& operator=(const ROOTWriter&) = delete; - - bool registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - using StoreCollection = std::pair; - void createBranches(const std::vector& collections); - void setBranches(const std::vector& collections); - - // members - std::string m_filename; - EventStore* m_store; - TFile* m_file; - TTree* m_datatree; - TTree* m_metadatatree; - TTree* m_runMDtree; - TTree* m_evtMDtree; - TTree* m_colMDtree; - std::vector m_collectionsToWrite{}; - // In order to avoid having to look up the branches from the datatree for - // every event, we cache them in this vector, that is populated the first - // time we write an event. Since the collections and their order do not - // change between events, the assocation between the collections to write - // and their branches is simply index based - std::vector m_collectionBranches{}; - - bool m_firstEvent{true}; -}; - -} // namespace podio -#endif diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index aa5e58c71..93b0c0e71 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -102,8 +101,6 @@ class SIOCollectionIDTableBlock : public sio::block { SIOCollectionIDTableBlock() : sio::block("CollectionIDs", sio::version::encode_version(0, 4)) { } - SIOCollectionIDTableBlock(podio::EventStore* store); - SIOCollectionIDTableBlock(std::vector&& names, std::vector&& ids, std::vector&& types, std::vector&& isSubsetColl) : sio::block("CollectionIDs", sio::version::encode_version(0, 3)), diff --git a/include/podio/SIOReader.h b/include/podio/SIOReader.h deleted file mode 100644 index 284dc13df..000000000 --- a/include/podio/SIOReader.h +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef PODIO_SIOREADER_H -#define PODIO_SIOREADER_H - -#include -#include -#include -#include -#include - -#include "podio/EventStore.h" -#include "podio/ICollectionProvider.h" -#include "podio/IReader.h" -#include "podio/SIOBlock.h" - -// -- sio headers -#include -#include -#include -#include - -namespace podio { - -class CollectionBase; -class CollectionIDTable; - -/** - This class has the function to read available data from disk - and to prepare collections and buffers. -**/ -class SIOReader : public IReader { - friend EventStore; - -public: - SIOReader(); - ~SIOReader() = default; - - // make non-copyable - SIOReader(const SIOReader&) = delete; - SIOReader& operator=(const SIOReader&) = delete; - - void openFile(const std::string& filename) override; - void closeFile() override; - - /// Read all collections requested - void readEvent() override; - - void goToEvent(unsigned iEvent) override; - - /// Read CollectionIDTable from SIO file - std::shared_ptr getCollectionIDTable() override { - return m_table; - } - - unsigned getEntries() const override { - return m_tocRecord.getNRecords("event_record"); - } - - /// Check if file is valid - bool isValid() const override; - - podio::version::Version currentFileVersion() const override { - return m_fileVersion; - } - - void endOfEvent() override; - -private: - /// Implementation for collection reading - CollectionBase* readCollection(const std::string& name) override; - - /// read event meta data for current event - GenericParameters* readEventMetaData() override; - - /// read the collection meta data - std::map* readCollectionMetaData() override; - - /// read the run meta data - std::map* readRunMetaData() override; - - /// read the TOC record - bool readFileTOCRecord(); - - /// reconstruct the TOC record from the file contents - void reconstructFileTOCRecord(); - -private: - void readCollectionIDTable(); - void readMetaDataRecord(const std::shared_ptr& mdBlock); - void createBlocks(); - - typedef std::pair Input; - std::vector m_inputs{}; - std::shared_ptr m_table{nullptr}; // Co-owned by the EventStore - int m_eventNumber{0}; - int m_lastEventRead{-1}; - std::vector m_typeNames{}; - std::vector m_subsetCollectionBits{}; - - std::shared_ptr m_eventMetaData{}; - std::shared_ptr m_runMetaData{}; - std::shared_ptr m_collectionMetaData{}; - - sio::ifstream m_stream{}; - sio::record_info m_rec_info{}; - sio::buffer m_info_buffer{sio::max_record_info_len}; - sio::buffer m_rec_buffer{sio::mbyte}; - sio::buffer m_unc_buffer{sio::mbyte}; - sio::block_list m_blocks{}; - - SIOFileTOCRecord m_tocRecord{}; - - podio::version::Version m_fileVersion{0}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/SIOWriter.h b/include/podio/SIOWriter.h deleted file mode 100644 index 7594ae74a..000000000 --- a/include/podio/SIOWriter.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef PODIO_SIOWRITER_H -#define PODIO_SIOWRITER_H - -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" -#include "podio/utilities/Deprecated.h" - -// SIO specific includes -#include - -#include -#include -#include - -// forward declarations - -namespace podio { -class DEPR_EVTSTORE SIOWriter { - -public: - SIOWriter(const std::string& filename, EventStore* store); - ~SIOWriter(); - - // non-copyable - SIOWriter(const SIOWriter&) = delete; - SIOWriter& operator=(const SIOWriter&) = delete; - - void registerForWrite(const std::string& name); - void writeEvent(); - void finish(); - -private: - void writeCollectionIDTable(); - sio::block_list createBlocks() const; - - // members - std::string m_filename{}; - EventStore* m_store = nullptr; - - std::shared_ptr m_eventMetaData; - sio::ofstream m_stream{}; - bool m_firstEvent{true}; - - std::shared_ptr m_runMetaData; - std::shared_ptr m_collectionMetaData; - SIOFileTOCRecord m_tocRecord{}; - std::vector m_collectionsToWrite{}; - - bool m_finished{false}; -}; - -} // namespace podio -#endif diff --git a/include/podio/TimedReader.h b/include/podio/TimedReader.h deleted file mode 100644 index 288258da3..000000000 --- a/include/podio/TimedReader.h +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef PODIO_TIMEDREADER_H -#define PODIO_TIMEDREADER_H - -#include "podio/BenchmarkRecorder.h" -#include "podio/BenchmarkUtil.h" - -#include "podio/GenericParameters.h" -#include "podio/IReader.h" - -#include - -namespace podio { - -template -class TimedReader : public IReader { - using ClockT = benchmark::ClockT; - -public: - template - TimedReader(benchmark::BenchmarkRecorder& recorder, Args&&... args) : - m_start(ClockT::now()), - m_reader(WrappedReader(std::forward(args)...)), - m_end(ClockT::now()), - m_recorder(recorder), - m_perEventTree(m_recorder.addTree( - "event_times", - {"read_collections", "read_ev_md", "read_run_md", "read_coll_md", "end_of_event", "read_event"})) { - m_recorder.addTree("setup_times", {"constructor", "open_file", "close_file", "read_collection_ids", "get_entries"}); - m_recorder.recordTime("setup_times", "constructor", m_end - m_start); - } - - ~TimedReader() { - // Timing deconstructors is not straight forward when wrapping a value. - // Since nothing is usually happening in them in any case, we simply don't - // do it. We still have to fill the setup_times tree here though. - m_recorder.Fill("setup_times"); - } - - /// Read Collection of given name - /// Does not set references yet. - CollectionBase* readCollection(const std::string& name) override { - const auto [result, duration] = benchmark::run_member_timed(m_reader, &IReader::readCollection, name); - // since we cannot in general know how many collections there will be read - // we simply sum up all the requests in an event and record that - m_totalCollectionReadTime += duration; - return result; - } - - /// Get CollectionIDTable of read-in data - std::shared_ptr getCollectionIDTable() override { - return runTimed(false, "read_collection_ids", &IReader::getCollectionIDTable); - } - - /// read event meta data from file - GenericParameters* readEventMetaData() override { - return runTimed(true, "read_ev_md", &IReader::readEventMetaData); - } - - std::map* readCollectionMetaData() override { - return runTimed(true, "read_coll_md", &IReader::readCollectionMetaData); - } - - std::map* readRunMetaData() override { - return runTimed(true, "read_run_md", &IReader::readRunMetaData); - } - - /// get the number of events available from this reader - unsigned getEntries() const override { - return runTimed(false, "get_entries", &IReader::getEntries); - } - - /// Prepare the reader to read the next event - void endOfEvent() override { - runVoidTimed(true, "end_of_event", &IReader::endOfEvent); - - m_perEventTree.recordTime("read_collections", m_totalCollectionReadTime); - m_perEventTree.Fill(); - m_totalCollectionReadTime = std::chrono::nanoseconds{0}; - } - - // not benchmarking this one - bool isValid() const override { - return m_reader.isValid(); - } - - void openFile(const std::string& filename) override { - runVoidTimed(false, "open_file", &IReader::openFile, filename); - } - - void closeFile() override { - runVoidTimed(false, "close_file", &IReader::closeFile); - } - - void readEvent() override { - runVoidTimed(true, "read_event", &IReader::readEvent); - } - - void goToEvent(unsigned ev) override { - // TODO: Do we need to time this? Not really used at the moment - m_reader.goToEvent(ev); - } - - podio::version::Version currentFileVersion() const override { - // no need to time this as it is really just a very simple get - return m_reader.currentFileVersion(); - } - -private: - void recordTime(bool perEvent, const std::string& step, ClockT::duration duration) const { - if (perEvent) { - m_perEventTree.recordTime(step, duration); - } else { - m_recorder.recordTime("setup_times", step, duration); - } - } - - template - inline std::invoke_result_t runTimed(bool perEvent, const std::string& step, - FuncT func, Args&&... args) { - const auto [result, duration] = benchmark::run_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - - return result; - } - - template - inline std::invoke_result_t runTimed(bool perEvent, const std::string& step, - FuncT func, Args&&... args) const { - const auto [result, duration] = benchmark::run_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - - return result; - } - - template - inline void runVoidTimed(bool perEvent, const std::string& step, FuncT func, Args&&... args) { - const auto duration = benchmark::run_void_member_timed(m_reader, func, std::forward(args)...); - - recordTime(perEvent, step, duration); - } - - // NOTE: c++ initializes its class members in the order they are defined not - // in the order in which they appear in the initializer list! - ClockT::time_point m_start; // to time the construction - WrappedReader m_reader; // the decorated reader that does the actual work - ClockT::time_point m_end; // to time the constructor call - - benchmark::BenchmarkRecorder& m_recorder; - // Keep a reference to this one around, to save the look-up in each event - benchmark::BenchmarkRecorderTree& m_perEventTree; - ClockT::duration m_totalCollectionReadTime{std::chrono::nanoseconds{0}}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/TimedWriter.h b/include/podio/TimedWriter.h deleted file mode 100644 index 90e154273..000000000 --- a/include/podio/TimedWriter.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef PODIO_TIMEDWRITER_H -#define PODIO_TIMEDWRITER_H - -#include "podio/BenchmarkRecorder.h" -#include "podio/BenchmarkUtil.h" -#include "podio/utilities/Deprecated.h" - -#include -#include - -namespace podio { - -template -class DEPR_EVTSTORE TimedWriter { - using ClockT = benchmark::ClockT; - -public: - template - TimedWriter(benchmark::BenchmarkRecorder& recorder, Args&&... args) : - m_start(ClockT::now()), - m_writer(WrappedWriter(std::forward(args)...)), - m_end(ClockT::now()), - m_recorder(recorder), - m_perEventTree(m_recorder.addTree("event_times", {"write_event"})) { - m_recorder.addTree("setup_times", {"constructor", "finish", "register_for_write"}); - m_recorder.recordTime("setup_times", "constructor", m_end - m_start); - } - - ~TimedWriter() { - m_recorder.recordTime("setup_times", "register_for_write", m_registerTime); - m_recorder.Fill("setup_times"); - } - - void registerForWrite(const std::string& name) { - // summing up the times it takes for all the collections to be registered - // here, since we do not know in advance how many collections there will be - // in the end - const auto duration = benchmark::run_void_member_timed(m_writer, &WrappedWriter::registerForWrite, name); - m_registerTime += duration; - } - - void writeEvent() { - m_perEventTree.recordTime("write_event", benchmark::run_void_member_timed(m_writer, &WrappedWriter::writeEvent)); - m_perEventTree.Fill(); - } - - void finish() { - m_recorder.recordTime("setup_times", "finish", benchmark::run_void_member_timed(m_writer, &WrappedWriter::finish)); - } - -private: - ClockT::time_point m_start; - WrappedWriter m_writer; - ClockT::time_point m_end; - benchmark::BenchmarkRecorder& m_recorder; - benchmark::BenchmarkRecorderTree& m_perEventTree; - ClockT::duration m_registerTime{std::chrono::nanoseconds{0}}; -}; - -} // namespace podio - -#endif diff --git a/include/podio/utilities/Deprecated.h b/include/podio/utilities/Deprecated.h deleted file mode 100644 index 566bca814..000000000 --- a/include/podio/utilities/Deprecated.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef PODIO_UTILITIES_DEPRECATED_H -#define PODIO_UTILITIES_DEPRECATED_H - -#define DEPR_EVTSTORE \ - [[deprecated("The EventStore based I/O model is deprecated and will be removed. Switch to the Frame based model.")]] - -#endif // PODIO_UTILITIES_DEPRECATED_H diff --git a/include/podio/utilities/IOHelpers.h b/include/podio/utilities/IOHelpers.h deleted file mode 100644 index b11ed76bc..000000000 --- a/include/podio/utilities/IOHelpers.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef PODIO_UTILITIES_IOHELPERS_H -#define PODIO_UTILITIES_IOHELPERS_H - -#ifndef PODIO_ENABLE_SIO - #define PODIO_ENABLE_SIO 0 -#endif - -#include "podio/IReader.h" - -#include -#include - -namespace podio { -std::unique_ptr createReader(const std::string& filename); -} - -#endif diff --git a/src/ASCIIWriter.cc b/src/ASCIIWriter.cc deleted file mode 100644 index f47eec8d6..000000000 --- a/src/ASCIIWriter.cc +++ /dev/null @@ -1,40 +0,0 @@ -// podio specific includes -#include "podio/ASCIIWriter.h" -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" - -namespace podio { - -ASCIIWriter::ASCIIWriter(const std::string& filename, EventStore* store) : - m_filename(filename), m_store(store), m_file(new std::ofstream) { - - m_file->open(filename, std::ofstream::binary); -} - -ASCIIWriter::~ASCIIWriter() { - delete m_file; -} - -void ASCIIWriter::writeEvent() { - - unsigned i = 0; - for (auto& coll : m_storedCollections) { - coll->prepareForWrite(); - - const std::string& name = m_collectionNames[i++]; - std::cout << " writing collection " << name << std::endl; - - *m_file << name << " "; - - ColWriterBase* wrt = m_map[name]; - - wrt->writeCollection(coll, *m_file); - } -} - -void ASCIIWriter::finish() { - - m_file->close(); -} - -} // namespace podio diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a68b94bfb..1e82ed26d 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,8 +50,6 @@ ENDFUNCTION() SET(core_sources CollectionIDTable.cc GenericParameters.cc - ASCIIWriter.cc - EventStore.cc DatamodelRegistry.cc DatamodelRegistryIOHelpers.cc UserDataCollection.cc @@ -63,14 +61,13 @@ SET(core_sources SET(core_headers ${PROJECT_SOURCE_DIR}/include/podio/CollectionBase.h ${PROJECT_SOURCE_DIR}/include/podio/CollectionIDTable.h - ${PROJECT_SOURCE_DIR}/include/podio/EventStore.h ${PROJECT_SOURCE_DIR}/include/podio/ICollectionProvider.h - ${PROJECT_SOURCE_DIR}/include/podio/IReader.h ${PROJECT_SOURCE_DIR}/include/podio/ObjectID.h ${PROJECT_SOURCE_DIR}/include/podio/UserDataCollection.h ${PROJECT_SOURCE_DIR}/include/podio/podioVersion.h ${PROJECT_SOURCE_DIR}/include/podio/DatamodelRegistry.h ${PROJECT_SOURCE_DIR}/include/podio/utilities/DatamodelRegistryIOHelpers.h + ${PROJECT_SOURCE_DIR}/include/podio/GenericParameters.h ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) @@ -80,8 +77,6 @@ target_compile_options(podio PRIVATE -pthread) # --- Root I/O functionality and corresponding dictionary SET(root_sources rootUtils.h - ROOTWriter.cc - ROOTReader.cc ROOTFrameWriter.cc ROOTFrameReader.cc ROOTLegacyReader.cc @@ -115,24 +110,9 @@ else() endif() -# --- Python EventStore for enabling (legacy) python bindings -SET(python_sources - IOHelpers.cc - PythonEventStore.cc - ) - -SET(python_headers - ${PROJECT_SOURCE_DIR}/include/podio/PythonEventStore.h -) -PODIO_ADD_LIB_AND_DICT(podioPythonStore "${python_headers}" "${python_sources}" python_selection.xml) -target_link_libraries(podioPythonStore PUBLIC podio::podio) -target_link_libraries(podioPythonStore PRIVATE podio::podioRootIO) - # --- SIO I/O functionality and corresponding dictionary if(ENABLE_SIO) SET(sio_sources - SIOReader.cc - SIOWriter.cc SIOBlockUserData.cc SIOBlock.cc SIOFrameWriter.cc @@ -152,14 +132,11 @@ if(ENABLE_SIO) target_link_libraries(podioSioIO PUBLIC podio::podio SIO::sio ${CMAKE_DL_LIBS} ${PODIO_FS_LIBS}) target_compile_definitions(podioSioIO PUBLIC PODIO_ENABLE_SIO=1) - # Make sure the legacy python bindings know about the SIO backend - target_link_libraries(podioPythonStore PRIVATE podioSioIO) - LIST(APPEND INSTALL_LIBRARIES podioSioIO podioSioIODict) endif() # --- Install everything -install(TARGETS podio podioDict podioPythonStore podioPythonStoreDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} +install(TARGETS podio podioDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} EXPORT podioTargets DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/src/IOHelpers.cc b/src/IOHelpers.cc deleted file mode 100644 index 7dd764083..000000000 --- a/src/IOHelpers.cc +++ /dev/null @@ -1,39 +0,0 @@ -#include "podio/utilities/IOHelpers.h" - -#include "podio/ROOTReader.h" - -#if PODIO_ENABLE_SIO - #include "podio/SIOReader.h" -#endif - -namespace podio { -std::unique_ptr createReader(const std::string& filename) { - const auto fileEnding = [&filename]() -> std::string { - const auto n = filename.rfind('.'); - if (n != std::string::npos) { - return filename.substr(n); - } - return ""; - }(); - - if (fileEnding.empty()) { - return nullptr; - } - - if (fileEnding == ".root") { - return std::make_unique(); - } else if (fileEnding == ".sio") { -#if PODIO_ENABLE_SIO - return std::make_unique(); -#else - std::cerr << "PODIO: You are trying to open a .sio file but podio has not been built with SIO support\nMake sure " - "to build PODIO with SIO support to be able to read .sio files" - << std::endl; - return nullptr; -#endif - } else { - return nullptr; - } -} - -} // namespace podio diff --git a/src/PythonEventStore.cc b/src/PythonEventStore.cc deleted file mode 100644 index 979bfb261..000000000 --- a/src/PythonEventStore.cc +++ /dev/null @@ -1,54 +0,0 @@ -#include "podio/PythonEventStore.h" - -#include "podio/ROOTReader.h" -#include "podio/utilities/IOHelpers.h" - -#include -#include -#include - -podio::PythonEventStore::PythonEventStore(const char* name) : m_reader(podio::createReader(name)), m_store() { - std::ifstream inputfile(name); - m_isZombie = inputfile.good() ? false : true; - - if (m_reader) { - if (m_isZombie && dynamic_cast(m_reader.get())) { - // the file could be a remote file that we cannot access but root - // knows how to handle via the xrootd protocol. - // if that is the case we ignore m_isZombie. - if (!std::string("root:/").compare(0, 6, name, 6)) { - m_isZombie = false; - } - } - } - - if (!m_isZombie) { - // at this point we have a combination of file and reader that should work - m_reader->openFile(name); - m_store.setReader(m_reader.get()); - } -} - -const podio::CollectionBase* podio::PythonEventStore::get(const char* name) { - const podio::CollectionBase* coll(nullptr); - m_store.get(name, coll); - return coll; -} - -void podio::PythonEventStore::endOfEvent() { - m_store.clear(); - m_reader->endOfEvent(); -} - -void podio::PythonEventStore::goToEvent(unsigned ievent) { - m_store.clear(); - m_reader->goToEvent(ievent); -} - -unsigned podio::PythonEventStore::getEntries() const { - return m_reader->getEntries(); -} - -const std::vector& podio::PythonEventStore::getCollectionNames() const { - return m_store.getCollectionIDTable()->names(); -} diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index efb26c867..4645d2a74 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -12,25 +12,6 @@ #endif namespace podio { -SIOCollectionIDTableBlock::SIOCollectionIDTableBlock(podio::EventStore* store) : - sio::block("CollectionIDs", sio::version::encode_version(0, 3)) { - const auto table = store->getCollectionIDTable(); - _names = table->names(); - _ids = table->ids(); - _types.reserve(_names.size()); - _isSubsetColl.reserve(_names.size()); - for (const int id : _ids) { - CollectionBase* tmp; - if (!store->get(id, tmp)) { - std::cerr - << "PODIO-ERROR cannot construct CollectionIDTableBlock because a collection is missing from the store (id: " - << id << ", name: " << table->name(id).value_or("") << ")" << std::endl; - } - - _types.emplace_back(tmp->getValueTypeName()); - _isSubsetColl.push_back(tmp->isSubsetCollection()); - } -} void SIOCollectionIDTableBlock::read(sio::read_device& device, sio::version_type version) { device.data(_names); diff --git a/src/SIOWriter.cc b/src/SIOWriter.cc deleted file mode 100644 index 1c1221c13..000000000 --- a/src/SIOWriter.cc +++ /dev/null @@ -1,124 +0,0 @@ -// podio specific includes -#include "podio/SIOWriter.h" -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" - -#include "sioUtils.h" - -// SIO specifc includes -#include "sio/block.h" -#include "sio/compression/zlib.h" - -namespace podio { - -SIOWriter::SIOWriter(const std::string& filename, EventStore* store) : - m_filename(filename), - m_store(store), - m_eventMetaData(std::make_shared()), - m_runMetaData(std::make_shared("RunMetaData")), - m_collectionMetaData(std::make_shared("CollectionMetaData")) { - - m_stream.open(filename, std::ios::binary); - - if (not m_stream.is_open()) { - SIO_THROW(sio::error_code::not_open, "Couldn't open output stream '" + filename + "'"); - } - - // TODO: re-visit this once metadata handling is done in better defined way - m_eventMetaData->metadata = m_store->eventMetaDataPtr(); - - m_runMetaData->data = m_store->getRunMetaDataMap(); - m_collectionMetaData->data = m_store->getColMetaDataMap(); - - auto& libLoader [[maybe_unused]] = SIOBlockLibraryLoader::instance(); -} - -SIOWriter::~SIOWriter() { - if (!m_finished) { - finish(); - } -} - -void SIOWriter::writeEvent() { - if (m_firstEvent) { - // Write the collectionIDs as a separate record at the beginning of the - // file. In this way they can easily be retrieved in the SIOReader without - // having to look for this specific record. - writeCollectionIDTable(); - m_firstEvent = false; - } - - auto blocks = createBlocks(); - m_tocRecord.addRecord("event_record", sio_utils::writeRecord(blocks, "event_record", m_stream)); -} - -sio::block_list SIOWriter::createBlocks() const { - sio::block_list blocks; - blocks.emplace_back(m_eventMetaData); - - for (const auto& name : m_collectionsToWrite) { - const podio::CollectionBase* col{nullptr}; - m_store->get(name, col); - col->prepareForWrite(); - - blocks.emplace_back(podio::SIOBlockFactory::instance().createBlock(col, name)); - } - - return blocks; -} - -void SIOWriter::finish() { - sio::block_list blocks{}; - blocks.push_back(m_runMetaData); - - m_tocRecord.addRecord(m_runMetaData->name(), sio_utils::writeRecord(blocks, m_runMetaData->name(), m_stream)); - - blocks.clear(); - blocks.push_back(m_collectionMetaData); - m_tocRecord.addRecord(m_collectionMetaData->name(), - sio_utils::writeRecord(blocks, m_collectionMetaData->name(), m_stream)); - - blocks.clear(); - auto tocRecordBlock = std::make_shared(); - tocRecordBlock->record = &m_tocRecord; - blocks.push_back(tocRecordBlock); - - const auto tocStartPos = sio_utils::writeRecord(blocks, sio_helpers::SIOTocRecordName, m_stream); - // Now that we know the position of the TOC Record, put this information - // into a final marker that can be identified and interpreted when reading - // again - uint64_t finalWords = (((uint64_t)sio_helpers::SIOTocMarker) << 32) | ((uint64_t)tocStartPos & 0xffffffff); - m_stream.write(reinterpret_cast(&finalWords), sizeof(finalWords)); - - m_stream.close(); - - m_finished = true; -} - -void SIOWriter::registerForWrite(const std::string& name) { - - const podio::CollectionBase* colB(nullptr); - m_store->get(name, colB); - - if (!colB) { - throw std::runtime_error(std::string("no such collection to write: ") + name); - } - // Check if we can instantiate the blocks here so that we can skip the checks later - if (auto blk = podio::SIOBlockFactory::instance().createBlock(colB, name); !blk) { - const auto typName = std::string(colB->getValueTypeName()); - throw std::runtime_error(std::string("could not create SIOBlock for type: ") + typName); - } - - m_collectionsToWrite.push_back(name); -} - -void SIOWriter::writeCollectionIDTable() { - sio::block_list blocks; - blocks.emplace_back(std::make_shared(m_store)); - blocks.emplace_back(std::make_shared(podio::version::build_version)); - - m_tocRecord.addRecord("file_metadata", sio_utils::writeRecord(blocks, "file_metadata", m_stream)); -} - -} // namespace podio diff --git a/src/python_selection.xml b/src/python_selection.xml deleted file mode 100644 index 61b39b94c..000000000 --- a/src/python_selection.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/selection.xml b/src/selection.xml index 4daca9192..03686376f 100644 --- a/src/selection.xml +++ b/src/selection.xml @@ -10,7 +10,6 @@ - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4132ff19c..ae0208500 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -48,9 +48,6 @@ list(APPEND ExternalData_URL_TEMPLATES "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" ) -add_executable(check_benchmark_outputs check_benchmark_outputs.cpp) -target_link_libraries(check_benchmark_outputs PRIVATE ROOT::Tree) - add_subdirectory(root_io) if (ENABLE_SIO) add_subdirectory(sio_io) @@ -61,7 +58,6 @@ add_subdirectory(schema_evolution) # Tests that don't fit into one of the broad categories above CREATE_PODIO_TEST(ostream_operator.cpp "") -CREATE_PODIO_TEST(write_ascii.cpp "") if (ENABLE_JULIA) message(STATUS "Julia Datamodel generation is enabled.") diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index af55f1532..87ac208aa 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -11,15 +11,8 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} - write - read - read_and_write read_and_write_associated - write_timed - read_timed check_benchmark_outputs - read-multiple - read-legacy-files-root_v00-13 read_frame_legacy_root read_frame_root_multiple write_python_frame_root @@ -29,12 +22,9 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ write_frame_root read_frame_root - check_benchmark_outputs_sio write_python_frame_sio read_python_frame_sio - write_ascii - relation_range pyunittest @@ -60,7 +50,6 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ ) foreach(version in @legacy_test_versions@) - list(APPEND CTEST_CUSTOM_TESTS_IGNORE read-legacy-files-root_${version}) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_root_${version}) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_legacy_root_${version}) endforeach() diff --git a/tests/check_benchmark_outputs.cpp b/tests/check_benchmark_outputs.cpp deleted file mode 100644 index eb268f9d4..000000000 --- a/tests/check_benchmark_outputs.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "TFile.h" -#include "TTree.h" - -#include -#include -#include - -constexpr int nExpectedEvents = 2000; -using StringVec = std::vector; - -bool verifyTree(TTree* tree, int expectedEntries, const StringVec& expectedBranches) { - const std::string treeName = tree->GetName(); - if (tree->GetEntries() != expectedEntries) { - std::cerr << "Tree \'" << treeName << "\' should have " << expectedEntries << " but has " << tree->GetEntries() - << std::endl; - return false; - } - - const auto* branches = tree->GetListOfBranches(); - for (const auto& branch : expectedBranches) { - bool found = false; - for (int i = 0; i < branches->GetEntries(); ++i) { - if (branch == branches->At(i)->GetName()) { - found = true; - break; - } - } - if (!found) { - std::cerr << "Branch \'" << branch << "\' was expected to be in Tree \'" << treeName - << "\' but could not be found" << std::endl; - return false; - } - } - - if ((unsigned)branches->GetEntries() != expectedBranches.size()) { - std::cerr << "Tree \'" << treeName << "\' has additional, unexpected branches" << std::endl; - return false; - } - - return true; -} - -void verifyBMFile(const char* fileName, const StringVec& setupBranches, const StringVec& eventBranches, - int expectedEvents = nExpectedEvents) { - TFile* bmFile = TFile::Open(fileName); - if (!bmFile) { - std::cerr << "Benchmark file \'" << fileName << "\' does not exist!" << std::endl; - std::exit(1); - } - if (!verifyTree(static_cast(bmFile->Get("setup_times")), 1, setupBranches)) { - std::cerr << "In file \'" << fileName << "\' setup_times Tree does not have the expected entries" << std::endl; - bmFile->Close(); - std::exit(1); - } - if (!verifyTree(static_cast(bmFile->Get("event_times")), expectedEvents, eventBranches)) { - std::cerr << "In file \'" << fileName << "\' event_times Tree does not have the expected entries" << std::endl; - bmFile->Close(); - std::exit(1); - } - - bmFile->Close(); -} - -/** - * We can't really make any checks on the actual execution times, but we can at - * least verify that the expected timing points are here. - */ -int main(int, char* argv[]) { - const StringVec writeBMSetupBranches = {"constructor", "finish", "register_for_write"}; - const StringVec writeBMEventBranches = {"write_event"}; - verifyBMFile(argv[1], writeBMSetupBranches, writeBMEventBranches); - - const StringVec readBMSetupBranches = {"constructor", "open_file", "close_file", "get_entries", - "read_collection_ids"}; - const StringVec readBMEventBranches = {"read_collections", "read_ev_md", "read_run_md", - "read_coll_md", "end_of_event", "read_event"}; - verifyBMFile(argv[2], readBMSetupBranches, readBMEventBranches); - - return 0; -} diff --git a/tests/read_test.h b/tests/read_test.h index 6382cb20a..2aee3fbaf 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -14,8 +14,6 @@ #include "datamodel/ExampleWithVectorMemberCollection.h" // podio specific includes -#include "podio/EventStore.h" -#include "podio/IReader.h" #include "podio/UserDataCollection.h" #include "podio/podioVersion.h" @@ -40,9 +38,9 @@ bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std } template -static constexpr bool isEventStore = std::is_same_v; +static constexpr bool isEventStore = false; -template +template void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersion) { float evtWeight = -1; @@ -463,45 +461,4 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } -void run_read_test(podio::IReader& reader) { - auto store = podio::EventStore(); - store.setReader(&reader); - - const auto nEvents = reader.getEntries(); - - // Some information in the test files dpends directly on the index. In - // read-multiple, the same file is essentially read twice, so that at the file - // change the index which is used for testing has to start at 0 again. This - // function just wraps that. The magic number of 2000 is the number of events - // that are present in each file that is written in write - const auto correctIndex = [nEvents](unsigned index) { - if (nEvents > 2000) { - return index % (nEvents / 2); - } - return index; - }; - - for (unsigned i = 0; i < nEvents; ++i) { - - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - - processEvent(store, correctIndex(i), reader.currentFileVersion()); - store.clear(); - reader.endOfEvent(); - } -} - -// Same as above but only for a specified event -void run_read_test_event(podio::IReader& reader, unsigned event) { - auto store = podio::EventStore(); - store.setReader(&reader); - - reader.goToEvent(event); - processEvent(store, event, reader.currentFileVersion()); - store.clear(); - reader.endOfEvent(); -} - #endif // PODIO_TESTS_READ_TEST_H diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index 099815022..f2dae7868 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -1,12 +1,6 @@ set(root_dependent_tests - write.cpp - read.cpp - read-multiple.cpp - relation_range.cpp - read_and_write.cpp + # relation_range.cpp read_and_write_associated.cpp - write_timed.cpp - read_timed.cpp read_frame_root.cpp write_frame_root.cpp read_frame_legacy_root.cpp @@ -29,11 +23,7 @@ endforeach() #--- set some dependencies between the different tests to ensure input generating ones are run first -set_property(TEST read PROPERTY DEPENDS write) -set_property(TEST read-multiple PROPERTY DEPENDS write) -set_property(TEST read_and_write PROPERTY DEPENDS write) set_property(TEST read_frame_legacy_root PROPERTY DEPENDS write) -set_property(TEST read_timed PROPERTY DEPENDS write_timed) set_tests_properties( read_frame_root @@ -48,8 +38,6 @@ if(ENABLE_RNTUPLE) set_property(TEST read_rntuple PROPERTY DEPENDS write_rntuple) endif() -add_test(NAME check_benchmark_outputs COMMAND check_benchmark_outputs write_benchmark_root.root read_benchmark_root.root) -set_property(TEST check_benchmark_outputs PROPERTY DEPENDS read_timed write_timed) message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") add_executable(read-legacy-files-root read-legacy-files-root.cpp) @@ -68,7 +56,6 @@ macro(ADD_PODIO_LEGACY_TEST version base_test input_file) # with existing headers from other installations ROOT_INCLUDE_PATH= ) - endmacro() ADD_PODIO_LEGACY_TEST(v00-13 read-legacy-files-root v00-13-example.root legacy_test_cases) diff --git a/tests/root_io/read-legacy-files-root.cpp b/tests/root_io/read-legacy-files-root.cpp deleted file mode 100644 index b0c7cee1c..000000000 --- a/tests/root_io/read-legacy-files-root.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -#include - -int main(int argc, char* argv[]) { - if (argc < 2) { - std::cerr << "Usage: read-legacy-files inputfile" << std::endl; - return 1; - } - - auto reader = podio::ROOTReader(); - reader.openFile(argv[1]); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read-multiple.cpp b/tests/root_io/read-multiple.cpp deleted file mode 100644 index e51743006..000000000 --- a/tests/root_io/read-multiple.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -int main() { - auto reader = podio::ROOTReader(); - reader.openFiles({"example.root", "example1.root"}); - - run_read_test(reader); - - reader.closeFiles(); - return 0; -} diff --git a/tests/root_io/read.cpp b/tests/root_io/read.cpp deleted file mode 100644 index d21ad81b0..000000000 --- a/tests/root_io/read.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "podio/ROOTReader.h" -#include "read_test.h" - -int main() { - auto reader = podio::ROOTReader(); - try { - reader.openFile("example.root"); - } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; - return 1; - } - - if (reader.currentFileVersion() != podio::version::build_version) { - return 1; - } - - run_read_test(reader); - - // jump back and forth a bit - run_read_test_event(reader, 10); - run_read_test_event(reader, 150); - run_read_test_event(reader, 120); - run_read_test_event(reader, 0); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read_and_write.cpp b/tests/root_io/read_and_write.cpp deleted file mode 100644 index 647eceadf..000000000 --- a/tests/root_io/read_and_write.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "read_test.h" -// podio specific includes -#include "podio/EventStore.h" -#include "podio/ROOTReader.h" -#include "podio/ROOTWriter.h" - -int main() { - auto reader = podio::ROOTReader(); - auto store = podio::EventStore(); - reader.openFile("example.root"); - store.setReader(&reader); - - // test writing a subset of the event to a new file - auto writer = podio::ROOTWriter("example_copy.root", &store); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - - unsigned nEvents = reader.getEntries(); - for (unsigned i = 0; i < nEvents; ++i) { - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - processEvent(store, i, reader.currentFileVersion()); - - writer.writeEvent(); - - store.clear(); - reader.endOfEvent(); - } - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/read_timed.cpp b/tests/root_io/read_timed.cpp deleted file mode 100644 index 30beaa885..000000000 --- a/tests/root_io/read_timed.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/ROOTReader.h" -#include "podio/TimedReader.h" -#include "read_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("read_benchmark_root.root"); - - podio::TimedReader reader(recorder); - reader.openFile("example_timed.root"); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/root_io/write.cpp b/tests/root_io/write.cpp deleted file mode 100644 index 00c99b8c5..000000000 --- a/tests/root_io/write.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "podio/EventStore.h" -#include "podio/ROOTWriter.h" -#include "write_test.h" - -int main(int, char**) { - auto store = podio::EventStore(); - podio::ROOTWriter writer("example.root", &store); - write(store, writer); - - // start from a clean slate for the second file - auto store2 = podio::EventStore(); - auto writer2 = podio::ROOTWriter("example1.root", &store2); - write(store2, writer2); -} diff --git a/tests/root_io/write_timed.cpp b/tests/root_io/write_timed.cpp deleted file mode 100644 index 8e2852c73..000000000 --- a/tests/root_io/write_timed.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/ROOTWriter.h" -#include "podio/TimedWriter.h" -#include "write_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("write_benchmark_root.root"); - podio::EventStore store; - podio::TimedWriter writer(recorder, "example_timed.root", &store); - - write(store, writer); - - return 0; -} diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index 2d24ce2f9..802746331 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -1,9 +1,4 @@ set(sio_dependent_tests - write_sio.cpp - read_sio.cpp - read_and_write_sio.cpp - write_timed_sio.cpp - read_timed_sio.cpp read_frame_sio.cpp write_frame_sio.cpp read_frame_legacy_sio.cpp @@ -15,16 +10,6 @@ foreach( sourcefile ${sio_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${sio_libs}") endforeach() -# These need to be linked against TTree explicitly, since it is not done -# through another library and the TimedReader/Writer decorators are -# header-only wrappers -target_link_libraries(write_timed_sio PRIVATE ROOT::Tree) -target_link_libraries(read_timed_sio PRIVATE ROOT::Tree) - -set_property(TEST read_sio PROPERTY DEPENDS write_sio) -set_property(TEST read_and_write_sio PROPERTY DEPENDS write_sio) -set_property(TEST read_timed_sio PROPERTY DEPENDS write_timed_sio) -set_property(TEST read_frame_legacy_sio PROPERTY DEPENDS write_sio) set_tests_properties( read_frame_sio @@ -35,9 +20,6 @@ set_tests_properties( write_frame_sio ) -add_test(NAME check_benchmark_outputs_sio COMMAND check_benchmark_outputs write_benchmark_sio.root read_benchmark_sio.root) -set_property(TEST check_benchmark_outputs_sio PROPERTY DEPENDS read_timed_sio write_timed_sio) - #--- Write via python and the SIO backend and see if we can read it back in in #--- c++ add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio sio_io.Writer) diff --git a/tests/sio_io/read_and_write_sio.cpp b/tests/sio_io/read_and_write_sio.cpp deleted file mode 100644 index 69f25d270..000000000 --- a/tests/sio_io/read_and_write_sio.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "read_test.h" - -#include "podio/EventStore.h" -#include "podio/SIOReader.h" -#include "podio/SIOWriter.h" - -int main() { - podio::SIOReader reader; - reader.openFile("example.sio"); - - auto store = podio::EventStore(); - store.setReader(&reader); - - auto writer = podio::SIOWriter("example_copy.sio", &store); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - - unsigned nEvents = reader.getEntries(); - for (unsigned i = 0; i < nEvents; ++i) { - if (i % 1000 == 0) { - std::cout << "reading event " << i << std::endl; - } - processEvent(store, i, reader.currentFileVersion()); - - writer.writeEvent(); - - store.clear(); - reader.endOfEvent(); - } - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/read_sio.cpp b/tests/sio_io/read_sio.cpp deleted file mode 100644 index 85ef72332..000000000 --- a/tests/sio_io/read_sio.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "podio/SIOReader.h" -#include "read_test.h" - -int main() { - // auto reader = podio::SIOReader(); - podio::SIOReader reader; // SIOReader has no copy c'tor ... - try { - reader.openFile("example.sio"); - } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; - return 1; - } - - if (reader.currentFileVersion() != podio::version::build_version) { - return 1; - } - - run_read_test(reader); - - // jump back and forth a bit - run_read_test_event(reader, 10); - run_read_test_event(reader, 150); - run_read_test_event(reader, 120); - run_read_test_event(reader, 0); - - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/read_timed_sio.cpp b/tests/sio_io/read_timed_sio.cpp deleted file mode 100644 index c109f0bee..000000000 --- a/tests/sio_io/read_timed_sio.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/SIOReader.h" -#include "podio/TimedReader.h" -#include "read_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("read_benchmark_sio.root"); - - podio::TimedReader reader(recorder); - reader.openFile("example_timed.sio"); - - run_read_test(reader); - - reader.closeFile(); - return 0; -} diff --git a/tests/sio_io/write_sio.cpp b/tests/sio_io/write_sio.cpp deleted file mode 100644 index 9068f312c..000000000 --- a/tests/sio_io/write_sio.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "podio/EventStore.h" -#include "podio/SIOWriter.h" -#include "write_test.h" - -int main(int, char**) { - podio::EventStore store; - podio::SIOWriter writer("example.sio", &store); - - write(store, writer); -} diff --git a/tests/sio_io/write_timed_sio.cpp b/tests/sio_io/write_timed_sio.cpp deleted file mode 100644 index 6edb8b771..000000000 --- a/tests/sio_io/write_timed_sio.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "podio/BenchmarkRecorder.h" -#include "podio/EventStore.h" -#include "podio/SIOWriter.h" -#include "podio/TimedWriter.h" -#include "write_test.h" - -int main() { - podio::benchmark::BenchmarkRecorder recorder("write_benchmark_sio.root"); - podio::EventStore store; - podio::TimedWriter writer(recorder, "example_timed.sio", &store); - - write(store, writer); - - return 0; -} diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index c49157b3f..d6003c284 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -40,7 +40,8 @@ if(NOT Catch2_FOUND) endif() find_package(Threads REQUIRED) -add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp) +# add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp) +add_executable(unittest_podio frame.cpp buffer_factory.cpp) target_link_libraries(unittest_podio PUBLIC TestDataModel PRIVATE Catch2::Catch2WithMain Threads::Threads podio::podioRootIO) if (ENABLE_SIO) target_link_libraries(unittest_podio PRIVATE podio::podioSioIO) diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index 35c936dae..a44733ba4 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -1155,9 +1155,6 @@ TEST_CASE("GenericParameters return types", "[generic-parameters][static-checks] } TEST_CASE("Missing files (ROOT readers)", "[basics]") { - auto root_reader = podio::ROOTReader(); - REQUIRE_THROWS_AS(root_reader.openFile("NonExistentFile.root"), std::runtime_error); - auto root_legacy_reader = podio::ROOTLegacyReader(); REQUIRE_THROWS_AS(root_legacy_reader.openFile("NonExistentFile.root"), std::runtime_error); @@ -1167,9 +1164,6 @@ TEST_CASE("Missing files (ROOT readers)", "[basics]") { #if PODIO_ENABLE_SIO TEST_CASE("Missing files (SIO readers)", "[basics]") { - auto sio_reader = podio::SIOReader(); - REQUIRE_THROWS_AS(sio_reader.openFile("NonExistentFile.sio"), std::runtime_error); - auto sio_legacy_reader = podio::SIOLegacyReader(); REQUIRE_THROWS_AS(sio_legacy_reader.openFile("NonExistentFile.sio"), std::runtime_error); diff --git a/tests/write_ascii.cpp b/tests/write_ascii.cpp deleted file mode 100644 index bc9dbe9c6..000000000 --- a/tests/write_ascii.cpp +++ /dev/null @@ -1,200 +0,0 @@ -// Data model -#include "datamodel/EventInfoCollection.h" -#include "datamodel/ExampleClusterCollection.h" -#include "datamodel/ExampleHitCollection.h" -#include "datamodel/ExampleMCCollection.h" -#include "datamodel/ExampleReferencingTypeCollection.h" -#include "datamodel/ExampleWithARelationCollection.h" -#include "datamodel/ExampleWithComponentCollection.h" -#include "datamodel/ExampleWithNamespaceCollection.h" -#include "datamodel/ExampleWithOneRelationCollection.h" -#include "datamodel/ExampleWithVectorMemberCollection.h" - -// STL -#include -#include - -// podio specific includes -#include "podio/ASCIIWriter.h" -#include "podio/EventStore.h" - -int main() { - - std::cout << "start processing" << std::endl; - - auto store = podio::EventStore(); - auto writer = podio::ASCIIWriter("example.txt", &store); - - auto& info = store.create("info"); - auto& mcps = store.create("mcparticles"); - auto& hits = store.create("hits"); - auto& clusters = store.create("clusters"); - auto& refs = store.create("refs"); - auto& refs2 = store.create("refs2"); - auto& comps = store.create("Component"); - auto& oneRels = store.create("OneRelation"); - auto& vecs = store.create("WithVectorMember"); - auto& namesps = store.create("WithNamespaceMember"); - auto& namesprels = store.create("WithNamespaceRelation"); - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("hits"); - writer.registerForWrite("clusters"); - writer.registerForWrite("refs"); - writer.registerForWrite("refs2"); - writer.registerForWrite("Component"); - writer.registerForWrite("OneRelation"); - writer.registerForWrite("WithVectorMember"); - writer.registerForWrite("WithNamespaceMember"); - writer.registerForWrite("WithNamespaceRelation"); - - unsigned nevents = 1; // 2000; - - for (unsigned i = 0; i < nevents; ++i) { - if (i % 1000 == 0) { - std::cout << "processing event " << i << std::endl; - } - - auto item1 = MutableEventInfo(); - item1.Number(i); - info.push_back(item1); - auto hit1 = MutableExampleHit(0xbad, 0., 0., 0., 23. + i); - auto hit2 = MutableExampleHit(0xcaffee, 1., 0., 0., 12. + i); - - hits.push_back(hit1); - hits.push_back(hit2); - - // ---- add some MC particles ---- - auto mcp0 = MutableExampleMC(); - auto mcp1 = MutableExampleMC(); - auto mcp2 = MutableExampleMC(); - auto mcp3 = MutableExampleMC(); - auto mcp4 = MutableExampleMC(); - auto mcp5 = MutableExampleMC(); - auto mcp6 = MutableExampleMC(); - auto mcp7 = MutableExampleMC(); - auto mcp8 = MutableExampleMC(); - auto mcp9 = MutableExampleMC(); - - mcps.push_back(mcp0); - mcps.push_back(mcp1); - mcps.push_back(mcp2); - mcps.push_back(mcp3); - mcps.push_back(mcp4); - mcps.push_back(mcp5); - mcps.push_back(mcp6); - mcps.push_back(mcp7); - mcps.push_back(mcp8); - mcps.push_back(mcp9); - - // --- add some daughter relations - - auto p = mcps[0]; - p.adddaughters(mcps[2]); - p.adddaughters(mcps[3]); - p.adddaughters(mcps[4]); - p.adddaughters(mcps[5]); - p = mcps[1]; - p.adddaughters(mcps[2]); - p.adddaughters(mcps[3]); - p.adddaughters(mcps[4]); - p.adddaughters(mcps[5]); - p = mcps[2]; - p.adddaughters(mcps[6]); - p.adddaughters(mcps[7]); - p.adddaughters(mcps[8]); - p.adddaughters(mcps[9]); - p = mcps[3]; - p.adddaughters(mcps[6]); - p.adddaughters(mcps[7]); - p.adddaughters(mcps[8]); - p.adddaughters(mcps[9]); - - //--- now fix the parent relations - for (unsigned j = 0, N = mcps.size(); j < N; ++j) { - p = mcps[j]; - for (auto it = p.daughters_begin(), end = p.daughters_end(); it != end; ++it) { - int dIndex = it->getObjectID().index; - auto d = mcps[dIndex]; - d.addparents(p); - } - } - //-------- print relations for debugging: - for (auto _p : mcps) { - std::cout << " particle " << _p.getObjectID().index << " has daughters: "; - for (auto it = _p.daughters_begin(), end = _p.daughters_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = _p.parents_begin(), end = _p.parents_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << std::endl; - } - //------------------------------- - - auto cluster = MutableExampleCluster(); - auto clu0 = MutableExampleCluster(); - auto clu1 = MutableExampleCluster(); - - clu0.addHits(hit1); - clu0.energy(hit1.energy()); - clu1.addHits(hit2); - clu1.energy(hit2.energy()); - cluster.addHits(hit1); - cluster.addHits(hit2); - cluster.energy(hit1.energy() + hit2.energy()); - cluster.addClusters(clu0); - cluster.addClusters(clu1); - - clusters.push_back(clu0); - clusters.push_back(clu1); - clusters.push_back(cluster); - - auto ref = MutableExampleReferencingType(); - refs.push_back(ref); - - auto ref2 = MutableExampleReferencingType(); - refs2.push_back(ref2); - - ref.addClusters(cluster); - ref.addRefs(ref2); - - auto comp = MutableExampleWithComponent(); - comp.component().data.x = 0; - comp.component().data.y = 1; - comp.component().data.z = i; - comps.push_back(comp); - - auto cyclic = MutableExampleReferencingType(); - cyclic.addRefs(cyclic); - refs.push_back(cyclic); - - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - - // write non-filled relation - auto oneRelEmpty = MutableExampleWithOneRelation(); - oneRels.push_back(oneRelEmpty); - - auto vec = MutableExampleWithVectorMember(); - vec.addcount(23); - vec.addcount(24); - vecs.push_back(vec); - - auto namesp = ex42::MutableExampleWithNamespace(); - namesp.component().x = 1; - namesp.component().y = i; - namesps.push_back(namesp); - - auto rel = ex42::MutableExampleWithARelation(); - rel.ref(namesp); - namesprels.push_back(rel); - - writer.writeEvent(); - store.clearCollections(); - } - - writer.finish(); -} From 9d5a064f5ee383e3332f9b45938a4c2f8c5240c5 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 15 Sep 2023 16:57:47 +0200 Subject: [PATCH 02/37] Remove EventStore python bindings --- python/EventStore.py | 6 -- python/podio/EventStore.py | 139 -------------------------------- python/podio/__init__.py | 22 +++-- python/podio/test_EventStore.py | 95 ---------------------- 4 files changed, 9 insertions(+), 253 deletions(-) delete mode 100644 python/EventStore.py delete mode 100644 python/podio/EventStore.py delete mode 100644 python/podio/test_EventStore.py diff --git a/python/EventStore.py b/python/EventStore.py deleted file mode 100644 index 5607f2da0..000000000 --- a/python/EventStore.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Legacy import wrapper for EventStore.""" - -import warnings -warnings.warn("You are using the legacy EventStore import. Switch to 'from podio import EventStore'", FutureWarning) - -from podio import EventStore # noqa: F401 # pylint: disable=wrong-import-position, unused-import diff --git a/python/podio/EventStore.py b/python/podio/EventStore.py deleted file mode 100644 index 068027aa4..000000000 --- a/python/podio/EventStore.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Python EventStore for reading files with podio generated datamodels""" - -import warnings -warnings.warn("The EventStore based I/O model is deprecated and will be removed. Switch to the Frame based model.", - FutureWarning) - -from ROOT import gSystem # pylint: disable=wrong-import-position -gSystem.Load("libpodioPythonStore") # noqa: E402 -from ROOT import podio # noqa: E402 # pylint: disable=wrong-import-position - - -def size(self): - """Override size function that can be attached as __len__ method to - collections""" - return self.size() - - -def getitem(self, key): - """Override getitem function that can be attached as __getitem__ method to - collections (see below why this is necessary sometimes)""" - return self.at(key) - - -class EventStore: - '''Interface to events in an podio root file. - Example of use: - events = EventStore(["example.root", "example1.root"]) - for iev, store in islice(enumerate(events), 0, 2): - particles = store.get("GenParticle"); - for i, p in islice(enumerate(particles), 0, 5): - print "particle ", i, p.ID(), p.P4().Pt - ''' - - def __init__(self, filenames): - '''Create an event list from the podio root file. - Parameters: - filenames: list of root files - you can of course provide a list containing a single - root file. you could use the glob module to get all - files matching a wildcard pattern. - ''' - if isinstance(filenames, str): - filenames = (filenames,) - self.files = filenames - self.stores = [] - self.current_store = None - for fname in self.files: - store = podio.PythonEventStore(fname) - if store.isZombie(): - raise ValueError(fname + ' does not exist.') - store.name = fname - if self.current_store is None: - self.current_store = store - self.stores.append((store.getEntries(), store)) - - def __str__(self): - result = "Content:" - result += "\n\t".join(n for n in self.current_store.getCollectionNames()) - return result - - def get(self, name): - '''Returns a collection. - Parameters: - name: name of the collection in the podio root file. - ''' - coll = self.current_store.get(name) - # adding length function - coll.__len__ = size - # enabling the use of [] notation on the collection - # cppyy defines the __getitem__ method if the underlying c++ class has an operator[] - # method. For some reason they do not conform to the usual signature and only - # pass one argument to the function they call. Here we simply check if we have to - # define the __getitem__ for the collection. - if not hasattr(coll, '__getitem__'): - coll.__getitem__ = getitem - return coll - - def collections(self): - """Return list of all collection names.""" - return [str(c) for c in self.current_store.getCollectionNames()] - - def metadata(self): - """Get the metadata of the current event as GenericParameters""" - return self.current_store.getEventMetaData() - - def isValid(self): - """Check if the EventStore is in a valid state""" - return self.current_store is not None and self.current_store.isValid() - - # def __getattr__(self, name): - # '''missing attributes are taken from self.current_store''' - # if name != 'current_store': - # return getattr(self.current_store, name) - # else: - # return None - - def current_filename(self): - '''Returns the name of the current file.''' - if self.current_store is None: - return None - return self.current_store.fname - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_val, trace): - for store in self.stores: - store[1].close() - - def __iter__(self): - '''iterate on events in the tree. - ''' - for _, store in self.stores: - self.current_store = store - for _ in range(store.getEntries()): - yield store - store.endOfEvent() - - def __getitem__(self, evnum): - '''Get event number evnum''' - current_store = None - rel_evnum = evnum - for nev, store in self.stores: - if rel_evnum < nev: - current_store = store - break - rel_evnum -= nev - if current_store is None: - raise ValueError('event number too large: ' + str(evnum)) - self.current_store = current_store - self.current_store.goToEvent(rel_evnum) - return self - - def __len__(self): - '''Returns the total number of events in all files.''' - nevts_all_files = 0 - for nev, _ in self.stores: - nevts_all_files += nev - return nevts_all_files diff --git a/python/podio/__init__.py b/python/podio/__init__.py index fd5fa7209..893758510 100644 --- a/python/podio/__init__.py +++ b/python/podio/__init__.py @@ -4,24 +4,21 @@ # Try to load podio, this is equivalent to trying to load libpodio.so and will # error if libpodio.so is not found but work if it's found try: - from ROOT import podio # noqa: F401 + from ROOT import podio # noqa: F401 except ImportError: - print('Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH') - raise + print("Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH") + raise from .frame import Frame from . import root_io, reading try: - # We try to import the sio bindings which may fail if ROOT is not able to - # load the dictionary. In this case they have most likely not been built and - # we just move on - from . import sio_io + # We try to import the sio bindings which may fail if ROOT is not able to + # load the dictionary. In this case they have most likely not been built and + # we just move on + from . import sio_io except ImportError: - pass - -from . import EventStore - + pass __all__ = [ "__version__", @@ -29,5 +26,4 @@ "root_io", "sio_io", "reading", - "EventStore" - ] +] diff --git a/python/podio/test_EventStore.py b/python/podio/test_EventStore.py deleted file mode 100644 index a8e4fb965..000000000 --- a/python/podio/test_EventStore.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Unit tests for the EventStore class""" - -from podio.EventStore import EventStore - - -class EventStoreBaseTestCaseMixin: - """EventStore unit tests - - These define some tests that should work regardless of the backend that is - used. In order to not have to duplicate this functionality for each backend, - this base class defines the common tests and inheriting classes define a - corresponding setUp method that sets up the correct EventStore and potentially - additional backend specific functionality - """ - def test_eventloop(self): - self.assertTrue(len(self.store) >= 0) - self.assertEqual(self.store.current_store.getEntries(), - len(self.store)) - for iev, event in enumerate(self.store): - self.assertTrue(event is not None) - if iev > 5: - break - - def test_navigation(self): - event0 = self.store[0] - self.assertEqual(event0.__class__, self.store.__class__) - - def test_collections(self): - evinfo = self.store.get("info") - self.assertTrue(len(evinfo) > 0) - particles = self.store.get("CollectionNotThere") - self.assertFalse(particles) - - def test_read_only(self): - hits = self.store.get("hits") - # testing that one can't modify attributes in - # read-only pods - self.assertEqual(hits[0].energy(), 23.) - hits[0].energy(10) - self.assertEqual(hits[0].energy(), 10) # oops - - def test_one_to_many(self): - clusters = self.store.get("clusters") - ref_hits = [] - # testing that cluster hits can be accessed and make sense - for cluster in clusters: - sume = 0 - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - ref_hits.append(hit) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - hits = self.store.get("hits") - # testing that the hits stored as a one to many relation - # in the cluster can be found in the hit collection - for hit in ref_hits: - self.assertTrue(hit in hits) - - def test_relation_range(self): - """Test that the RelationRange functionality is also accessible in python""" - clusters = self.store.get("clusters") - hits = self.store.get("hits") - - for cluster in clusters: - sume = 0 - for hit in cluster.Hits(): - self.assertTrue(hit in hits) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - - def test_hash(self): - clusters = self.store.get("clusters") - ref_hits = [] - # testing that cluster hits can be accessed and make sense - for cluster in clusters: - sume = 0 - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - ref_hits.append(hit) - sume += hit.energy() - self.assertEqual(cluster.energy(), sume) - hits = self.store.get("hits") - if hits[0] == ref_hits[0]: - self.assertEqual(hits[0].getObjectID().index, - ref_hits[0].getObjectID().index) - self.assertEqual(hits[0].getObjectID().collectionID, - ref_hits[0].getObjectID().collectionID) - self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID()) - # testing that the hits stored as a one to many relation - # import pdb; pdb.set_trace() - - def test_context_managers(self): - with EventStore([self.filename]) as store: - self.assertTrue(len(store) >= 0) - self.assertTrue(store.isValid()) From 642f80ae7b977960e7bc83831b986293f1885985 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 15 Sep 2023 17:00:16 +0200 Subject: [PATCH 03/37] Remove unnecessary EventStore --- tests/unittests/CMakeLists.txt | 3 +-- tests/unittests/unittest.cpp | 38 ++++++++++++---------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index d6003c284..c49157b3f 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -40,8 +40,7 @@ if(NOT Catch2_FOUND) endif() find_package(Threads REQUIRED) -# add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp) -add_executable(unittest_podio frame.cpp buffer_factory.cpp) +add_executable(unittest_podio unittest.cpp frame.cpp buffer_factory.cpp) target_link_libraries(unittest_podio PUBLIC TestDataModel PRIVATE Catch2::Catch2WithMain Threads::Threads podio::podioRootIO) if (ENABLE_SIO) target_link_libraries(unittest_podio PRIVATE podio::podioSioIO) diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index a44733ba4..febc55686 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -12,14 +12,10 @@ #include "catch2/matchers/catch_matchers_vector.hpp" // podio specific includes -#include "datamodel/MutableExampleHit.h" -#include "podio/EventStore.h" -#include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/ROOTFrameReader.h" #include "podio/ROOTFrameWriter.h" #include "podio/ROOTLegacyReader.h" -#include "podio/ROOTReader.h" #include "podio/podioVersion.h" #ifndef PODIO_ENABLE_SIO @@ -28,7 +24,6 @@ #if PODIO_ENABLE_SIO #include "podio/SIOFrameReader.h" #include "podio/SIOLegacyReader.h" - #include "podio/SIOReader.h" #endif #if PODIO_ENABLE_RNTUPLE @@ -52,11 +47,10 @@ #include "podio/UserDataCollection.h" TEST_CASE("AutoDelete", "[basics][memory-management]") { - auto store = podio::EventStore(); + auto coll = EventInfoCollection(); auto hit1 = MutableEventInfo(); auto hit2 = MutableEventInfo(); auto hit3 = MutableEventInfo(); - auto& coll = store.create("info"); coll.push_back(hit1); hit3 = hit2; } @@ -205,11 +199,10 @@ TEST_CASE("Container lifetime", "[basics][memory-management]") { } TEST_CASE("Invalid_refs", "[basics][relations]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); auto hit1 = hits.create(0xcaffeeULL, 0., 0., 0., 0.); - auto hit2 = MutableExampleHit(); - auto& clusters = store.create("clusters"); + auto hit2 = ExampleHit(); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); cluster.addHits(hit1); cluster.addHits(hit2); @@ -217,8 +210,7 @@ TEST_CASE("Invalid_refs", "[basics][relations]") { } TEST_CASE("Looping", "[basics]") { - auto store = podio::EventStore(); - auto& coll = store.create("name"); + auto coll = ExampleHitCollection(); auto hit1 = coll.create(0xbadULL, 0., 0., 0., 0.); auto hit2 = coll.create(0xcaffeeULL, 1., 1., 1., 1.); for (auto&& i : coll) { @@ -235,7 +227,7 @@ TEST_CASE("Looping", "[basics]") { REQUIRE(coll[0].energy() == 0); REQUIRE(coll[1].energy() == 1); - auto& constColl = store.get("name"); + const auto& constColl = coll; int index = 0; for (auto hit : constColl) { auto energy = hit.energy(); @@ -244,8 +236,7 @@ TEST_CASE("Looping", "[basics]") { } TEST_CASE("Notebook", "[basics]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); for (unsigned i = 0; i < 12; ++i) { auto hit = hits.create(0xcaffeeULL, 0., 0., 0., double(i)); } @@ -285,11 +276,10 @@ TEST_CASE("Podness", "[basics][code-gen]") { } TEST_CASE("Referencing", "[basics][relations]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); + auto hits = ExampleHitCollection(); auto hit1 = hits.create(0x42ULL, 0., 0., 0., 0.); auto hit2 = hits.create(0x42ULL, 1., 1., 1., 1.); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); cluster.addHits(hit1); cluster.addHits(hit2); @@ -302,8 +292,7 @@ TEST_CASE("Referencing", "[basics][relations]") { TEST_CASE("VariadicCreate", "[basics]") { // Test that objects created via the variadic create template function handle relations correctly - auto store = podio::EventStore(); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto variadic_cluster = clusters.create(3.14f); auto normal_cluster = clusters.create(); @@ -315,11 +304,10 @@ TEST_CASE("VariadicCreate", "[basics]") { } TEST_CASE("write_buffer", "[basics][io]") { - auto store = podio::EventStore(); - auto& coll = store.create("data"); + auto coll = ExampleHitCollection(); auto hit1 = coll.create(0x42ULL, 0., 0., 0., 0.); auto hit2 = coll.create(0x42ULL, 1., 1., 1., 1.); - auto& clusters = store.create("clusters"); + auto clusters = ExampleClusterCollection(); auto cluster = clusters.create(); // add a few related objects to also exercise relation writing cluster.addHits(hit1); @@ -333,7 +321,7 @@ TEST_CASE("write_buffer", "[basics][io]") { REQUIRE_NOTHROW(clusters.prepareForWrite()); REQUIRE(clusters.getBuffers().data == buffers.data); - auto& ref_coll = store.create("onerel"); + auto ref_coll = ExampleWithOneRelationCollection(); auto withRef = ref_coll.create(); REQUIRE_NOTHROW(ref_coll.prepareForWrite()); } From a125ae65dbed0cae206cb2543555604c9d52c4c0 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 09:12:49 +0200 Subject: [PATCH 04/37] Remove implementation files and python tests --- python/podio/test_EventStoreRoot.py | 51 ----- python/podio/test_EventStoreSio.py | 33 ---- src/EventStore.cc | 140 ------------- src/ROOTReader.cc | 295 ---------------------------- src/ROOTWriter.cc | 145 -------------- src/SIOReader.cc | 241 ----------------------- 6 files changed, 905 deletions(-) delete mode 100644 python/podio/test_EventStoreRoot.py delete mode 100644 python/podio/test_EventStoreSio.py delete mode 100644 src/EventStore.cc delete mode 100644 src/ROOTReader.cc delete mode 100644 src/ROOTWriter.cc delete mode 100644 src/SIOReader.cc diff --git a/python/podio/test_EventStoreRoot.py b/python/podio/test_EventStoreRoot.py deleted file mode 100644 index 522e389d7..000000000 --- a/python/podio/test_EventStoreRoot.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Python unit tests for the ROOT backend""" - -import unittest -import os - -from ROOT import TFile - -from test_EventStore import EventStoreBaseTestCaseMixin # pylint: disable=import-error - -from podio.EventStore import EventStore - - -class EventStoreRootTestCase(EventStoreBaseTestCaseMixin, unittest.TestCase): - """Test cases for root input files""" - def setUp(self): - """Setup an EventStore reading from a ROOT file""" - self.filename = 'root_io/example.root' - self.assertTrue(os.path.isfile(self.filename)) - self.store = EventStore([self.filename]) - - def test_chain(self): - self.store = EventStore([self.filename, - self.filename]) - rootfile = TFile(self.filename) - events = rootfile.Get(str('events')) - numbers = [] - for iev, _ in enumerate(self.store): - evinfo = self.store.get("info") - numbers.append(evinfo[0].Number()) - self.assertEqual(iev + 1, 2 * events.GetEntries()) # pylint: disable=undefined-loop-variable - # testing that numbers is [0, .. 1999, 0, .. 1999] - self.assertEqual(numbers, list(range(events.GetEntries())) * 2) - # trying to go to an event beyond the last one - self.assertRaises(ValueError, self.store.__getitem__, 4001) - # this is in the first event in the second file, - # so its event number should be 0. - self.assertEqual(self.store[2000].get("info")[0].Number(), 0) - - def test_no_file(self): - '''Test that non-accessible files are gracefully handled.''' - with self.assertRaises(ValueError): - self.store = EventStore('foo.root') - - -if __name__ == '__main__': - # NOTE: These tests are really not intended to be run directly as they depend - # on quite some environment setup as well as externally produced inputs. - # See the CMakeLists.txt file in the tests folder for the specifics of that - # environment and the inputs - unittest.main() diff --git a/python/podio/test_EventStoreSio.py b/python/podio/test_EventStoreSio.py deleted file mode 100644 index c8d19b852..000000000 --- a/python/podio/test_EventStoreSio.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Python unit tests for the SIO backend""" - -import unittest -import os - -from test_utils import SKIP_SIO_TESTS # pylint: disable=import-error -from test_EventStore import EventStoreBaseTestCaseMixin # pylint: disable=import-error - -from podio.EventStore import EventStore - - -@unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") -class EventStoreSioTestCase(EventStoreBaseTestCaseMixin, unittest.TestCase): - """Test cases for root input files""" - def setUp(self): - """setup an EventStore reading an SIO file""" - self.filename = 'sio_io/example.sio' - self.assertTrue(os.path.isfile(self.filename)) - self.store = EventStore([self.filename]) - - def test_no_file(self): - '''Test that non-accessible files are gracefully handled.''' - with self.assertRaises(ValueError): - self.store = EventStore('foo.sio') - - -if __name__ == '__main__': - # NOTE: These tests are really not intended to be run directly as they depend - # on quite some environment setup as well as externally produced inputs. - # See the CMakeLists.txt file in the tests folder for the specifics of that - # environment and the inputs - unittest.main() diff --git a/src/EventStore.cc b/src/EventStore.cc deleted file mode 100644 index dde8961f8..000000000 --- a/src/EventStore.cc +++ /dev/null @@ -1,140 +0,0 @@ - -// podio specific includes -#include "podio/EventStore.h" -#include "podio/CollectionBase.h" -#include "podio/IReader.h" - -namespace podio { - -EventStore::EventStore() : m_table(new CollectionIDTable()) { -} - -EventStore::~EventStore() { - for (auto& coll : m_collections) { - delete coll.second; - } -} - -bool EventStore::get(uint32_t id, CollectionBase*& collection) const { - auto val = m_retrievedIDs.insert(id); - bool success = false; - if (val.second == true) { - // collection not yet retrieved in recursive-call - auto name = m_table->name(id).value(); - success = doGet(name, collection, true); - } else { - // collection already requested in recursive call - // do not set the references to break collection dependency-cycle - auto name = m_table->name(id).value(); - success = doGet(name, collection, false); - } - // fg: the set should only be cleared at the end of event (in clear() ) ... - // m_retrievedIDs.erase(id); - return success; -} - -void EventStore::registerCollection(const std::string& name, podio::CollectionBase* coll) { - m_collections.emplace_back(name, coll); - auto id = m_table->add(name); - coll->setID(id); -} - -bool EventStore::isValid() const { - return m_reader->isValid(); -} - -bool EventStore::doGet(const std::string& name, CollectionBase*& collection, bool setReferences) const { - auto result = std::find_if(begin(m_collections), end(m_collections), - [&name](const CollPair& item) -> bool { return name == item.first; }); - if (result != end(m_collections)) { - auto tmp = result->second; - if (tmp != nullptr) { - collection = tmp; - return true; - } - } else if (m_reader != nullptr) { - auto tmp = m_reader->readCollection(name); - if (setReferences == true) { - if (tmp != nullptr) { - tmp->setReferences(this); - // check again whether collection exists - // it may have been created on-demand already - if (collectionRegistered(name) == false) { - m_collections.emplace_back(std::make_pair(name, tmp)); - } - } - } - collection = tmp; - if (tmp != nullptr) { - return true; - } - } else { - return false; - } - return false; -} - -GenericParameters& EventStore::getEventMetaData() { - - if (m_evtMD.empty() && m_reader != nullptr) { - GenericParameters* tmp = m_reader->readEventMetaData(); - m_evtMD = std::move(*tmp); - delete tmp; - } - return m_evtMD; -} - -GenericParameters& EventStore::getRunMetaData(int runID) { - - if (m_runMDMap.empty() && m_reader != nullptr) { - RunMDMap* tmp = m_reader->readRunMetaData(); - m_runMDMap = std::move(*tmp); - delete tmp; - } - return m_runMDMap[runID]; -} - -GenericParameters& EventStore::getCollectionMetaData(uint32_t colID) { - - if (m_colMDMap.empty() && m_reader != nullptr) { - ColMDMap* tmp = m_reader->readCollectionMetaData(); - m_colMDMap = std::move(*tmp); - delete tmp; - } - return m_colMDMap[colID]; -} - -void EventStore::clearCollections() { - for (auto& coll : m_collections) { - coll.second->clear(); - } - m_evtMD.clear(); -} - -void EventStore::clear() { - for (auto& coll : m_collections) { - coll.second->clear(); - delete coll.second; - } - - m_evtMD.clear(); - clearCaches(); -} - -void EventStore::clearCaches() { - m_collections.clear(); - m_retrievedIDs.clear(); -} - -bool EventStore::collectionRegistered(const std::string& name) const { - auto result = std::find_if(begin(m_collections), end(m_collections), - [&name](const CollPair& item) -> bool { return name == item.first; }); - return (result != end(m_collections)); -} - -void EventStore::setReader(IReader* reader) { - m_reader = reader; - setCollectionIDTable(reader->getCollectionIDTable()); -} - -} // namespace podio diff --git a/src/ROOTReader.cc b/src/ROOTReader.cc deleted file mode 100644 index 5b91378fe..000000000 --- a/src/ROOTReader.cc +++ /dev/null @@ -1,295 +0,0 @@ -#include "rootUtils.h" - -// podio specific includes -#include "podio/CollectionBase.h" -#include "podio/CollectionIDTable.h" -#include "podio/GenericParameters.h" -#include "podio/ROOTReader.h" - -// ROOT specific includes -#include "TChain.h" -#include "TClass.h" -#include "TFile.h" -#include "TTree.h" -#include "TTreeCache.h" -#include -#include - -namespace podio { -// todo: see https://github.com/AIDASoft/podio/issues/290 -ROOTReader::~ROOTReader() { // NOLINT(modernize-use-equals-default) -} - -std::pair ROOTReader::getLocalTreeAndEntry(const std::string& treename) { - auto localEntry = m_chain->LoadTree(m_eventNumber); - auto* tree = static_cast(m_chain->GetFile()->Get(treename.c_str())); - return {tree, localEntry}; -} - -GenericParameters* ROOTReader::readEventMetaData() { - auto* emd = new GenericParameters(); - auto [evt_metadatatree, entry] = getLocalTreeAndEntry("evt_metadata"); - auto* branch = root_utils::getBranch(evt_metadatatree, "evtMD"); - branch->SetAddress(&emd); - evt_metadatatree->GetEntry(entry); - return emd; -} -std::map* ROOTReader::readCollectionMetaData() { - auto* emd = new std::map; - auto* col_metadatatree = getLocalTreeAndEntry("col_metadata").first; - auto* branch = root_utils::getBranch(col_metadatatree, "colMD"); - branch->SetAddress(&emd); - col_metadatatree->GetEntry(0); - return emd; -} -std::map* ROOTReader::readRunMetaData() { - auto* emd = new std::map; - auto* run_metadatatree = getLocalTreeAndEntry("run_metadata").first; - auto* branch = root_utils::getBranch(run_metadatatree, "runMD"); - branch->SetAddress(&emd); - run_metadatatree->GetEntry(0); - return emd; -} - -CollectionBase* ROOTReader::readCollection(const std::string& name) { - // has the collection already been constructed? - auto p = - std::find_if(begin(m_inputs), end(m_inputs), [&name](const ROOTReader::Input& t) { return t.second == name; }); - if (p != end(m_inputs)) { - return p->first; - } - - // Do we know about this collection? If so, read it - if (const auto& info = m_storedClasses.find(name); info != m_storedClasses.end()) { - return getCollection(*info); - } - - // At this point this collection is definitely not in this file, because we - // have no information on how to construct it in the first place - return nullptr; -} - -CollectionBase* ROOTReader::getCollection(const std::pair& collInfo) { - const auto& name = collInfo.first; - const auto& [theClass, collectionClass, index] = collInfo.second; - auto& branches = m_collectionBranches[index]; - - auto* collection = static_cast(collectionClass->New()); - auto collBuffers = collection->getBuffers(); - // If we have a valid data buffer class we know that have to read data, - // otherwise we are handling a subset collection - if (theClass) { - collBuffers.data = theClass->New(); - } else { - collection->setSubsetCollection(); - } - - const auto localEntry = m_chain->LoadTree(m_eventNumber); - // After switching trees in the chain, branch pointers get invalidated so - // they need to be reassigned. - // NOTE: root 6.22/06 requires that we get completely new branches here, - // with 6.20/04 we could just re-set them - if (localEntry == 0) { - branches.data = root_utils::getBranch(m_chain, name.c_str()); - - // reference collections - if (auto* refCollections = collBuffers.references) { - for (size_t i = 0; i < refCollections->size(); ++i) { - const auto brName = root_utils::refBranch(name, i); - branches.refs[i] = root_utils::getBranch(m_chain, brName.c_str()); - } - } - - // vector members - if (auto* vecMembers = collBuffers.vectorMembers) { - for (size_t i = 0; i < vecMembers->size(); ++i) { - const auto brName = root_utils::vecBranch(name, i); - branches.vecs[i] = root_utils::getBranch(m_chain, brName.c_str()); - } - } - } - - // set the addresses - root_utils::setCollectionAddresses(collection->getBuffers(), branches); - - return readCollectionData(branches, collection, localEntry, name); -} - -CollectionBase* ROOTReader::readCollectionData(const root_utils::CollectionBranches& branches, - CollectionBase* collection, Long64_t entry, const std::string& name) { - // Read all data - if (branches.data) { - branches.data->GetEntry(entry); - } - for (auto* br : branches.refs) { - br->GetEntry(entry); - } - for (auto* br : branches.vecs) { - br->GetEntry(entry); - } - - // do the unpacking - const auto id = m_table->collectionID(name).value(); - collection->setID(id); - collection->prepareAfterRead(); - - m_inputs.emplace_back(collection, name); - return collection; -} - -void ROOTReader::openFile(const std::string& filename) { - openFiles({filename}); -} - -void ROOTReader::openFiles(const std::vector& filenames) { - m_chain = new TChain("events"); - for (const auto& filename : filenames) { - //-1 forces the headers to be read so that - // the validity of the files can be checked - if (!m_chain->Add(filename.c_str(), -1)) { - delete m_chain; - throw std::runtime_error("File " + filename + " couldn't be found"); - } - } - - // read the meta data and build the collectionBranches cache - // NOTE: This is a small pessimization, if we do not read all collections - // afterwards, but it makes the handling much easier in general - auto metadatatree = static_cast(m_chain->GetFile()->Get("metadata")); - m_table = std::make_shared(); - auto* table = m_table.get(); - metadatatree->SetBranchAddress("CollectionIDs", &table); - - podio::version::Version* versionPtr{nullptr}; - if (auto* versionBranch = root_utils::getBranch(metadatatree, "PodioVersion")) { - versionBranch->SetAddress(&versionPtr); - metadatatree->GetEntry(0); - } - m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; - - // Read the collection type info - // For versions <0.13.1 it does not exist and has to be rebuilt from scratch - if (m_fileVersion < podio::version::Version{0, 13, 1}) { - - std::cout << "PODIO: Reconstructing CollectionTypeInfo branch from other sources in file: \'" - << m_chain->GetFile()->GetName() << "\'" << std::endl; - metadatatree->GetEntry(0); - const auto collectionInfo = root_utils::reconstructCollectionInfo(m_chain, *m_table); - createCollectionBranches(collectionInfo); - - } else if (m_fileVersion < podio::version::Version{0, 16, 4}) { - - auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); - auto collectionInfoWithoutSchema = new std::vector; - auto collectionInfo = new std::vector; - collInfoBranch->SetAddress(&collectionInfoWithoutSchema); - metadatatree->GetEntry(0); - for (const auto& [collID, collType, isSubsetColl] : *collectionInfoWithoutSchema) { - collectionInfo->emplace_back(collID, collType, isSubsetColl, 0); - } - createCollectionBranches(*collectionInfo); - delete collectionInfoWithoutSchema; - delete collectionInfo; - - } else { - - auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); - - auto collectionInfo = new std::vector; - collInfoBranch->SetAddress(&collectionInfo); - metadatatree->GetEntry(0); - createCollectionBranches(*collectionInfo); - delete collectionInfo; - } - - delete versionPtr; -} - -void ROOTReader::closeFile() { - closeFiles(); -} - -void ROOTReader::closeFiles() { - delete m_chain; -} - -void ROOTReader::readEvent() { - m_chain->GetEntry(m_eventNumber); - // first prepare all collections in memory... - for (auto inputs : m_inputs) { - inputs.first->prepareAfterRead(); - } - // ...then clean-up the references between them - // for(auto inputs : m_inputs){ - // inputs.first->setReferences(m_registry); - - // } -} -bool ROOTReader::isValid() const { - return m_chain->GetFile()->IsOpen() && !m_chain->GetFile()->IsZombie(); -} - -void ROOTReader::endOfEvent() { - ++m_eventNumber; - m_inputs.clear(); -} - -unsigned ROOTReader::getEntries() const { - return m_chain->GetEntries(); -} - -void ROOTReader::goToEvent(unsigned eventNumber) { - m_eventNumber = eventNumber; - m_inputs.clear(); -} - -void ROOTReader::createCollectionBranches(const std::vector& collInfo) { - size_t collectionIndex{0}; - - for (const auto& [collID, collType, isSubsetColl, collSchemaVersion] : collInfo) { - // We only write collections that are in the collectionIDTable, so no need - // to check here - const auto name = m_table->name(collID).value(); - - root_utils::CollectionBranches branches{}; - const auto collectionClass = TClass::GetClass(collType.c_str()); - - // Make sure that ROOT actually knows about this datatype before running - // into a potentially cryptic segmentation fault by accessing the nullptr - if (!collectionClass) { - std::cerr << "PODIO: Cannot create the collection type \'" << collType << "\' stored in branch \'" << name - << "\'. Contents of this branch cannot be read." << std::endl; - continue; - } - // Need the collection here to setup all the branches. Have to manage the - // temporary collection ourselves - auto collection = - std::unique_ptr(static_cast(collectionClass->New())); - collection->setSubsetCollection(isSubsetColl); - - if (!isSubsetColl) { - // This branch is guaranteed to exist since only collections that are - // also written to file are in the info metadata that we work with here - branches.data = root_utils::getBranch(m_chain, name.c_str()); - } - - const auto buffers = collection->getBuffers(); - for (size_t i = 0; i < buffers.references->size(); ++i) { - const auto brName = root_utils::refBranch(name, i); - branches.refs.push_back(root_utils::getBranch(m_chain, brName.c_str())); - } - - for (size_t i = 0; i < buffers.vectorMembers->size(); ++i) { - const auto brName = root_utils::vecBranch(name, i); - branches.vecs.push_back(root_utils::getBranch(m_chain, brName.c_str())); - } - - const auto bufferClassName = "std::vector<" + std::string(collection->getDataTypeName()) + ">"; - const auto bufferClass = isSubsetColl ? nullptr : TClass::GetClass(bufferClassName.c_str()); - - m_storedClasses.emplace(name, std::make_tuple(bufferClass, collectionClass, collectionIndex++)); - m_collectionBranches.push_back(branches); - } -} - -} // namespace podio diff --git a/src/ROOTWriter.cc b/src/ROOTWriter.cc deleted file mode 100644 index 73e9f8e1f..000000000 --- a/src/ROOTWriter.cc +++ /dev/null @@ -1,145 +0,0 @@ -#include "rootUtils.h" - -// podio specific includes -#include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/ROOTWriter.h" -#include "podio/podioVersion.h" - -// ROOT specifc includes -#include "TFile.h" -#include "TTree.h" - -namespace podio { -ROOTWriter::ROOTWriter(const std::string& filename, EventStore* store) : - m_filename(filename), - m_store(store), - m_file(new TFile(filename.c_str(), "RECREATE", "data file")), - m_datatree(new TTree("events", "Events tree")), - m_metadatatree(new TTree("metadata", "Metadata tree")), - m_runMDtree(new TTree("run_metadata", "Run metadata tree")), - m_evtMDtree(new TTree("evt_metadata", "Event metadata tree")), - m_colMDtree(new TTree("col_metadata", "Collection metadata tree")) { - - m_evtMDtree->Branch("evtMD", "GenericParameters", m_store->eventMetaDataPtr()); -} - -ROOTWriter::~ROOTWriter() { - delete m_file; -} - -void ROOTWriter::writeEvent() { - std::vector collections; - collections.reserve(m_collectionsToWrite.size()); - for (const auto& name : m_collectionsToWrite) { - const podio::CollectionBase* coll; - m_store->get(name, coll); - collections.emplace_back(name, const_cast(coll)); - collections.back().second->prepareForWrite(); - } - - if (m_firstEvent) { - createBranches(collections); - m_firstEvent = false; - } else { - setBranches(collections); - } - - m_datatree->Fill(); - m_evtMDtree->Fill(); -} - -void ROOTWriter::createBranches(const std::vector& collections) { - for (auto& [name, coll] : collections) { - root_utils::CollectionBranches branches; - const auto collBuffers = coll->getBuffers(); - if (collBuffers.data) { - // only create the data buffer branch if necessary - - const auto collClassName = "vector<" + std::string(coll->getDataTypeName()) + ">"; - - branches.data = m_datatree->Branch(name.c_str(), collClassName.c_str(), collBuffers.data); - } - - // reference collections - if (auto refColls = collBuffers.references) { - int i = 0; - for (auto& c : (*refColls)) { - const auto brName = root_utils::refBranch(name, i); - branches.refs.push_back(m_datatree->Branch(brName.c_str(), c.get())); - ++i; - } - } - - // vector members - if (auto vminfo = collBuffers.vectorMembers) { - int i = 0; - for (auto& [type, vec] : (*vminfo)) { - const auto typeName = "vector<" + type + ">"; - const auto brName = root_utils::vecBranch(name, i); - branches.vecs.push_back(m_datatree->Branch(brName.c_str(), typeName.c_str(), vec)); - ++i; - } - } - - m_collectionBranches.push_back(branches); - } -} - -void ROOTWriter::setBranches(const std::vector& collections) { - size_t iCollection = 0; - for (auto& coll : collections) { - const auto& branches = m_collectionBranches[iCollection]; - root_utils::setCollectionAddresses(coll.second->getBuffers(), branches); - - iCollection++; - } -} - -void ROOTWriter::finish() { - // now we want to safe the metadata. This includes info about the - // collections - const auto collIDTable = m_store->getCollectionIDTable(); - m_metadatatree->Branch("CollectionIDs", collIDTable); - - // collectionID, collection type, subset collection - std::vector collectionInfo; - collectionInfo.reserve(m_collectionsToWrite.size()); - for (const auto& name : m_collectionsToWrite) { - const auto collID = collIDTable->collectionID(name).value(); - const podio::CollectionBase* coll{nullptr}; - // No check necessary, only registered collections possible - m_store->get(name, coll); - const auto collType = coll->getTypeName(); - // const auto collType = "std::vector<" + coll->getDataTypeName() + ">"; - collectionInfo.emplace_back(collID, std::move(collType), coll->isSubsetCollection(), coll->getSchemaVersion()); - } - - m_metadatatree->Branch("CollectionTypeInfo", &collectionInfo); - - podio::version::Version podioVersion = podio::version::build_version; - m_metadatatree->Branch("PodioVersion", &podioVersion); - - m_metadatatree->Fill(); - - m_colMDtree->Branch("colMD", "std::map", m_store->getColMetaDataMap()); - m_colMDtree->Fill(); - m_runMDtree->Branch("runMD", "std::map", m_store->getRunMetaDataMap()); - m_runMDtree->Fill(); - - m_file->Write(); - m_file->Close(); -} - -bool ROOTWriter::registerForWrite(const std::string& name) { - const podio::CollectionBase* tmp_coll(nullptr); - if (!m_store->get(name, tmp_coll)) { - std::cerr << "no such collection to write, throw exception." << std::endl; - return false; - } - - m_collectionsToWrite.push_back(name); - return true; -} - -} // namespace podio diff --git a/src/SIOReader.cc b/src/SIOReader.cc deleted file mode 100644 index 0ad0d9e15..000000000 --- a/src/SIOReader.cc +++ /dev/null @@ -1,241 +0,0 @@ -// podio specific includes -#include "podio/SIOReader.h" - -#include "podio/CollectionBase.h" -#include "podio/CollectionIDTable.h" -#include "podio/EventStore.h" -#include "podio/SIOBlock.h" -#include "sio/definitions.h" - -#include -#include - -namespace podio { - -SIOReader::SIOReader() : - m_eventNumber(0), - m_eventMetaData(std::make_shared()), - m_runMetaData(std::make_shared("RunMetaData")), - m_collectionMetaData(std::make_shared("CollectionMetaData")) { - auto& libLoader [[maybe_unused]] = SIOBlockLibraryLoader::instance(); -} - -CollectionBase* SIOReader::readCollection(const std::string& name) { - if (m_lastEventRead != m_eventNumber) { - readEvent(); - } - - // Have we unpacked this already? - auto p = - std::find_if(begin(m_inputs), end(m_inputs), [&name](const SIOReader::Input& t) { return t.second == name; }); - - if (p != end(m_inputs)) { - p->first->setID(m_table->collectionID(name).value()); - p->first->prepareAfterRead(); - return p->first; - } - - return nullptr; -} - -std::map* SIOReader::readCollectionMetaData() { - // Always read a new map, because the EventStore takes ownership - m_collectionMetaData->data = new ColMDMap(); - readMetaDataRecord(m_collectionMetaData); - return m_collectionMetaData->data; -} - -std::map* SIOReader::readRunMetaData() { - // Always read a new map, because the EventStore takes ownership - m_runMetaData->data = new RunMDMap(); - readMetaDataRecord(m_runMetaData); - return m_runMetaData->data; -} - -podio::GenericParameters* SIOReader::readEventMetaData() { - if (m_lastEventRead != m_eventNumber) { - readEvent(); - } - return m_eventMetaData->metadata; -} - -void SIOReader::openFile(const std::string& filename) { - m_stream.open(filename, std::ios::binary); - if (!this->isValid()) { - throw std::runtime_error("File " + filename + " couldn't be found"); - } - readCollectionIDTable(); - - if (!readFileTOCRecord()) { - reconstructFileTOCRecord(); - } -} - -void SIOReader::closeFile() { - m_stream.close(); -} - -void SIOReader::readEvent() { - // recreate the blocks, since the contents are owned and managed by the - // EventStore - createBlocks(); - - // skip possible intermediate records that are not event data - sio::api::go_to_record(m_stream, "event_record"); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - sio::api::read_blocks(m_unc_buffer.span(), m_blocks); - - for (size_t i = 1; i < m_blocks.size(); ++i) { - auto* blk = static_cast(m_blocks[i].get()); - m_inputs.emplace_back(blk->getCollection(), m_table->names()[i - 1]); - } - - m_lastEventRead = m_eventNumber; -} - -bool SIOReader::isValid() const { - return m_stream.good(); -} - -void SIOReader::endOfEvent() { - ++m_eventNumber; - m_blocks.clear(); - m_inputs.clear(); -} - -void SIOReader::goToEvent(unsigned eventNumber) { - // If we are already past the desired event number, rewind to the start first - if (eventNumber < (unsigned)m_eventNumber) { - m_stream.clear(); - m_stream.seekg(0); - m_eventNumber = 0; - } - - sio::api::go_to_record(m_stream, "event_record"); - if ((eventNumber - m_eventNumber) > 0) { - sio::api::skip_n_records(m_stream, eventNumber - m_eventNumber); - } - m_eventNumber = eventNumber; - - m_inputs.clear(); - m_blocks.clear(); -} - -void SIOReader::createBlocks() { - // make sure that the first block is EventMetaData as it is also the first - // during wrting - m_eventMetaData->metadata = new GenericParameters(); // will be managed by EventStore (?) - m_blocks.push_back(m_eventMetaData); - - for (size_t i = 0; i < m_typeNames.size(); ++i) { - const bool subsetColl = !m_subsetCollectionBits.empty() && m_subsetCollectionBits[i]; - auto blk = podio::SIOBlockFactory::instance().createBlock(m_typeNames[i], m_table->names()[i], subsetColl); - m_blocks.push_back(blk); - } -} - -void SIOReader::readCollectionIDTable() { - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks; - blocks.emplace_back(std::make_shared()); - blocks.emplace_back(std::make_shared()); - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - auto* idTableBlock = static_cast(blocks[0].get()); - m_table = std::make_shared(); - m_table.reset(idTableBlock->getTable()); - m_typeNames = idTableBlock->getTypeNames(); - m_subsetCollectionBits = idTableBlock->getSubsetCollectionBits(); - m_fileVersion = static_cast(blocks[1].get())->version; -} - -void SIOReader::readMetaDataRecord(const std::shared_ptr& mdBlock) { - const auto currPos = m_stream.tellg(); - sio::api::go_to_record(m_stream, mdBlock->name()); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks{}; - blocks.push_back(mdBlock); - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - m_stream.seekg(currPos); -} - -void SIOReader::reconstructFileTOCRecord() { - try { - // use a simple unary predicate that always returns true, and hence skips - // over all records, but as a sideffect populates the tocRecord - sio::api::skip_records(m_stream, [&](const sio::record_info& rec_info) { - m_tocRecord.addRecord(rec_info._name, rec_info._file_start); - return true; - }); - } catch (sio::exception& e) { - if (e.code() != sio::error_code::eof) { - SIO_RETHROW(e, e.code(), e.what()); - } - } - - // rewind to the start of the file - m_stream.clear(); - m_stream.seekg(0); -} - -bool SIOReader::readFileTOCRecord() { - // Check if there is a dedicated marker at the end of the file that tells us - // where the TOC actually starts - m_stream.seekg(-sio_helpers::SIOTocInfoSize, std::ios_base::end); - uint64_t firstWords{0}; - m_stream.read(reinterpret_cast(&firstWords), sizeof(firstWords)); - - const uint32_t marker = (firstWords >> 32) & 0xffffffff; - if (marker == sio_helpers::SIOTocMarker) { - const uint32_t position = firstWords & 0xffffffff; - m_stream.seekg(position); - - sio::record_info rec_info; - sio::api::read_record_info(m_stream, rec_info, m_info_buffer); - sio::api::read_record_data(m_stream, rec_info, m_rec_buffer); - - m_unc_buffer.resize(rec_info._uncompressed_length); - sio::zlib_compression compressor; - compressor.uncompress(m_rec_buffer.span(), m_unc_buffer); - - sio::block_list blocks; - auto tocBlock = std::make_shared(); - tocBlock->record = &m_tocRecord; - blocks.push_back(tocBlock); - - sio::api::read_blocks(m_unc_buffer.span(), blocks); - - m_unc_buffer.clear(); - m_rec_buffer.clear(); - m_stream.seekg(0); - return true; - } - - m_stream.clear(); - m_stream.seekg(0); - return false; -} -} // namespace podio From 8c22306c6f71f7f78b0f960dd04e52ea8d8d3915 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Wed, 11 Oct 2023 17:59:35 +0200 Subject: [PATCH 05/37] Move test case to Frame --- tests/unittests/frame.cpp | 2 ++ tests/unittests/unittest.cpp | 14 -------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/unittests/frame.cpp b/tests/unittests/frame.cpp index cc6be08e1..623b65a56 100644 --- a/tests/unittests/frame.cpp +++ b/tests/unittests/frame.cpp @@ -22,6 +22,8 @@ TEST_CASE("Frame collections", "[frame][basics]") { auto& coll = event.get("clusters"); REQUIRE(coll[0].energy() == 3.14f); REQUIRE(coll[1].energy() == 42.0f); + + REQUIRE_FALSE(event.get("non-existant")); } TEST_CASE("Frame parameters", "[frame][basics]") { diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index febc55686..f9ccc3db0 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -55,20 +55,6 @@ TEST_CASE("AutoDelete", "[basics][memory-management]") { hit3 = hit2; } -TEST_CASE("Basics", "[basics][memory-management]") { - auto store = podio::EventStore(); - // Adding - auto& collection = store.create("name"); - auto hit1 = collection.create(0xcaffeeULL, 0., 0., 0., 0.); // initialize w/ value - auto hit2 = collection.create(); // default initialize - hit2.energy(12.5); - // Retrieving - const ExampleHitCollection* coll2(nullptr); - REQUIRE(store.get("name", coll2)); - const ExampleHitCollection* coll3(nullptr); - REQUIRE_FALSE(store.get("wrongName", coll3)); -} - TEST_CASE("Assignment-operator ref count", "[basics][memory-management]") { // Make sure that the assignment operator handles the reference count // correctly. (Will trigger in an ASan build if it is not the case) From 211ed0a8ad03f0f1864810a987151a31d42be7b5 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Wed, 11 Oct 2023 18:00:24 +0200 Subject: [PATCH 06/37] Remove test case that no longer applies --- tests/unittests/unittest.cpp | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index f9ccc3db0..9912620f8 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -81,31 +81,6 @@ TEST_CASE("ostream-operator", "[basics]") { REQUIRE(sstr.str() == "[not available]"); } -TEST_CASE("Clearing", "[UBSAN-FAIL][ASAN-FAIL][THREAD-FAIL][basics][memory-management]") { - auto store = podio::EventStore(); - auto& hits = store.create("hits"); - auto& clusters = store.create("clusters"); - auto& oneRels = store.create("OneRelation"); - auto nevents = unsigned(1000); - for (unsigned i = 0; i < nevents; ++i) { - hits.clear(); - clusters.clear(); - auto hit1 = hits.create(); - auto hit2 = MutableExampleHit(); - hit1.energy(double(i)); - auto cluster = clusters.create(); - cluster.addHits(hit1); - cluster.addHits(hit2); - hits.push_back(hit2); - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - } - hits.clear(); - REQUIRE(hits.empty()); -} - TEST_CASE("Cloning", "[basics][memory-management]") { bool success = true; auto hit = MutableExampleHit(); @@ -488,11 +463,6 @@ TEST_CASE("UserInitialization", "[basics][code-gen]") { REQUIRE(ex.comp().arr[1] == 3.4); } -TEST_CASE("NonPresentCollection", "[basics][event-store]") { - auto store = podio::EventStore(); - REQUIRE_THROWS_AS(store.get("NonPresentCollection"), std::runtime_error); -} - TEST_CASE("Collection size and empty", "[basics][collections]") { ExampleClusterCollection coll{}; REQUIRE(coll.empty()); @@ -512,11 +482,7 @@ TEST_CASE("const correct indexed access to const collections", "[const-correctne } TEST_CASE("const correct indexed access to collections", "[const-correctness]") { - auto store = podio::EventStore(); - auto& collection = store.create("irrelevant name"); - - STATIC_REQUIRE(std::is_same_v); // collection created by store should not - // be const + auto collection = ExampleHitCollection(); STATIC_REQUIRE(std::is_same_v); // non-const collections should have // indexed access to mutable objects From 295fd03e92852bc23cfb279f17a1f0b49f1fce59 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 08:54:55 +0200 Subject: [PATCH 07/37] Remove no longer existing files from being installed --- python/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 273ecd4a4..99f16dc8b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,7 +4,7 @@ SET(podio_PYTHON_INSTALLDIR ${podio_PYTHON_INSTALLDIR} PARENT_SCOPE) set(to_install podio_class_generator.py podio_schema_evolution.py - EventStore.py) + ) install(FILES ${to_install} DESTINATION ${podio_PYTHON_INSTALLDIR}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e82ed26d..b284abbd8 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,8 +153,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libpodioDict_rdict.pcm ${CMAKE_CURRENT_BINARY_DIR}/podioRootIODictDict.rootmap ${CMAKE_CURRENT_BINARY_DIR}/libpodioRootIODict_rdict.pcm - ${CMAKE_CURRENT_BINARY_DIR}/podioPythonStoreDictDict.rootmap - ${CMAKE_CURRENT_BINARY_DIR}/libpodioPythonStoreDict_rdict.pcm DESTINATION "${CMAKE_INSTALL_LIBDIR}") if (ENABLE_SIO) From 8b185b8fe7b702ef8ebdcaa45d7b5a1bbeab452b Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 09:23:06 +0200 Subject: [PATCH 08/37] Switch relation_range test to Frame based I/O --- tests/root_io/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index f2dae7868..961ee8a3a 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -1,5 +1,5 @@ set(root_dependent_tests - # relation_range.cpp + relation_range.cpp read_and_write_associated.cpp read_frame_root.cpp write_frame_root.cpp From ca555f6d8e23653e430f3c5f6070d2b8859e62de Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 09:54:19 +0200 Subject: [PATCH 09/37] Adapt legacy reader test for root No longer depend on writing the legacy file in the process --- tests/root_io/CMakeLists.txt | 9 +++------ tests/root_io/read_frame_legacy_root.cpp | 18 +++++------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index 961ee8a3a..aec1d46d6 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -3,9 +3,8 @@ set(root_dependent_tests read_and_write_associated.cpp read_frame_root.cpp write_frame_root.cpp - read_frame_legacy_root.cpp - read_frame_root_multiple.cpp read_python_frame_root.cpp + read_frame_root_multiple.cpp read_and_write_frame_root.cpp ) if(ENABLE_RNTUPLE) @@ -21,10 +20,6 @@ foreach( sourcefile ${root_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${root_libs}") endforeach() - -#--- set some dependencies between the different tests to ensure input generating ones are run first -set_property(TEST read_frame_legacy_root PROPERTY DEPENDS write) - set_tests_properties( read_frame_root read_frame_root_multiple @@ -38,6 +33,8 @@ if(ENABLE_RNTUPLE) set_property(TEST read_rntuple PROPERTY DEPENDS write_rntuple) endif() +add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) +target_link_libraries(read_frame_legacy_root PRIVATE "${root_libs}") message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") add_executable(read-legacy-files-root read-legacy-files-root.cpp) diff --git a/tests/root_io/read_frame_legacy_root.cpp b/tests/root_io/read_frame_legacy_root.cpp index f6c410220..6d98f79ac 100644 --- a/tests/root_io/read_frame_legacy_root.cpp +++ b/tests/root_io/read_frame_legacy_root.cpp @@ -6,25 +6,17 @@ #include int main(int argc, char* argv[]) { - std::string inputFile = "example.root"; - bool assertBuildVersion = true; - if (argc == 2) { - inputFile = argv[1]; - assertBuildVersion = false; + if (argc != 2) { + std::cerr << "usage: read_frame_legacy_frame inputfile" << std::endl; + return 1; } + const std::string inputFile = argv[1]; auto reader = podio::ROOTLegacyReader(); try { reader.openFile(inputFile); } catch (const std::runtime_error& e) { - std::cout << "File (" << inputFile << ")could not be opened, aborting." << std::endl; - return 1; - } - - if (assertBuildVersion && reader.currentFileVersion() != podio::version::build_version) { - std::cerr << "The podio build version could not be read back correctly. " - << "(expected:" << podio::version::build_version << ", actual: " << reader.currentFileVersion() << ")" - << std::endl; + std::cout << "File (" << inputFile << ") could not be opened, aborting." << std::endl; return 1; } From 9fa7477f4ab950b3aa42fdf92db9c4c9b6f0fe60 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 10:11:45 +0200 Subject: [PATCH 10/37] Adapt the legacy tests for sio --- tests/sio_io/CMakeLists.txt | 16 ++++++++++++++-- tests/sio_io/read_frame_legacy_sio.cpp | 12 +++++++++--- tests/sio_io/read_frame_sio.cpp | 13 ++++++++++--- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index 802746331..4345af654 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -1,7 +1,6 @@ set(sio_dependent_tests read_frame_sio.cpp write_frame_sio.cpp - read_frame_legacy_sio.cpp read_and_write_frame_sio.cpp read_python_frame_sio.cpp ) @@ -10,7 +9,6 @@ foreach( sourcefile ${sio_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${sio_libs}") endforeach() - set_tests_properties( read_frame_sio read_and_write_frame_sio @@ -25,3 +23,17 @@ set_tests_properties( add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio sio_io.Writer) PODIO_SET_TEST_ENV(write_python_frame_sio) set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio) + +add_executable(read_frame_legacy_sio read_frame_legacy_sio.cpp) +target_link_libraries(read_frame_legacy_sio PRIVATE "${sio_libs}" TestDataModel) + +# If the variable is cached and defined now, we have inputs and can add the +# legacy file read test +if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) + message(STATUS "Using test inputs stored in: " ${PODIO_TEST_INPUT_DATA_DIR}) + + foreach(version IN LISTS "00-16-05;00-16-06") + ADD_PODIO_LEGACY_TEST(${version} read_frame_sio example_frame.sio legacy_test_cases) + ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_sio example.sio legacy_test_cases) + endforeach() +endif() diff --git a/tests/sio_io/read_frame_legacy_sio.cpp b/tests/sio_io/read_frame_legacy_sio.cpp index b8fbf009f..6809d13c0 100644 --- a/tests/sio_io/read_frame_legacy_sio.cpp +++ b/tests/sio_io/read_frame_legacy_sio.cpp @@ -5,12 +5,18 @@ #include -int main() { +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cerr << "usage: read_frame_legacy_sio inputfile" << std::endl; + return 1; + } + + const auto inputFile = argv[1]; auto reader = podio::SIOLegacyReader(); try { - reader.openFile("example.sio"); + reader.openFile(inputFile); } catch (const std::runtime_error& e) { - std::cout << "File could not be opened, aborting." << std::endl; + std::cout << "File (" << inputFile << ") could not be opened, aborting." << std::endl; return 1; } diff --git a/tests/sio_io/read_frame_sio.cpp b/tests/sio_io/read_frame_sio.cpp index 5f69d4da1..625a7748e 100644 --- a/tests/sio_io/read_frame_sio.cpp +++ b/tests/sio_io/read_frame_sio.cpp @@ -3,7 +3,14 @@ #include "podio/SIOFrameReader.h" -int main() { - return read_frames("example_frame.sio") + - test_frame_aux_info("example_frame.sio"); +int main(int argc, char* argv[]) { + std::string inputFile = "example_frame.sio"; + bool assertBuildVersion = true; + if (argc == 2) { + inputFile = argv[1]; + assertBuildVersion = false; + } + + return read_frames(inputFile, assertBuildVersion) + + test_frame_aux_info(inputFile); } From 8b83519c03ff6d92d9270368936c77ed6acea74a Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 11:49:33 +0200 Subject: [PATCH 11/37] Switch to downloaded legacy inputs for podio-dump tests --- tools/CMakeLists.txt | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 06c5118fe..a01716735 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -19,22 +19,33 @@ if(BUILD_TESTING) PODIO_SET_TEST_ENV(${name}) set_tests_properties(${name} PROPERTIES - DEPENDS ${depends_on} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} ) + if (depends_on) + set_tests_properties(${name} PROPERTIES + DEPENDS ${depends_on} + ) + endif() endfunction() CREATE_DUMP_TEST(podio-dump-help _dummy_target_ --help) - CREATE_DUMP_TEST(podio-dump-root-legacy "write" ${PROJECT_BINARY_DIR}/tests/root_io/example.root) CREATE_DUMP_TEST(podio-dump-root "write_frame_root" ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) CREATE_DUMP_TEST(podio-dump-detailed-root "write_frame_root" --detailed --category other_events --entries 2:3 ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) - CREATE_DUMP_TEST(podio-dump-detailed-root-legacy "write" --detailed --entries 2:3 ${PROJECT_BINARY_DIR}/tests/root_io/example.root) + + + if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) + CREATE_DUMP_TEST(podio-dump-root-legacy "" ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.root) + CREATE_DUMP_TEST(podio-dump-detailed-root-legacy "" --detailed --entries 2:3 ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.root) + endif() if (ENABLE_SIO) - CREATE_DUMP_TEST(podio-dump-sio-legacy "write_sio" ${PROJECT_BINARY_DIR}/tests/sio_io/example.sio) CREATE_DUMP_TEST(podio-dump-sio "write_frame_sio" --entries 4:7 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) CREATE_DUMP_TEST(podio-dump-detailed-sio "write_frame_sio" --detailed --entries 9 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) - CREATE_DUMP_TEST(podio-dump-detailed-sio-legacy "write_sio" --detailed --entries 9 ${PROJECT_BINARY_DIR}/tests/sio_io/example.sio) + + if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) + CREATE_DUMP_TEST(podio-dump-sio-legacy "" ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.sio) + CREATE_DUMP_TEST(podio-dump-detailed-sio-legacy "" --detailed --entries 2:3 ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.sio) + endif() endif() if (ENABLE_RNTUPLE) From 184c151e8dc28d2618695d7da1fc505596ec9a48 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:36:31 +0200 Subject: [PATCH 12/37] Switch pyunittests to use downloaded legacy data --- cmake/podioTest.cmake | 1 + python/podio/test_ReaderRoot.py | 3 ++- python/podio/test_ReaderSio.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmake/podioTest.cmake b/cmake/podioTest.cmake index fb92ddce7..6440d5b20 100644 --- a/cmake/podioTest.cmake +++ b/cmake/podioTest.cmake @@ -14,6 +14,7 @@ function(PODIO_SET_TEST_ENV test) PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT} PODIO_BASE=${PROJECT_SOURCE_DIR} ENABLE_SIO=${ENABLE_SIO} + PODIO_TEST_INPUT_DATA_DIR=${PODIO_TEST_INPUT_DATA_DIR} ) endfunction() diff --git a/python/podio/test_ReaderRoot.py b/python/podio/test_ReaderRoot.py index a7bdf98f8..7c956f24a 100644 --- a/python/podio/test_ReaderRoot.py +++ b/python/podio/test_ReaderRoot.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Python unit tests for the ROOT backend (using Frames)""" +import os import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error @@ -19,4 +20,4 @@ class RootLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): """Setup a reader, reading from the example files""" - self.reader = LegacyReader('root_io/example.root') + self.reader = LegacyReader(os.path.join(os.environ["PODIO_TEST_INPUT_DATA_DIR"], "v00-16-06", "example.root")) diff --git a/python/podio/test_ReaderSio.py b/python/podio/test_ReaderSio.py index ef7b86b6b..d8db8b9bf 100644 --- a/python/podio/test_ReaderSio.py +++ b/python/podio/test_ReaderSio.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Python unit tests for the SIO backend (using Frames)""" +import os import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error @@ -22,4 +23,4 @@ class SIOLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): def setUp(self): """Setup a reader, reading from the example files""" from podio.sio_io import LegacyReader # pylint: disable=import-outside-toplevel - self.reader = LegacyReader('sio_io/example.sio') + self.reader = LegacyReader(os.path.join(os.environ["PODIO_TEST_INPUT_DATA_DIR"], "v00-16-06", "example.sio")) From fff8dada623cb47238b25881ddfb66b985277803 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:40:43 +0200 Subject: [PATCH 13/37] Remove more mentions of EventStore --- include/podio/ROOTFrameReader.h | 1 - include/podio/ROOTLegacyReader.h | 1 - 2 files changed, 2 deletions(-) diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index c152c3772..b7a8d1c46 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -32,7 +32,6 @@ namespace detail { } // namespace detail -class EventStore; class CollectionBase; class CollectionIDTable; class GenericParameters; diff --git a/include/podio/ROOTLegacyReader.h b/include/podio/ROOTLegacyReader.h index 2a2a8b621..71894debd 100644 --- a/include/podio/ROOTLegacyReader.h +++ b/include/podio/ROOTLegacyReader.h @@ -29,7 +29,6 @@ namespace detail { using CollectionInfo = std::tuple; } // namespace detail -class EventStore; class CollectionBase; class CollectionIDTable; class GenericParameters; From 0467a99e63ed4997f4f867edef5bacdb64251bdb Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:40:52 +0200 Subject: [PATCH 14/37] Remove now unused files --- tests/read.py | 21 --- tests/write_test.h | 346 --------------------------------------------- 2 files changed, 367 deletions(-) delete mode 100644 tests/read.py delete mode 100644 tests/write_test.h diff --git a/tests/read.py b/tests/read.py deleted file mode 100644 index 3428b3c60..000000000 --- a/tests/read.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Integration test for python interface when reading a file with a podio -generated datamodel -""" - -from __future__ import absolute_import, unicode_literals, print_function - -from podio.EventStore import EventStore - -if __name__ == '__main__': - - FILENAME = 'example.root' - store = EventStore([FILENAME]) # pylint: disable=invalid-name # too strict before 2.5.0 - for i, event in enumerate(store): - if i % 1000 == 0: - print('reading event', i) - evinfo = store.get("info")[0] - clusters = store.get("clusters") - for cluster in clusters: - for ihit in range(cluster.Hits_size()): - hit = cluster.Hits(ihit) - print(' Referenced hit has an energy of', hit.energy()) diff --git a/tests/write_test.h b/tests/write_test.h deleted file mode 100644 index 492fb049b..000000000 --- a/tests/write_test.h +++ /dev/null @@ -1,346 +0,0 @@ -#ifndef PODIO_TESTS_WRITE_TEST_H // NOLINT(llvm-header-guard): folder structure not suitable -#define PODIO_TESTS_WRITE_TEST_H // NOLINT(llvm-header-guard): folder structure not suitable - -// Data model -#include "datamodel/EventInfoCollection.h" -#include "datamodel/ExampleClusterCollection.h" -#include "datamodel/ExampleHitCollection.h" -#include "datamodel/ExampleMCCollection.h" -#include "datamodel/ExampleReferencingTypeCollection.h" -#include "datamodel/ExampleWithARelationCollection.h" -#include "datamodel/ExampleWithArrayCollection.h" -#include "datamodel/ExampleWithComponentCollection.h" -#include "datamodel/ExampleWithFixedWidthIntegersCollection.h" -#include "datamodel/ExampleWithNamespaceCollection.h" -#include "datamodel/ExampleWithOneRelationCollection.h" -#include "datamodel/ExampleWithVectorMemberCollection.h" - -#include "podio/EventStore.h" -#include "podio/UserDataCollection.h" - -// STL -#include -#include -#include -#include - -template -void write(podio::EventStore& store, WriterT& writer) { - std::cout << "start processing" << std::endl; - - auto& info = store.create("info"); - auto& mcps = store.create("mcparticles"); - auto& moreMCs = store.create("moreMCs"); - auto& mcpsRefs = store.create("mcParticleRefs"); - mcpsRefs.setSubsetCollection(); - auto& hits = store.create("hits"); - auto& hitRefs = store.create("hitRefs"); - hitRefs.setSubsetCollection(); - auto& clusters = store.create("clusters"); - auto& refs = store.create("refs"); - auto& refs2 = store.create("refs2"); - auto& comps = store.create("Component"); - auto& oneRels = store.create("OneRelation"); - auto& vecs = store.create("WithVectorMember"); - auto& namesps = store.create("WithNamespaceMember"); - auto& namesprels = store.create("WithNamespaceRelation"); - auto& cpytest = store.create("WithNamespaceRelationCopy"); - auto& arrays = store.create("arrays"); - auto& fixedWidthInts = store.create("fixedWidthInts"); - auto& usrInts = store.create>("userInts"); - auto& usrDoubles = store.create>("userDoubles"); - - writer.registerForWrite("info"); - writer.registerForWrite("mcparticles"); - writer.registerForWrite("moreMCs"); - writer.registerForWrite("mcParticleRefs"); - writer.registerForWrite("hits"); - writer.registerForWrite("hitRefs"); - writer.registerForWrite("clusters"); - writer.registerForWrite("refs"); - writer.registerForWrite("refs2"); - writer.registerForWrite("Component"); - writer.registerForWrite("OneRelation"); - writer.registerForWrite("WithVectorMember"); - writer.registerForWrite("WithNamespaceMember"); - writer.registerForWrite("WithNamespaceRelation"); - writer.registerForWrite("WithNamespaceRelationCopy"); - writer.registerForWrite("arrays"); - writer.registerForWrite("fixedWidthInts"); - writer.registerForWrite("userInts"); - writer.registerForWrite("userDoubles"); - - unsigned nevents = 2000; - - for (unsigned i = 0; i < nevents; ++i) { - if (i % 1000 == 0) { - std::cout << "processing event " << i << std::endl; - } - - auto item1 = MutableEventInfo(); - item1.Number(i); - info.push_back(item1); - - auto& evtMD = store.getEventMetaData(); - evtMD.setValue("UserEventWeight", (float)100. * i); - std::stringstream ss; - ss << " event_number_" << i; - evtMD.setValue("UserEventName", ss.str()); - evtMD.setValue("SomeVectorData", {1, 2, 3, 4}); - evtMD.setValue("SomeVectorData", {i * 1.1, i * 2.2}); - - auto& colMD = store.getCollectionMetaData(hits.getID()); - colMD.setValue("CellIDEncodingString", "system:8,barrel:3,layer:6,slice:5,x:-16,y:-16"); - - auto hit1 = MutableExampleHit(0xbad, 0., 0., 0., 23. + i); - auto hit2 = MutableExampleHit(0xcaffee, 1., 0., 0., 12. + i); - - hits.push_back(hit1); - hits.push_back(hit2); - - hitRefs.push_back(hit2); - hitRefs.push_back(hit1); - - // ---- add some MC particles ---- - auto mcp0 = MutableExampleMC(); - auto mcp1 = MutableExampleMC(); - auto mcp2 = MutableExampleMC(); - auto mcp3 = MutableExampleMC(); - auto mcp4 = MutableExampleMC(); - auto mcp5 = MutableExampleMC(); - auto mcp6 = MutableExampleMC(); - auto mcp7 = MutableExampleMC(); - auto mcp8 = MutableExampleMC(); - auto mcp9 = MutableExampleMC(); - - mcps.push_back(mcp0); - mcps.push_back(mcp1); - mcps.push_back(mcp2); - mcps.push_back(mcp3); - mcps.push_back(mcp4); - mcps.push_back(mcp5); - mcps.push_back(mcp6); - mcps.push_back(mcp7); - mcps.push_back(mcp8); - mcps.push_back(mcp9); - - auto mcp = mcps[0]; - mcp.adddaughters(mcps[2]); - mcp.adddaughters(mcps[3]); - mcp.adddaughters(mcps[4]); - mcp.adddaughters(mcps[5]); - mcp = mcps[1]; - mcp.adddaughters(mcps[2]); - mcp.adddaughters(mcps[3]); - mcp.adddaughters(mcps[4]); - mcp.adddaughters(mcps[5]); - mcp = mcps[2]; - mcp.adddaughters(mcps[6]); - mcp.adddaughters(mcps[7]); - mcp.adddaughters(mcps[8]); - mcp.adddaughters(mcps[9]); - mcp = mcps[3]; - mcp.adddaughters(mcps[6]); - mcp.adddaughters(mcps[7]); - mcp.adddaughters(mcps[8]); - mcp.adddaughters(mcps[9]); - - //--- now fix the parent relations - // use a range-based for loop here to see if we get mutable objects from the - // begin/end iterators - for (auto mc : mcps) { - for (auto p : mc.daughters()) { - int dIndex = p.getObjectID().index; - auto d = mcps[dIndex]; - d.addparents(p); - } - } - //-------- print relations for debugging: - for (auto p : mcps) { - std::cout << " particle " << p.getObjectID().index << " has daughters: "; - for (auto it = p.daughters_begin(), end = p.daughters_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = p.parents_begin(), end = p.parents_end(); it != end; ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << std::endl; - - // make sure that this does not crash when we do it on an immutable object - ExampleMC constP{p}; - std::cout << "The const particle still has the same relations: daughters: "; - for (auto it = constP.daughters_begin(); it != constP.daughters_end(); ++it) { - std::cout << " " << it->getObjectID().index; - } - std::cout << " and parents: "; - for (auto it = constP.parents_begin(); it != constP.parents_end(); ++it) { - std::cout << " " << it->getObjectID().index; - } - } - //------------------------------- - - // ----------------- create a second MC collection ----------------- - // Can use it to test subset collections that store elements from multiple - // collections - for (const auto&& mc : mcps) { - moreMCs.push_back(mc.clone()); - } - - // ----------------- add all "odd" mc particles into a subset collection - for (auto p : mcps) { - if (p.id().index % 2) { - mcpsRefs.push_back(p); - } - } - // ----------------- add the "even" counterparts from a different collection - for (auto p : moreMCs) { - if (p.id().index % 2 == 0) { - mcpsRefs.push_back(p); - } - } - - if (mcpsRefs.size() != mcps.size()) { - throw std::runtime_error( - "The mcParticleRefs collection should now contain as many elements as the mcparticles collection"); - } - //------------------------------- - - auto cluster = MutableExampleCluster(); - auto clu0 = MutableExampleCluster(); - auto clu1 = MutableExampleCluster(); - - clu0.addHits(hit1); - clu0.energy(hit1.energy()); - clu1.addHits(hit2); - clu1.energy(hit2.energy()); - cluster.addHits(hit1); - cluster.addHits(hit2); - cluster.energy(hit1.energy() + hit2.energy()); - cluster.addClusters(clu0); - cluster.addClusters(clu1); - - clusters.push_back(clu0); - clusters.push_back(clu1); - clusters.push_back(cluster); - - auto ref = MutableExampleReferencingType(); - refs.push_back(ref); - - auto ref2 = MutableExampleReferencingType(); - refs2.push_back(ref2); - - ref.addClusters(cluster); - ref.addRefs(ref2); - - auto comp = MutableExampleWithComponent(); - comp.component().data.x = 0; - comp.component().data.y = 1; - comp.component().data.z = i; - comps.push_back(comp); - - auto cyclic = MutableExampleReferencingType(); - cyclic.addRefs(cyclic); - refs.push_back(cyclic); - - auto oneRel = MutableExampleWithOneRelation(); - oneRel.cluster(cluster); - oneRels.push_back(oneRel); - - // write non-filled relation - auto oneRelEmpty = MutableExampleWithOneRelation(); - oneRels.push_back(oneRelEmpty); - - auto vec = MutableExampleWithVectorMember(); - vec.addcount(i); - vec.addcount(i + 10); - vecs.push_back(vec); - auto vec1 = MutableExampleWithVectorMember(); - vec1.addcount(i + 1); - vec1.addcount(i + 11); - vecs.push_back(vec1); - - for (int j = 0; j < 5; j++) { - auto rel = ex42::MutableExampleWithARelation(); - rel.number(0.5 * j); - auto exWithNamesp = ex42::MutableExampleWithNamespace(); - exWithNamesp.component().x = i; - exWithNamesp.component().y = 1000 * i; - namesps.push_back(exWithNamesp); - if (j != 3) { // also check for empty relations - rel.ref(exWithNamesp); - for (int k = 0; k < 5; k++) { - auto namesp = ex42::MutableExampleWithNamespace(); - namesp.x(3 * k); - namesp.component().y = k; - namesps.push_back(namesp); - rel.addrefs(namesp); - } - } - namesprels.push_back(rel); - } - for (auto&& namesprel : namesprels) { - cpytest.push_back(namesprel.clone()); - } - - std::array arrayTest = {0, 0, 2, 3}; - std::array arrayTest2 = {4, 4, 2 * static_cast(i)}; - NotSoSimpleStruct a; - a.data.p = arrayTest2; - ex2::NamespaceStruct nstruct; - nstruct.x = static_cast(i); - std::array structArrayTest = {nstruct, nstruct, nstruct, nstruct}; - auto array = MutableExampleWithArray(a, arrayTest, arrayTest, arrayTest, arrayTest, structArrayTest); - array.myArray(1, i); - array.arrayStruct(a); - arrays.push_back(array); - - auto maxValues = fixedWidthInts.create(); - maxValues.fixedI16(std::numeric_limits::max()); // 2^(16 - 1) - 1 == 32767 - maxValues.fixedU32(std::numeric_limits::max()); // 2^32 - 1 == 4294967295 - maxValues.fixedU64(std::numeric_limits::max()); // 2^64 - 1 == 18446744073709551615 - auto& maxComp = maxValues.fixedWidthStruct(); - maxComp.fixedUnsigned16 = std::numeric_limits::max(); // 2^16 - 1 == 65535 - maxComp.fixedInteger64 = std::numeric_limits::max(); // 2^(64 -1) - 1 == 9223372036854775807 - maxComp.fixedInteger32 = std::numeric_limits::max(); // 2^(32 - 1) - 1 == 2147483647 - - auto minValues = fixedWidthInts.create(); - minValues.fixedI16(std::numeric_limits::min()); // -2^(16 - 1) == -32768 - minValues.fixedU32(std::numeric_limits::min()); // 0 - minValues.fixedU64(std::numeric_limits::min()); // 0 - auto& minComp = minValues.fixedWidthStruct(); - minComp.fixedUnsigned16 = std::numeric_limits::min(); // 0 - minComp.fixedInteger64 = std::numeric_limits::min(); // -2^(64 - 1) == -9223372036854775808 - minComp.fixedInteger32 = std::numeric_limits::min(); // -2^(32 - 1) == -2147483648 - - auto arbValues = fixedWidthInts.create(); - arbValues.fixedI16(-12345); - arbValues.fixedU32(1234567890); - arbValues.fixedU64(1234567890123456789); - auto& arbComp = arbValues.fixedWidthStruct(); - arbComp.fixedUnsigned16 = 12345; - arbComp.fixedInteger32 = -1234567890; - arbComp.fixedInteger64 = -1234567890123456789ll; - - // add some plain ints as user data - auto& uivec = usrInts; - uivec.resize(i + 1); - int myInt = 0; - for (auto& iu : uivec) { - iu = myInt++; - } - // and some user double values - unsigned nd = 100; - usrDoubles.resize(nd); - for (unsigned id = 0; id < nd; ++id) { - usrDoubles[id] = 42.; - } - - writer.writeEvent(); - store.clearCollections(); - } - - writer.finish(); -} - -#endif // PODIO_TESTS_WRITE_TEST_H From a0592eda3bc4ae9336a0e901c993125097cf8293 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:48:04 +0200 Subject: [PATCH 15/37] Remove EventStore remnants from I/O tests --- tests/read_test.h | 91 ++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 64 deletions(-) diff --git a/tests/read_test.h b/tests/read_test.h index 2aee3fbaf..9bfffe145 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -14,6 +14,7 @@ #include "datamodel/ExampleWithVectorMemberCollection.h" // podio specific includes +#include "podio/Frame.h" #include "podio/UserDataCollection.h" #include "podio/podioVersion.h" @@ -37,19 +38,8 @@ bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std return true; } -template -static constexpr bool isEventStore = false; - -template -void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersion) { - - float evtWeight = -1; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - evtWeight = evtMD.template getValue("UserEventWeight"); - } else { - evtWeight = store.template getParameter("UserEventWeight"); - } +void processEvent(const podio::Frame& store, int eventNum, podio::version::Version fileVersion) { + const float evtWeight = store.getParameter("UserEventWeight"); if (evtWeight != (float)100. * eventNum) { std::cout << " read UserEventWeight: " << evtWeight << " - expected : " << (float)100. * eventNum << std::endl; throw std::runtime_error("Couldn't read event meta data parameters 'UserEventWeight'"); @@ -57,13 +47,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi std::stringstream ss; ss << " event_number_" << eventNum; - std::string evtName = ""; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - evtName = evtMD.template getValue("UserEventName"); - } else { - evtName = store.template getParameter("UserEventName"); - } + const auto& evtName = store.getParameter("UserEventName"); if (evtName != ss.str()) { std::cout << " read UserEventName: " << evtName << " - expected : " << ss.str() << std::endl; @@ -71,13 +55,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion > podio::version::Version{0, 14, 1}) { - std::vector someVectorData{}; - if constexpr (isEventStore) { - const auto& evtMD = store.getEventMetaData(); - someVectorData = evtMD.template getValue>("SomeVectorData"); - } else { - someVectorData = store.template getParameter>("SomeVectorData"); - } + const auto& someVectorData = store.getParameter>("SomeVectorData"); if (someVectorData.size() != 4) { throw std::runtime_error("Couldn't read event meta data parameters: 'SomeVectorData'"); } @@ -88,19 +66,17 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - if constexpr (!isEventStore) { - if (fileVersion > podio::version::Version{0, 16, 2}) { - const auto doubleParams = store.template getParameter>("SomeVectorData"); - if (doubleParams.size() != 2 || doubleParams[0] != eventNum * 1.1 || doubleParams[1] != eventNum * 2.2) { - throw std::runtime_error("Could not read event parameter: 'SomeDoubleValues' correctly"); - } + if (fileVersion > podio::version::Version{0, 16, 2}) { + const auto& doubleParams = store.getParameter>("SomeVectorData"); + if (doubleParams.size() != 2 || doubleParams[0] != eventNum * 1.1 || doubleParams[1] != eventNum * 2.2) { + throw std::runtime_error("Could not read event parameter: 'SomeDoubleValues' correctly"); } } try { // not assigning to a variable, because it will remain unused, we just want // the exception here - store.template get("notthere"); + store.get("notthere"); } catch (const std::runtime_error& err) { if (std::string(err.what()) != "No collection \'notthere\' is present in the EventStore") { throw std::runtime_error("Trying to get non present collection \'notthere' should throw an exception"); @@ -108,23 +84,10 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } // read collection meta data - auto& hits = store.template get("hits"); - if constexpr (isEventStore) { - const auto& colMD = store.getCollectionMetaData(hits.getID()); - const auto& es = colMD.template getValue("CellIDEncodingString"); - if (es != std::string("system:8,barrel:3,layer:6,slice:5,x:-16,y:-16")) { - std::cout << " meta data from collection 'hits' with id = " << hits.getID() - << " read CellIDEncodingString: " << es << " - expected : system:8,barrel:3,layer:6,slice:5,x:-16,y:-16" - << std::endl; - throw std::runtime_error("Couldn't read event meta data parameters 'CellIDEncodingString'"); - } - - } else { - // TODO: Integrate this into the frame workflow somehow - } + auto& hits = store.get("hits"); if (fileVersion > podio::version::Version{0, 14, 0}) { - auto& hitRefs = store.template get("hitRefs"); + auto& hitRefs = store.get("hitRefs"); if (hitRefs.size() != hits.size()) { throw std::runtime_error("hit and subset hit collection do not have the same size"); } @@ -133,7 +96,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& clusters = store.template get("clusters"); + auto& clusters = store.get("clusters"); if (clusters.isValid()) { auto cluster = clusters[0]; for (auto i = cluster.Hits_begin(), end = cluster.Hits_end(); i != end; ++i) { @@ -147,7 +110,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi // Read the mcParticleRefs before reading any of the other collections that // are referenced to make sure that all the necessary relations are handled // correctly - auto& mcpRefs = store.template get("mcParticleRefs"); + auto& mcpRefs = store.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -169,7 +132,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& mcps = store.template get("mcparticles"); + auto& mcps = store.get("mcparticles"); if (!mcps.isValid()) { throw std::runtime_error("Collection 'mcparticles' should be present"); } @@ -237,7 +200,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi // Load the subset collection first to ensure that it pulls in objects taht // have not been read yet - auto& mcpRefs = store.template get("mcParticleRefs"); + auto& mcpRefs = store.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -249,7 +212,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& moreMCs = store.template get("moreMCs"); + auto& moreMCs = store.get("moreMCs"); // First check that the two mc collections that we store are the same if (mcps.size() != moreMCs.size()) { @@ -282,7 +245,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } // std::cout << "Fetching collection 'refs'" << std::endl; - auto& refs = store.template get("refs"); + auto& refs = store.get("refs"); if (refs.isValid()) { auto ref = refs[0]; for (auto cluster : ref.Clusters()) { @@ -294,7 +257,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'refs' should be present"); } // std::cout << "Fetching collection 'OneRelation'" << std::endl; - auto& rels = store.template get("OneRelation"); + auto& rels = store.get("OneRelation"); if (rels.isValid()) { // std::cout << "Referenced object has an energy of " << (*rels)[0].cluster().energy() << std::endl; } else { @@ -302,7 +265,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } // std::cout << "Fetching collection 'WithVectorMember'" << std::endl; - auto& vecs = store.template get("WithVectorMember"); + auto& vecs = store.get("WithVectorMember"); if (vecs.isValid()) { if (vecs.size() != 2) { throw std::runtime_error("Collection 'WithVectorMember' should have two elements'"); @@ -329,13 +292,13 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'WithVectorMember' should be present"); } - auto& comps = store.template get("Component"); + auto& comps = store.get("Component"); if (comps.isValid()) { auto comp = comps[0]; int a [[maybe_unused]] = comp.component().data.x + comp.component().data.z; } - auto& arrays = store.template get("arrays"); + auto& arrays = store.get("arrays"); if (arrays.isValid() && arrays.size() != 0) { auto array = arrays[0]; if (array.myArray(1) != eventNum) { @@ -351,8 +314,8 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi throw std::runtime_error("Collection 'arrays' should be present"); } - auto& nmspaces = store.template get("WithNamespaceRelation"); - auto& copies = store.template get("WithNamespaceRelationCopy"); + auto& nmspaces = store.get("WithNamespaceRelation"); + auto& copies = store.get("WithNamespaceRelationCopy"); auto cpytest = ex42::ExampleWithARelationCollection{}; if (nmspaces.isValid() && copies.isValid()) { @@ -391,7 +354,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion >= podio::version::Version{0, 13, 1}) { - const auto& fixedWidthInts = store.template get("fixedWidthInts"); + const auto& fixedWidthInts = store.get("fixedWidthInts"); if (not fixedWidthInts.isValid() or fixedWidthInts.size() != 3) { throw std::runtime_error("Collection \'fixedWidthInts\' should be present and have 3 elements"); } @@ -425,7 +388,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } if (fileVersion >= podio::version::Version{0, 13, 2}) { - auto& usrInts = store.template get>("userInts"); + auto& usrInts = store.get>("userInts"); if (usrInts.size() != (unsigned)eventNum + 1) { throw std::runtime_error("Could not read all userInts properly (expected: " + std::to_string(eventNum + 1) + @@ -447,7 +410,7 @@ void processEvent(StoreT& store, int eventNum, podio::version::Version fileVersi } } - auto& usrDbl = store.template get>("userDoubles"); + auto& usrDbl = store.get>("userDoubles"); if (usrDbl.size() != 100) { throw std::runtime_error( "Could not read all userDoubles properly (expected: 100, actual: " + std::to_string(usrDbl.size()) + ")"); From 22bd52942ce18944a155e5b72840330e627bc602 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:48:45 +0200 Subject: [PATCH 16/37] Rename store to event --- tests/read_test.h | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/read_test.h b/tests/read_test.h index 9bfffe145..4761df401 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -38,8 +38,8 @@ bool check_fixed_width_value(FixedWidthT actual, FixedWidthT expected, const std return true; } -void processEvent(const podio::Frame& store, int eventNum, podio::version::Version fileVersion) { - const float evtWeight = store.getParameter("UserEventWeight"); +void processEvent(const podio::Frame& event, int eventNum, podio::version::Version fileVersion) { + const float evtWeight = event.getParameter("UserEventWeight"); if (evtWeight != (float)100. * eventNum) { std::cout << " read UserEventWeight: " << evtWeight << " - expected : " << (float)100. * eventNum << std::endl; throw std::runtime_error("Couldn't read event meta data parameters 'UserEventWeight'"); @@ -47,7 +47,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi std::stringstream ss; ss << " event_number_" << eventNum; - const auto& evtName = store.getParameter("UserEventName"); + const auto& evtName = event.getParameter("UserEventName"); if (evtName != ss.str()) { std::cout << " read UserEventName: " << evtName << " - expected : " << ss.str() << std::endl; @@ -55,7 +55,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } if (fileVersion > podio::version::Version{0, 14, 1}) { - const auto& someVectorData = store.getParameter>("SomeVectorData"); + const auto& someVectorData = event.getParameter>("SomeVectorData"); if (someVectorData.size() != 4) { throw std::runtime_error("Couldn't read event meta data parameters: 'SomeVectorData'"); } @@ -67,7 +67,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } if (fileVersion > podio::version::Version{0, 16, 2}) { - const auto& doubleParams = store.getParameter>("SomeVectorData"); + const auto& doubleParams = event.getParameter>("SomeVectorData"); if (doubleParams.size() != 2 || doubleParams[0] != eventNum * 1.1 || doubleParams[1] != eventNum * 2.2) { throw std::runtime_error("Could not read event parameter: 'SomeDoubleValues' correctly"); } @@ -76,7 +76,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi try { // not assigning to a variable, because it will remain unused, we just want // the exception here - store.get("notthere"); + event.get("notthere"); } catch (const std::runtime_error& err) { if (std::string(err.what()) != "No collection \'notthere\' is present in the EventStore") { throw std::runtime_error("Trying to get non present collection \'notthere' should throw an exception"); @@ -84,10 +84,10 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } // read collection meta data - auto& hits = store.get("hits"); + auto& hits = event.get("hits"); if (fileVersion > podio::version::Version{0, 14, 0}) { - auto& hitRefs = store.get("hitRefs"); + auto& hitRefs = event.get("hitRefs"); if (hitRefs.size() != hits.size()) { throw std::runtime_error("hit and subset hit collection do not have the same size"); } @@ -96,7 +96,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } } - auto& clusters = store.get("clusters"); + auto& clusters = event.get("clusters"); if (clusters.isValid()) { auto cluster = clusters[0]; for (auto i = cluster.Hits_begin(), end = cluster.Hits_end(); i != end; ++i) { @@ -110,7 +110,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi // Read the mcParticleRefs before reading any of the other collections that // are referenced to make sure that all the necessary relations are handled // correctly - auto& mcpRefs = store.get("mcParticleRefs"); + auto& mcpRefs = event.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -132,7 +132,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } } - auto& mcps = store.get("mcparticles"); + auto& mcps = event.get("mcparticles"); if (!mcps.isValid()) { throw std::runtime_error("Collection 'mcparticles' should be present"); } @@ -200,7 +200,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi // Load the subset collection first to ensure that it pulls in objects taht // have not been read yet - auto& mcpRefs = store.get("mcParticleRefs"); + auto& mcpRefs = event.get("mcParticleRefs"); if (!mcpRefs.isValid()) { throw std::runtime_error("Collection 'mcParticleRefs' should be present"); } @@ -212,7 +212,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } } - auto& moreMCs = store.get("moreMCs"); + auto& moreMCs = event.get("moreMCs"); // First check that the two mc collections that we store are the same if (mcps.size() != moreMCs.size()) { @@ -245,7 +245,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } // std::cout << "Fetching collection 'refs'" << std::endl; - auto& refs = store.get("refs"); + auto& refs = event.get("refs"); if (refs.isValid()) { auto ref = refs[0]; for (auto cluster : ref.Clusters()) { @@ -257,7 +257,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi throw std::runtime_error("Collection 'refs' should be present"); } // std::cout << "Fetching collection 'OneRelation'" << std::endl; - auto& rels = store.get("OneRelation"); + auto& rels = event.get("OneRelation"); if (rels.isValid()) { // std::cout << "Referenced object has an energy of " << (*rels)[0].cluster().energy() << std::endl; } else { @@ -265,7 +265,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } // std::cout << "Fetching collection 'WithVectorMember'" << std::endl; - auto& vecs = store.get("WithVectorMember"); + auto& vecs = event.get("WithVectorMember"); if (vecs.isValid()) { if (vecs.size() != 2) { throw std::runtime_error("Collection 'WithVectorMember' should have two elements'"); @@ -292,13 +292,13 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi throw std::runtime_error("Collection 'WithVectorMember' should be present"); } - auto& comps = store.get("Component"); + auto& comps = event.get("Component"); if (comps.isValid()) { auto comp = comps[0]; int a [[maybe_unused]] = comp.component().data.x + comp.component().data.z; } - auto& arrays = store.get("arrays"); + auto& arrays = event.get("arrays"); if (arrays.isValid() && arrays.size() != 0) { auto array = arrays[0]; if (array.myArray(1) != eventNum) { @@ -314,8 +314,8 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi throw std::runtime_error("Collection 'arrays' should be present"); } - auto& nmspaces = store.get("WithNamespaceRelation"); - auto& copies = store.get("WithNamespaceRelationCopy"); + auto& nmspaces = event.get("WithNamespaceRelation"); + auto& copies = event.get("WithNamespaceRelationCopy"); auto cpytest = ex42::ExampleWithARelationCollection{}; if (nmspaces.isValid() && copies.isValid()) { @@ -354,7 +354,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } if (fileVersion >= podio::version::Version{0, 13, 1}) { - const auto& fixedWidthInts = store.get("fixedWidthInts"); + const auto& fixedWidthInts = event.get("fixedWidthInts"); if (not fixedWidthInts.isValid() or fixedWidthInts.size() != 3) { throw std::runtime_error("Collection \'fixedWidthInts\' should be present and have 3 elements"); } @@ -388,7 +388,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } if (fileVersion >= podio::version::Version{0, 13, 2}) { - auto& usrInts = store.get>("userInts"); + auto& usrInts = event.get>("userInts"); if (usrInts.size() != (unsigned)eventNum + 1) { throw std::runtime_error("Could not read all userInts properly (expected: " + std::to_string(eventNum + 1) + @@ -410,7 +410,7 @@ void processEvent(const podio::Frame& store, int eventNum, podio::version::Versi } } - auto& usrDbl = store.get>("userDoubles"); + auto& usrDbl = event.get>("userDoubles"); if (usrDbl.size() != 100) { throw std::runtime_error( "Could not read all userDoubles properly (expected: 100, actual: " + std::to_string(usrDbl.size()) + ")"); From 99a5a3c124c9c27f880a7e310c261c3a0da29d5f Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 12:56:12 +0200 Subject: [PATCH 17/37] Remove test case that is covered in unittests --- tests/read_test.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/read_test.h b/tests/read_test.h index 4761df401..cbbc02181 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -73,16 +73,6 @@ void processEvent(const podio::Frame& event, int eventNum, podio::version::Versi } } - try { - // not assigning to a variable, because it will remain unused, we just want - // the exception here - event.get("notthere"); - } catch (const std::runtime_error& err) { - if (std::string(err.what()) != "No collection \'notthere\' is present in the EventStore") { - throw std::runtime_error("Trying to get non present collection \'notthere' should throw an exception"); - } - } - // read collection meta data auto& hits = event.get("hits"); From 9cb049a562f70e3083f0c8139395b5e3d388dbc3 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 14:07:54 +0200 Subject: [PATCH 18/37] Fix clang-tidy complaints --- tests/read_test.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/read_test.h b/tests/read_test.h index cbbc02181..e38be95d5 100644 --- a/tests/read_test.h +++ b/tests/read_test.h @@ -289,7 +289,7 @@ void processEvent(const podio::Frame& event, int eventNum, podio::version::Versi } auto& arrays = event.get("arrays"); - if (arrays.isValid() && arrays.size() != 0) { + if (arrays.isValid() && !arrays.empty()) { auto array = arrays[0]; if (array.myArray(1) != eventNum) { throw std::runtime_error("Array not properly set."); From 5932572df1020d33b12cc56fe04850b51823be5e Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 15:47:36 +0200 Subject: [PATCH 19/37] Adapt test environment based on availability of test data --- cmake/podioTest.cmake | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmake/podioTest.cmake b/cmake/podioTest.cmake index 6440d5b20..3e4f63e16 100644 --- a/cmake/podioTest.cmake +++ b/cmake/podioTest.cmake @@ -3,9 +3,8 @@ function(PODIO_SET_TEST_ENV test) # We need to convert this into a list of arguments that can be used as environment variable list(JOIN PODIO_IO_HANDLERS " " IO_HANDLERS) - set_property(TEST ${test} - PROPERTY ENVIRONMENT - LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$:$<$:$>:$ENV{LD_LIBRARY_PATH} + set(test_environment + LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$:$<$:$>:$ENV{LD_LIBRARY_PATH} PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH} PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests ROOT_INCLUDE_PATH=${PROJECT_BINARY_DIR}/tests/datamodel:${PROJECT_SOURCE_DIR}/include @@ -14,7 +13,15 @@ function(PODIO_SET_TEST_ENV test) PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT} PODIO_BASE=${PROJECT_SOURCE_DIR} ENABLE_SIO=${ENABLE_SIO} - PODIO_TEST_INPUT_DATA_DIR=${PODIO_TEST_INPUT_DATA_DIR} + ) + if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) + list(APPEND test_environment + PODIO_TEST_INPUT_DATA_DIR=${PODIO_TEST_INPUT_DATA_DIR} + ) + endif() + + set_property(TEST ${test} + PROPERTY ENVIRONMENT "${test_environment}" ) endfunction() From 9f5f60ad29069049190152de3cd215767d32d135 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 12 Oct 2023 15:50:56 +0200 Subject: [PATCH 20/37] Skip tests if data not available --- python/podio/test_ReaderRoot.py | 2 ++ python/podio/test_ReaderSio.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/podio/test_ReaderRoot.py b/python/podio/test_ReaderRoot.py index 7c956f24a..c88d97e15 100644 --- a/python/podio/test_ReaderRoot.py +++ b/python/podio/test_ReaderRoot.py @@ -5,6 +5,7 @@ import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error +from podio.test_utils import LEGACY_DATA_AVAILABLE from podio.root_io import Reader, LegacyReader @@ -16,6 +17,7 @@ def setUp(self): self.reader = Reader('root_io/example_frame.root') +@unittest.skipIf(not LEGACY_DATA_AVAILABLE, "no legacy input data available") class RootLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): diff --git a/python/podio/test_ReaderSio.py b/python/podio/test_ReaderSio.py index d8db8b9bf..985ba5149 100644 --- a/python/podio/test_ReaderSio.py +++ b/python/podio/test_ReaderSio.py @@ -5,7 +5,7 @@ import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error -from test_utils import SKIP_SIO_TESTS # pylint: disable=import-error +from test_utils import SKIP_SIO_TESTS, LEGACY_DATA_AVAILABLE # pylint: disable=import-error @unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") @@ -17,7 +17,7 @@ def setUp(self): self.reader = Reader('sio_io/example_frame.sio') -@unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") +@unittest.skipIf(SKIP_SIO_TESTS or not LEGACY_DATA_AVAILABLE, "no SIO support or data not available") class SIOLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): From 59afde66f39084ef076effc091480f37c2a855d3 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 13 Oct 2023 10:35:03 +0200 Subject: [PATCH 21/37] Remove EventStore from UserDataCollection doc --- doc/userdata.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/userdata.md b/doc/userdata.md index 544c79c75..b2df04568 100644 --- a/doc/userdata.md +++ b/doc/userdata.md @@ -6,17 +6,18 @@ data* via the `podio::UserDataCollection`. It gives the user access to a the data stored in the EDM classes for each event. ## Example usage -Creating or getting a `UserDataCollection` via the `EventStore` works the same -as with any other collection of the EDM via the `create` or `get` functions: +Creating or getting a `UserDataCollection` via the `Frame` works the same +as with any other collection of the EDM via the `put` or `get` functions: ```cpp #include "podio/UserDataCollection.h" -// Create a collection -auto& userFloats = store.create>("userFloats"); +// Create a collection and put it into a Frame +userFloats = podio::UserDataCollection(); +frame.put(std::move(userFloats), "userFloats"); // get a collection -const auto& userData = store.get>("userFloats"); +const auto& userData = frame.get>("userFloats"); ``` The interface of the `UserDataCollection` is similar to a basic version of the From a91e5193cae1fe57459495d4eede3a195b41c44f Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 13 Oct 2023 10:40:21 +0200 Subject: [PATCH 22/37] Update backend documentation to remove EventStore mentions --- doc/advanced_topics.md | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index c2235f565..2bab327cc 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -32,26 +32,14 @@ Before writing out a collection, the data need to be put into the proper structu ### Reading Back-End -There are two possibilities to implement a reading-back end. In case one uses the `podio::EventStore`, one simply has to implement the `IReader` interface. - -If not taking advantage of this implementation, the data reader or the event store have to implement the `ICollectionProvider` interface. Reading of a collection happens then similar to: - -```cpp - // ... - // your creation of the collection and reading of the PODs from disk - // ... - collection->setBuffer(buffer); - auto refCollections = collection->referenceCollections(); - // ... - // your filling of refCollections from disk - // ... - collection->setID( ); - collection->prepareAfterRead(); - // ... - collection->setReferences( &collectionProvider ); -``` - -The strong assumption here is that all references are being followed up directly and no later on-demand reading is done. +The main requirement for a reading backend is its capability of reading back all +the necessary data from which a collection can be constructed in the form of +`podio::CollectionReadBuffers`. From thes buffers collections can then be +constructed. Each instance has to contain the (type erased) POD buffers (as a +`std::vector`), the (possibly empty) vectors of `podio::ObjectID`s that contain +the relation information as well the (possibly empty) vectors for the vector +member buffers, which are currently stored as pairs of the type (as a +`std::string`) and (type erased) data buffers in the form of `std::vector`s. ### Dumping JSON From 412da04437b6876e7e9e58233d18e9fe5cf71566 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 13 Oct 2023 10:47:04 +0200 Subject: [PATCH 23/37] Update main documentation to remove EventStore --- doc/advanced_topics.md | 9 ++---- doc/examples.md | 67 ++++++++++++------------------------------ 2 files changed, 22 insertions(+), 54 deletions(-) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index 2bab327cc..e7ffef0c0 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -82,12 +82,9 @@ As explained in the section about mutability of data, thread-safety is only guar During the calls of `prepareForWriting` and `prepareAfterReading` on collections other operations like object creation or addition will lead to an inconsistent state. ### Not-thread-safe components -The example event store provided with PODIO is as of writing not thread-safe. Neither is the chosen serialization. - -## Implementing a transient Event Class - -PODIO contains one example `podio::EventStore` class. -To implement your own transient event store, the only requirement is to set the collectionID of each collection to a unique ID on creation. +The Readers and Writers that ship with podio are assumed to run on a single +thread only (more precisely we assume that each Reader or Writer doesn't have to +synchronize with any other for file operations). ## Running pre-commit diff --git a/doc/examples.md b/doc/examples.md index 49f91b856..903980a80 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -102,28 +102,30 @@ Passing in a size argument is optional; If no argument is passed all elements wi if an argument is passed only as many elements as requested will be returned. If the collection holds less elements than are requested, only as elements as are available will be returned. -### EventStore functionality +### `podio::Frame` container -The event store contained in the package is for *educational* purposes and kept very minimal. It has two main methods: +The `podio::Frame` is the main container for containing and grouping collections +together. It has two main methods: ```cpp - /// create a new collection + /// Store a collection template - T& create(const std::string& name); + const T& put(T&& coll, const std::string& name); - /// access a collection. + /// access a collection template - const T& get(const std::string& name); + const& T get(const std::string& name); ``` -Please note that a `put` method for collections is not foreseen. +Note that for `put`ting collections into the Frame an explicit `std::move` is +necessary to highlight the change of ownership that happens in this case. ### Object Retrieval Collections can be retrieved explicitly: ```cpp - auto& hits = store.get("hits"); + auto& hits = frame.get("hits"); if (hits.isValid()) { ... } ``` @@ -135,51 +137,20 @@ Or implicitly when following an object reference. In both cases the access to da Sometimes it is necessary or useful to store additional data that is not directly foreseen in the EDM. This could be configuration parameters of simulation jobs, or parameter descriptions like cell-ID encoding etc. PODIO currently allows to store such meta data in terms of a `GenericParameters` class that holds an arbitrary number of named parameters of type `int, float, string` or vectors if these. -Meta data can be stored and retrieved from the `EventStore` for runs, collections and events via -the three methods: -```cpp -virtual GenericParameters& EventStore::getRunMetaData(int runID); -virtual GenericParameters& EventStore::getEventMetaData(); -virtual GenericParameters& EventStore::getCollectionMetaData(int colID); -``` - -- example for writing event data: -```cpp -auto& evtMD = store.getEventMetaData() ; -evtMD.setValue( "UserEventWeight" , (float) 100.*i ) ; -``` -- example for reading event data: -```cpp -auto& evtMD = store.getEventMetaData() ; -float evtWeight = evtMD.getFloatVal( "UserEventWeight" ) ; - -``` - -- example for writing collection meta data: - -```cpp -auto& hits = store.create("hits"); -// ... -auto& colMD = store.getCollectionMetaData( hits.getID() ); -colMD.setValue("CellIDEncodingString","system:8,barrel:3,layer:6,slice:5,x:-16,y:-16"); -``` - -- example for reading collection meta data - -```cpp -auto colMD = store.getCollectionMetaData( hits.getID() ); -std::string es = colMD.getStringVal("CellIDEncodingString") ; -``` - +Meta data can be stored and retrieved from the `Frame` via the templated `putParameter` and `getParameter` methods. #### Python Interface -The class `EventStore` provides all the necessary (read) access to event files. It can be used as follows: +The `Reader` and `Writer` classes in the `root_io` and `sio_io` submodules +provide all the necessary functionality to read and write event files. An +example of reading files looks like this: + ```python - from EventStore import EventStore - store = EventStore() - for event in store: + from podio.root_io import Reader + + reader = Reader("one or many input files") + for event in reader.get("events"): hits = store.get("hits") for hit in hits: # ... From bf7672ec35d20f9573412511be6a326e6426a148 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Fri, 13 Oct 2023 11:13:28 +0200 Subject: [PATCH 24/37] Remove spurious whitespace --- doc/examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples.md b/doc/examples.md index 903980a80..b883ca06f 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -149,7 +149,7 @@ example of reading files looks like this: ```python from podio.root_io import Reader - reader = Reader("one or many input files") + reader = Reader("one or many input files") for event in reader.get("events"): hits = store.get("hits") for hit in hits: From 734c96e1f82ca3e7f77319205fc51cd8dcfbb310 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Fri, 13 Oct 2023 13:31:47 +0200 Subject: [PATCH 25/37] Remove even more spurious whitespace --- doc/examples.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/examples.md b/doc/examples.md index b883ca06f..568563e0b 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -148,7 +148,6 @@ example of reading files looks like this: ```python from podio.root_io import Reader - reader = Reader("one or many input files") for event in reader.get("events"): hits = store.get("hits") From c602c4a27d5347492fe0b3b23ad2ae6e333c4a5c Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 8 Dec 2023 16:24:17 +0100 Subject: [PATCH 26/37] Remove nonexistant test again after rebase --- tests/root_io/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index aec1d46d6..8aea717c7 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -37,8 +37,6 @@ add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) target_link_libraries(read_frame_legacy_root PRIVATE "${root_libs}") message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") -add_executable(read-legacy-files-root read-legacy-files-root.cpp) -target_link_libraries(read-legacy-files-root PRIVATE TestDataModel TestDataModelDict podio::podioRootIO) # Add a legacy test case based on a base executable and a version for which an # input file exists @@ -55,10 +53,7 @@ macro(ADD_PODIO_LEGACY_TEST version base_test input_file) ) endmacro() -ADD_PODIO_LEGACY_TEST(v00-13 read-legacy-files-root v00-13-example.root legacy_test_cases) - foreach(version IN LISTS legacy_test_versions) - ADD_PODIO_LEGACY_TEST(${version} read-legacy-files-root ${version}-example.root legacy_test_cases) ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root legacy_test_cases) ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root legacy_test_cases) endforeach() From 4fc18c2bd21592b09e56051a85673eba53c50482 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 10:21:15 +0100 Subject: [PATCH 27/37] [format]: Fix style comments --- python/podio/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/podio/__init__.py b/python/podio/__init__.py index 893758510..4c68b7a58 100644 --- a/python/podio/__init__.py +++ b/python/podio/__init__.py @@ -4,21 +4,21 @@ # Try to load podio, this is equivalent to trying to load libpodio.so and will # error if libpodio.so is not found but work if it's found try: - from ROOT import podio # noqa: F401 + from ROOT import podio # noqa: F401 except ImportError: - print("Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH") - raise + print("Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH") + raise from .frame import Frame from . import root_io, reading try: - # We try to import the sio bindings which may fail if ROOT is not able to - # load the dictionary. In this case they have most likely not been built and - # we just move on - from . import sio_io + # We try to import the sio bindings which may fail if ROOT is not able to + # load the dictionary. In this case they have most likely not been built and + # we just move on + from . import sio_io except ImportError: - pass + pass __all__ = [ "__version__", @@ -26,4 +26,4 @@ "root_io", "sio_io", "reading", -] + ] From 40577cf8d8e8387fb64ede1a8e500b2766c1aa30 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 11:42:05 +0100 Subject: [PATCH 28/37] Make python unittests work again --- cmake/podioTest.cmake | 7 +------ python/podio/test_ReaderRoot.py | 6 ++---- python/podio/test_ReaderSio.py | 7 +++---- python/podio/test_utils.py | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cmake/podioTest.cmake b/cmake/podioTest.cmake index 3e4f63e16..804e0822b 100644 --- a/cmake/podioTest.cmake +++ b/cmake/podioTest.cmake @@ -13,13 +13,8 @@ function(PODIO_SET_TEST_ENV test) PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT} PODIO_BASE=${PROJECT_SOURCE_DIR} ENABLE_SIO=${ENABLE_SIO} + PODIO_BUILD_BASE=${PROJECT_BINARY_DIR} ) - if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) - list(APPEND test_environment - PODIO_TEST_INPUT_DATA_DIR=${PODIO_TEST_INPUT_DATA_DIR} - ) - endif() - set_property(TEST ${test} PROPERTY ENVIRONMENT "${test_environment}" ) diff --git a/python/podio/test_ReaderRoot.py b/python/podio/test_ReaderRoot.py index c88d97e15..eeeb2256f 100644 --- a/python/podio/test_ReaderRoot.py +++ b/python/podio/test_ReaderRoot.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 """Python unit tests for the ROOT backend (using Frames)""" -import os import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error -from podio.test_utils import LEGACY_DATA_AVAILABLE +from podio.test_utils import get_legacy_input from podio.root_io import Reader, LegacyReader @@ -17,9 +16,8 @@ def setUp(self): self.reader = Reader('root_io/example_frame.root') -@unittest.skipIf(not LEGACY_DATA_AVAILABLE, "no legacy input data available") class RootLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): """Setup a reader, reading from the example files""" - self.reader = LegacyReader(os.path.join(os.environ["PODIO_TEST_INPUT_DATA_DIR"], "v00-16-06", "example.root")) + self.reader = LegacyReader(get_legacy_input("v00-16-06-example.root")) diff --git a/python/podio/test_ReaderSio.py b/python/podio/test_ReaderSio.py index 985ba5149..5db31a0da 100644 --- a/python/podio/test_ReaderSio.py +++ b/python/podio/test_ReaderSio.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 """Python unit tests for the SIO backend (using Frames)""" -import os import unittest from test_Reader import ReaderTestCaseMixin, LegacyReaderTestCaseMixin # pylint: disable=import-error -from test_utils import SKIP_SIO_TESTS, LEGACY_DATA_AVAILABLE # pylint: disable=import-error +from podio.test_utils import SKIP_SIO_TESTS, get_legacy_input @unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") @@ -17,10 +16,10 @@ def setUp(self): self.reader = Reader('sio_io/example_frame.sio') -@unittest.skipIf(SKIP_SIO_TESTS or not LEGACY_DATA_AVAILABLE, "no SIO support or data not available") +@unittest.skipIf(SKIP_SIO_TESTS, "no SIO support") class SIOLegacyReaderTestCase(LegacyReaderTestCaseMixin, unittest.TestCase): """Test cases for the legacy root input files and reader.""" def setUp(self): """Setup a reader, reading from the example files""" from podio.sio_io import LegacyReader # pylint: disable=import-outside-toplevel - self.reader = LegacyReader(os.path.join(os.environ["PODIO_TEST_INPUT_DATA_DIR"], "v00-16-06", "example.sio")) + self.reader = LegacyReader(get_legacy_input("v00-16-06-example.sio")) diff --git a/python/podio/test_utils.py b/python/podio/test_utils.py index 948571293..c69de90a6 100644 --- a/python/podio/test_utils.py +++ b/python/podio/test_utils.py @@ -4,3 +4,18 @@ import os SKIP_SIO_TESTS = os.environ.get("SKIP_SIO_TESTS", "1") == "1" + + +def get_legacy_input(filename): + """Try to get a legacy input file by name from the ExternalData that is + fetched by CMake. + + Returns either the absolute path to the actual file or an empty string. + """ + try: + datafile = os.path.join(os.environ["PODIO_BUILD_BASE"], "tests", "input_files", filename) + if os.path.isfile(datafile): + return os.path.abspath(datafile) + except KeyError: + pass + return "" From ab337a55c82032908ce61ef7575e38371ec1717c Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 13:25:49 +0100 Subject: [PATCH 29/37] Make SIO legacy tests run --- tests/CMakeLists.txt | 25 ++++++++++++++++++------- tests/root_io/CMakeLists.txt | 26 ++++++++------------------ tests/sio_io/CMakeLists.txt | 18 +++++++++--------- tests/sio_io/read_frame_legacy_sio.cpp | 7 ------- 4 files changed, 35 insertions(+), 41 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ae0208500..c4ae542bb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,18 +36,27 @@ PODIO_ADD_ROOT_IO_DICT(ExtensionDataModelDict ExtensionDataModel "${ext_headers} PODIO_ADD_SIO_IO_BLOCKS(ExtensionDataModel "${ext_headers}" "${ext_sources}") -set(legacy_test_versions - v00-16 - v00-16-02 - v00-16-05 - v00-16-06 -) - include(ExternalData) list(APPEND ExternalData_URL_TEMPLATES "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" ) + +# Add a legacy test case based on a base executable and a version for which an +# input file exists +macro(ADD_PODIO_LEGACY_TEST version base_test input_file) + ExternalData_Add_Test(legacy_test_cases + NAME ${base_test}_${version} + COMMAND ${base_test} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} + ) + set_property(TEST ${base_test}_${version} PROPERTY ENVIRONMENT + LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$ENV{LD_LIBRARY_PATH} + # Clear the ROOT_INCLUDE_PATH for the tests, to avoid potential conflicts + # with existing headers from other installations + ROOT_INCLUDE_PATH= + ) +endmacro() + add_subdirectory(root_io) if (ENABLE_SIO) add_subdirectory(sio_io) @@ -91,3 +100,5 @@ endif() configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY) ExternalData_Add_Target(legacy_test_cases) + +message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index 8aea717c7..d09383e4b 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -36,26 +36,16 @@ endif() add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) target_link_libraries(read_frame_legacy_root PRIVATE "${root_libs}") -message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") - -# Add a legacy test case based on a base executable and a version for which an -# input file exists -macro(ADD_PODIO_LEGACY_TEST version base_test input_file) - ExternalData_Add_Test(legacy_test_cases - NAME ${base_test}_${version} - COMMAND ${base_test} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} - ) - set_property(TEST ${base_test}_${version} PROPERTY ENVIRONMENT - LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$ENV{LD_LIBRARY_PATH} - # Clear the ROOT_INCLUDE_PATH for the tests, to avoid potential conflicts - # with existing headers from other installations - ROOT_INCLUDE_PATH= - ) -endmacro() +set(legacy_test_versions + v00-16 + v00-16-02 + v00-16-05 + v00-16-06 +) foreach(version IN LISTS legacy_test_versions) - ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root legacy_test_cases) - ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root legacy_test_cases) + ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root) + ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root) endforeach() #--- Write via python and the ROOT backend and see if we can read it back in in diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index 4345af654..10a377d71 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -27,13 +27,13 @@ set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio) add_executable(read_frame_legacy_sio read_frame_legacy_sio.cpp) target_link_libraries(read_frame_legacy_sio PRIVATE "${sio_libs}" TestDataModel) -# If the variable is cached and defined now, we have inputs and can add the -# legacy file read test -if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) - message(STATUS "Using test inputs stored in: " ${PODIO_TEST_INPUT_DATA_DIR}) - foreach(version IN LISTS "00-16-05;00-16-06") - ADD_PODIO_LEGACY_TEST(${version} read_frame_sio example_frame.sio legacy_test_cases) - ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_sio example.sio legacy_test_cases) - endforeach() -endif() +set(legacy_test_versions + v00-16-05 + v00-16-06 +) + +foreach(version IN LISTS legacy_test_versions) + ADD_PODIO_LEGACY_TEST(${version} read_frame_sio ${version}-example_frame.sio) + ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_sio ${version}-example.sio) +endforeach() diff --git a/tests/sio_io/read_frame_legacy_sio.cpp b/tests/sio_io/read_frame_legacy_sio.cpp index 6809d13c0..5dbc123d3 100644 --- a/tests/sio_io/read_frame_legacy_sio.cpp +++ b/tests/sio_io/read_frame_legacy_sio.cpp @@ -20,13 +20,6 @@ int main(int argc, char* argv[]) { return 1; } - if (reader.currentFileVersion() != podio::version::build_version) { - std::cerr << "The podio build version could not be read back correctly. " - << "(expected:" << podio::version::build_version << ", actual: " << reader.currentFileVersion() << ")" - << std::endl; - return 1; - } - if (reader.getEntries("events") != 2000) { std::cerr << "Could not read back the number of events correctly. " << "(expected:" << 2000 << ", actual: " << reader.getEntries("events") << ")" << std::endl; From 1cfc878da4ce34732d7426ee6e7c17fe94f54631 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 14:18:13 +0100 Subject: [PATCH 30/37] Switch podio-dump tests to use ExternalData --- CMakeLists.txt | 9 +++++++++ tests/CMakeLists.txt | 8 -------- tools/CMakeLists.txt | 34 +++++++++++++++++++++++++--------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 99fd96e2f..0bbb9bc71 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,11 +191,20 @@ add_subdirectory(src) SET(podio_PYTHON_DIR ${PROJECT_SOURCE_DIR}/python CACHE PATH "Path to the podio python directory") if(BUILD_TESTING) + include(ExternalData) + list(APPEND ExternalData_URL_TEMPLATES + "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" + ) include(cmake/podioTest.cmake) add_subdirectory(tests) endif() add_subdirectory(tools) add_subdirectory(python) + +if(BUILD_TESTING) + # Make sure to fetch all data, after all legacy test cases have been added + ExternalData_Add_Target(legacy_test_cases) +endif() #--- add CMake infrastructure -------------------------------------------------- include(cmake/podioCreateConfig.cmake) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c4ae542bb..8f899a094 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,12 +36,6 @@ PODIO_ADD_ROOT_IO_DICT(ExtensionDataModelDict ExtensionDataModel "${ext_headers} PODIO_ADD_SIO_IO_BLOCKS(ExtensionDataModel "${ext_headers}" "${ext_sources}") -include(ExternalData) -list(APPEND ExternalData_URL_TEMPLATES - "https://key4hep.web.cern.ch:443/testFiles/podio/%(hash)" - ) - - # Add a legacy test case based on a base executable and a version for which an # input file exists macro(ADD_PODIO_LEGACY_TEST version base_test input_file) @@ -99,6 +93,4 @@ endif() # Customize CTest to potentially disable some of the tests with known problems configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY) -ExternalData_Add_Target(legacy_test_cases) - message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index a01716735..2d3a5e13d 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -28,24 +28,40 @@ if(BUILD_TESTING) endif() endfunction() + # Helper function for easily creating "tests" that simply execute podio-dump + # with different arguments on legacy input files. Not crashing is considered + # success. + # + # Args: + # name the name suffix of the test (the actual name will be assembled) + # version the legacy version of the input + # input_file the input file to use + function(CREATE_LEGACY_DUMP_TEST name version input_file) + set(_name podio-dump-legacy_${name}_${version}) + ExternalData_Add_Test(legacy_test_cases + NAME ${_name} + COMMAND ./podio-dump ${ARGN} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} + ) + PODIO_SET_TEST_ENV(${_name}) + + set_tests_properties(${_name} PROPERTIES + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + ) + endfunction() + CREATE_DUMP_TEST(podio-dump-help _dummy_target_ --help) CREATE_DUMP_TEST(podio-dump-root "write_frame_root" ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) CREATE_DUMP_TEST(podio-dump-detailed-root "write_frame_root" --detailed --category other_events --entries 2:3 ${PROJECT_BINARY_DIR}/tests/root_io/example_frame.root) - - if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) - CREATE_DUMP_TEST(podio-dump-root-legacy "" ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.root) - CREATE_DUMP_TEST(podio-dump-detailed-root-legacy "" --detailed --entries 2:3 ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.root) - endif() + CREATE_LEGACY_DUMP_TEST("root" v00-16-06 v00-16-06-example.root) + CREATE_LEGACY_DUMP_TEST("root-detailed" v00-16-06 v00-16-06-example.root --detailed --entries 2:3) if (ENABLE_SIO) CREATE_DUMP_TEST(podio-dump-sio "write_frame_sio" --entries 4:7 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) CREATE_DUMP_TEST(podio-dump-detailed-sio "write_frame_sio" --detailed --entries 9 ${PROJECT_BINARY_DIR}/tests/sio_io/example_frame.sio) - if (DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR}) - CREATE_DUMP_TEST(podio-dump-sio-legacy "" ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.sio) - CREATE_DUMP_TEST(podio-dump-detailed-sio-legacy "" --detailed --entries 2:3 ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-06/example.sio) - endif() + CREATE_LEGACY_DUMP_TEST("sio" v00-16-06 v00-16-06-example.sio) + CREATE_LEGACY_DUMP_TEST("sio-detailed" v00-16-06 v00-16-06-example.sio --detailed --entries 2:3) endif() if (ENABLE_RNTUPLE) From a0fcdfa98b2117235f014e55407a8773b9c0d884 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 16:37:48 +0100 Subject: [PATCH 31/37] Add neessary Frame include --- tests/unittests/unittest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/unittest.cpp b/tests/unittests/unittest.cpp index 9912620f8..1be4b89c5 100644 --- a/tests/unittests/unittest.cpp +++ b/tests/unittests/unittest.cpp @@ -12,6 +12,7 @@ #include "catch2/matchers/catch_matchers_vector.hpp" // podio specific includes +#include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/ROOTFrameReader.h" #include "podio/ROOTFrameWriter.h" From c0423f2ceef256b13457b5bc16193e1e5ec239b0 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 19:23:44 +0100 Subject: [PATCH 32/37] Guard GenericParameter friend-ness for RNTuple support --- include/podio/GenericParameters.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/podio/GenericParameters.h b/include/podio/GenericParameters.h index eeee3696f..8466de9cb 100644 --- a/include/podio/GenericParameters.h +++ b/include/podio/GenericParameters.h @@ -18,10 +18,12 @@ class write_device; using version_type = uint32_t; // from sio/definitions } // namespace sio +#if PODIO_ENABLE_RNTUPLE namespace podio { class ROOTNTupleReader; class ROOTNTupleWriter; } // namespace podio +#endif #define DEPR_NON_TEMPLATE \ [[deprecated("Non-templated access will be removed. Switch to templated access functionality")]] @@ -150,8 +152,11 @@ class GenericParameters { friend void writeGenericParameters(sio::write_device& device, const GenericParameters& parameters); friend void readGenericParameters(sio::read_device& device, GenericParameters& parameters, sio::version_type version); - friend ROOTNTupleReader; + +#if PODIO_ENABLE_RNTUPLE friend ROOTNTupleWriter; + friend ROOTNTupleReader; +#endif /// Get a reference to the internal map for a given type template From 8d7f26e918145fc751550ca64399e7abc89cf5c5 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 19:58:12 +0100 Subject: [PATCH 33/37] Adapt sanitizer ignored test cases after renaming some tests --- tests/CMakeLists.txt | 12 ++++++++++++ tests/CTestCustom.cmake | 15 ++++++++++----- tests/root_io/CMakeLists.txt | 9 +-------- tests/sio_io/CMakeLists.txt | 8 +------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8f899a094..38a44f9ed 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,8 +51,20 @@ macro(ADD_PODIO_LEGACY_TEST version base_test input_file) ) endmacro() +set(root_legacy_test_versions + v00-16 + v00-16-02 + v00-16-05 + v00-16-06 +) + add_subdirectory(root_io) if (ENABLE_SIO) + set(sio_legacy_test_versions + v00-16-05 + v00-16-06 + ) + add_subdirectory(sio_io) endif() add_subdirectory(unittests) diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index 87ac208aa..396c32b78 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -30,15 +30,15 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ pyunittest podio-dump-help - podio-dump-root-legacy podio-dump-root podio-dump-detailed-root - podio-dump-detailed-root-legacy + podio-dump-legacy_root_v00-16-06 + podio-dump-legacy_root-detailed_v00-16-06 - podio-dump-sio-legacy podio-dump-sio podio-dump-detailed-sio - podio-dump-detailed-sio-legacy + podio-dump-legacy_sio_v00-16-06 + podio-dump-legacy_sio-detailed_v00-16-06 datamodel_def_store_roundtrip_root datamodel_def_store_roundtrip_root_extension @@ -49,11 +49,16 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ read_new_data_root ) - foreach(version in @legacy_test_versions@) + foreach(version in @root_legacy_test_versions@) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_root_${version}) list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_legacy_root_${version}) endforeach() + foreach(version in @sio_legacy_test_versions@) + list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_sio_${version}) + list(APPEND CTEST_CUSTOM_TESTS_IGNORE read_frame_legacy_sio_${version}) + endforeach() + # ostream_operator is working with Memory sanitizer (at least locally) if("@USE_SANITIZER@" MATCHES "Memory(WithOrigin)?") list(REMOVE_ITEM CTEST_CUSTOM_TESTS_IGNORE ostream_operator) diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index d09383e4b..1bc906755 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -36,14 +36,7 @@ endif() add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) target_link_libraries(read_frame_legacy_root PRIVATE "${root_libs}") -set(legacy_test_versions - v00-16 - v00-16-02 - v00-16-05 - v00-16-06 -) - -foreach(version IN LISTS legacy_test_versions) +foreach(version IN LISTS root_legacy_test_versions) ADD_PODIO_LEGACY_TEST(${version} read_frame_root ${version}-example_frame.root) ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_root ${version}-example.root) endforeach() diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index 10a377d71..b3987ba5a 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -27,13 +27,7 @@ set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio) add_executable(read_frame_legacy_sio read_frame_legacy_sio.cpp) target_link_libraries(read_frame_legacy_sio PRIVATE "${sio_libs}" TestDataModel) - -set(legacy_test_versions - v00-16-05 - v00-16-06 -) - -foreach(version IN LISTS legacy_test_versions) +foreach(version IN LISTS sio_legacy_test_versions) ADD_PODIO_LEGACY_TEST(${version} read_frame_sio ${version}-example_frame.sio) ADD_PODIO_LEGACY_TEST(${version} read_frame_legacy_sio ${version}-example.sio) endforeach() From 4004079425a36e135cb36cbce52301d7fad16f2e Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 12 Dec 2023 20:54:46 +0100 Subject: [PATCH 34/37] Use usual test environment for legacy tests --- tests/CMakeLists.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 38a44f9ed..f5a17ea33 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,12 +43,7 @@ macro(ADD_PODIO_LEGACY_TEST version base_test input_file) NAME ${base_test}_${version} COMMAND ${base_test} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}} ) - set_property(TEST ${base_test}_${version} PROPERTY ENVIRONMENT - LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$ENV{LD_LIBRARY_PATH} - # Clear the ROOT_INCLUDE_PATH for the tests, to avoid potential conflicts - # with existing headers from other installations - ROOT_INCLUDE_PATH= - ) + PODIO_SET_TEST_ENV(${base_test}_${version}) endmacro() set(root_legacy_test_versions From 65fbe657e640da5e5be7051e571c68cead63008d Mon Sep 17 00:00:00 2001 From: tmadlener Date: Wed, 13 Dec 2023 10:23:16 +0100 Subject: [PATCH 35/37] Remove unnecessary cmake variable --- tools/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 2d3a5e13d..4f4a277db 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -11,11 +11,6 @@ if(BUILD_TESTING) # depends_on the target name of the test that produces the required input file function(CREATE_DUMP_TEST name depends_on) add_test(NAME ${name} COMMAND ./podio-dump ${ARGN}) - - set(SIO_LD_PATH "") - if (ENABLE_SIO) - set(SIO_LD_PATH $) - endif() PODIO_SET_TEST_ENV(${name}) set_tests_properties(${name} PROPERTIES From b58098c7b24edaee5f9598d397c4200d471a60e0 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Wed, 13 Dec 2023 10:33:47 +0100 Subject: [PATCH 36/37] Fix documentation typo --- doc/advanced_topics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index e7ffef0c0..ad8e23118 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -34,7 +34,7 @@ Before writing out a collection, the data need to be put into the proper structu The main requirement for a reading backend is its capability of reading back all the necessary data from which a collection can be constructed in the form of -`podio::CollectionReadBuffers`. From thes buffers collections can then be +`podio::CollectionReadBuffers`. From these buffers collections can then be constructed. Each instance has to contain the (type erased) POD buffers (as a `std::vector`), the (possibly empty) vectors of `podio::ObjectID`s that contain the relation information as well the (possibly empty) vectors for the vector From 901622306f06372b762977f35c352b1d3234b921 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Wed, 13 Dec 2023 10:34:56 +0100 Subject: [PATCH 37/37] Move status message to more appropriate place --- CMakeLists.txt | 1 + tests/CMakeLists.txt | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bbb9bc71..9dcd45ce7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ add_subdirectory(python) if(BUILD_TESTING) # Make sure to fetch all data, after all legacy test cases have been added ExternalData_Add_Target(legacy_test_cases) + message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present") endif() #--- add CMake infrastructure -------------------------------------------------- include(cmake/podioCreateConfig.cmake) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f5a17ea33..b8233c528 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -99,5 +99,3 @@ endif() # Customize CTest to potentially disable some of the tests with known problems configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY) - -message(STATUS "Test inputs will be stored in: ${ExternalData_OBJECT_STORES} if they are not already present")