From 7f2826f4cd05c1309242c6a343b90b94e8c06ad7 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 17 Mar 2023 16:16:46 +0100 Subject: [PATCH 01/42] Add a RNTuple writer --- include/podio/ROOTNTupleWriter.h | 60 +++++++++++ src/ROOTNTupleWriter.cc | 174 +++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 include/podio/ROOTNTupleWriter.h create mode 100644 src/ROOTNTupleWriter.cc diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h new file mode 100644 index 000000000..1927bceee --- /dev/null +++ b/include/podio/ROOTNTupleWriter.h @@ -0,0 +1,60 @@ +#ifndef PODIO_ROOTNTUPLEWRITER_H +#define PODIO_ROOTNTUPLEWRITER_H + +#include "podio/CollectionBase.h" +#include "podio/CollectionBranches.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" + +#include "TBranch.h" +#include "TFile.h" +#include "TChain.h" +#include "podio/Frame.h" +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace rnt = ROOT::Experimental; +// forward declarations +class TFile; +class TTree; +class TChain; + +namespace podio { + +class ROOTNTupleWriter { +public: + ROOTNTupleWriter(const std::string& filename); + // ~ROOTNTupleWriter(); + + ROOTNTupleWriter(const ROOTNTupleWriter&) = delete; + ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; + + void writeFrame(const podio::Frame& frame, const std::string& category); + void writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite); + void finish(); + +private: + + using StoreCollection = std::pair; + std::unique_ptr createModels(const std::vector& collections); + + std::unique_ptr< rnt::RNTupleModel > m_metadata; + rnt::REntry* m_entry; + std::map> m_writers; + + std::unique_ptr m_file; + + std::unique_ptr< rnt::RNTupleWriter > m_metadata_writer; + }; + + DatamodelDefinitionCollector m_datamodelCollector{}; + +} //namespace podio + +#endif //PODIO_ROOTNTUPLEWRITER_H diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc new file mode 100644 index 000000000..923f388e6 --- /dev/null +++ b/src/ROOTNTupleWriter.cc @@ -0,0 +1,174 @@ +#include +#include +#include + +#include "rootUtils.h" + +#include "podio/CollectionBase.h" +#include "podio/EventStore.h" +#include "podio/ROOTWriter.h" +#include "podio/podioVersion.h" + +// ROOT specifc includes +#include "TFile.h" + +#include "podio/ROOTNTupleWriter.h" + +#include "datamodel/ExampleMCData.h" + +namespace podio { + +ROOTNTupleWriter::ROOTNTupleWriter(const std::string& filename) : + m_metadata(nullptr), + m_file(new TFile(filename.c_str(),"RECREATE","data file")) + { + m_metadata = rnt::RNTupleModel::Create(); + + } + +void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { + writeFrame(frame, category, frame.getAvailableCollections()); +} + +void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category, + const std::vector& collsToWrite) { + + std::vector collections; + collections.reserve(collsToWrite.size()); + for (const auto& name : collsToWrite) { + auto* coll = frame.getCollectionForWrite(name); + collections.emplace_back(name, const_cast(coll)); + } + + if (m_writers.find(category) == m_writers.end()) { + auto model = createModels(collections); + m_writers[category] = rnt::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); + } + + m_entry = m_writers[category]->GetModel()->GetDefaultEntry(); + + rnt::RNTupleWriteOptions options; + options.SetCompression(ROOT::RCompressionSetting::EDefaults::kUseGeneralPurpose); + + for (const auto& [name, coll] : collections) { + // coll->prepareForWrite(); + auto collBuffers = coll->getBuffers(); + if (collBuffers.data) { + std::cout << "Capturing unsafe " << name << std::endl; + + // auto v = (std::vector*)collBuffers.data; + // std::cout << "Size is " << v->size() << std::endl; + // std::cout << (*v)[0].energy << std::endl; + // for (auto& x : *v) { + // std::cout << "Inside loop " << std::endl; + // std::cout << x.energy << std::endl; + // } + + m_entry->CaptureValueUnsafe(name, (void*)collBuffers.data); + + // std::cout << "After capturing " << std::endl; + // v = (std::vector*)collBuffers.data; + // std::cout << "Size is " << v->size() << std::endl; + // std::cout << (*v)[0].energy << std::endl; + // for (auto& x : *v) { + // std::cout << "Inside loop " << std::endl; + // std::cout << x.energy << std::endl; + // } + + // auto ptr = (std::vector*)collBuffers.data; + // auto start = (char*)ptr->data(); + // for (int i = 0; i < 13; ++i) { + // printf("%x ", start[i]); + // } + // std::cout << std::endl; + + } + + if (auto refColls = collBuffers.references) { + int i = 0; + for (auto& c : (*refColls)) { + const auto brName = root_utils::refBranch(name, i++); + m_entry->CaptureValueUnsafe(brName, c.get()); + } + } + + // if (auto vmInfo = collBuffers.vectorMembers) { + // int i = 0; + // for (auto& [type, vec] : (*vmInfo)) { + // const auto typeName = "vector<" + type + ">"; + // const auto brName = root_utils::vecBranch(name, i++); + // std::cout << typeName << " " << brName << std::endl; + // auto ptr = (std::vector*)vec; + // std::cout << "Size is " << ptr->size(); + // if (m_first) { + // // m_entry->CaptureValueUnsafe(brName, (void*) vec); + // m_first = false; + // } + // } + // } + } + m_writers[category]->Fill(); +} + +std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { + auto model = rnt::RNTupleModel::Create(); + for (auto& [name, coll] : collections) { + const auto collBuffers = coll->getBuffers(); + + if (collBuffers.data) { + auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; + std::cout << name << " " << collClassName << std::endl; + auto field = rnt::Detail::RFieldBase::Create(name, collClassName).Unwrap(); + model->AddField(std::move(field)); + } + + if (auto refColls = collBuffers.references) { + int i = 0; + for (auto& c : (*refColls)) { + const auto brName = root_utils::refBranch(name, i); + auto collClassName = "vector"; + auto field = rnt::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); + model->AddField(std::move(field)); + ++i; + } + } + + // if (auto vminfo = collBuffers.vectorMembers) { + // int i = 0; + // for (auto& [type, vec] : (*vminfo)) { + // const auto typeName = "vector<" + type + ">"; + // const auto brName = root_utils::vecBranch(name, i); + // auto field = rnt::Detail::RFieldBase::Create(brName, typeName).Unwrap(); + // model->AddField(std::move(field)); + // ++i; + // } + // } + } + model->Freeze(); + return model; +} + +void ROOTNTupleWriter::finish() { + + auto podioVersion = podio::version::build_version; + auto version_field = m_metadata->MakeField>(root_utils::versionBranchName); + *version_field = {podioVersion.major, podioVersion.minor, podioVersion.patch}; + + auto edmDefinitions = m_datamodelCollector.getDatamodelDefinitionsToWrite(); + auto edm_field = m_metadata->MakeField>>(root_utils::edmDefBranchName); + *edm_field = edmDefinitions; + + m_metadata->Freeze(); + m_metadata_writer = rnt::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); + + m_metadata_writer->Fill(); + + m_file->Write(); + + // All the tuple writers have to be deleted before the file so that they flush + // unwritten output + m_writers.clear(); + m_metadata_writer.reset(); +} + +} //namespace podio From 78d1162efd17bc1bb4d01c9171c6555477bacaec Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 20 Mar 2023 13:09:13 +0100 Subject: [PATCH 02/42] Cleanup and add a reader --- include/podio/ROOTNTupleReader.h | 120 +++++++++++++++++++ include/podio/ROOTNTupleWriter.h | 23 ++-- src/ROOTNTupleReader.cc | 193 +++++++++++++++++++++++++++++++ src/ROOTNTupleWriter.cc | 43 ++++++- 4 files changed, 363 insertions(+), 16 deletions(-) create mode 100644 include/podio/ROOTNTupleReader.h create mode 100644 src/ROOTNTupleReader.cc diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h new file mode 100644 index 000000000..30db2db2e --- /dev/null +++ b/include/podio/ROOTNTupleReader.h @@ -0,0 +1,120 @@ +#ifndef PODIO_ROOTNTUPLEREADER_H +#define PODIO_ROOTNTUPLEREADER_H + +#include "podio/ROOTFrameData.h" +#include "podio/CollectionBranches.h" +#include "podio/ICollectionProvider.h" +#include "podio/podioVersion.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace rnt = ROOT::Experimental; +// forward declarations +class TClass; +class TFile; + +namespace podio { + +namespace detail { + // Information about the data vector as wall as the collection class type + // and the index in the collection branches cache vector + using CollectionInfo = std::tuple; + +} // namespace detail + +class EventStore; +class CollectionBase; +class Registry; +class CollectionIDTable; +class GenericParameters; +/** +This class has the function to read available data from disk +and to prepare collections and buffers. +**/ +class ROOTNTupleReader{ + friend EventStore; + +public: + ROOTNTupleReader() = default; + ~ROOTNTupleReader() = default; + + // non-copyable + ROOTNTupleReader(const ROOTNTupleReader&) = delete; + ROOTNTupleReader& operator=(const ROOTNTupleReader&) = delete; + + void openFile(const std::string& filename); + void openFiles(const std::vector& filename); + + /** + * Read the next data entry from which a Frame can be constructed for the + * given name. In case there are no more entries left for this name or in + * case there is no data for this name, this returns a nullptr. + */ + std::unique_ptr readNextEntry(const std::string& name); + + /** + * Read the specified data entry from which a Frame can be constructed for + * the given name. In case the entry does not exist for this name or in case + * there is no data for this name, this returns a nullptr. + */ + std::unique_ptr readEntry(const std::string& name, const unsigned entry); + + /// Returns number of entries for the given name + unsigned getEntries(const std::string& name); + + /// Get the build version of podio that has been used to write the current file + podio::version::Version currentFileVersion() const { + return m_fileVersion; + } + + void closeFile(); + +private: + std::vector> m_files; + std::unique_ptr m_file; + std::unique_ptr m_metadata; + + +private: + + /** + * Initialze the passed CategoryInfo by setting up the necessary branches, + * collection infos and all necessary meta data to be able to read entries + * with this name + */ + void initCategory(const std::string& category); + + GenericParameters readEventMetaData(const std::string& name); + + std::vector m_availCategories{}; ///< All available categories from this file + + podio::version::Version m_fileVersion{0, 0, 0}; + DatamodelDefinitionHolder m_datamodelHolder{}; + + std::map>> m_readers; + std::map> m_metadata_readers; + std::vector m_filenames; + std::map m_entries; + + std::map> m_collectionId; + std::map> m_collectionName; + std::map> m_collectionType; + std::map> m_isSubsetCollection; + + +}; + +} // namespace podio + +#endif diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 1927bceee..dc04d6a8b 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -2,12 +2,9 @@ #define PODIO_ROOTNTUPLEWRITER_H #include "podio/CollectionBase.h" -#include "podio/CollectionBranches.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" -#include "TBranch.h" #include "TFile.h" -#include "TChain.h" #include "podio/Frame.h" #include #include @@ -20,10 +17,8 @@ #include namespace rnt = ROOT::Experimental; -// forward declarations + class TFile; -class TTree; -class TChain; namespace podio { @@ -44,17 +39,23 @@ class ROOTNTupleWriter { using StoreCollection = std::pair; std::unique_ptr createModels(const std::vector& collections); - std::unique_ptr< rnt::RNTupleModel > m_metadata; + std::unique_ptr m_metadata {nullptr}; rnt::REntry* m_entry; std::map> m_writers; + std::unique_ptr m_metadataWriter {nullptr}; - std::unique_ptr m_file; - - std::unique_ptr< rnt::RNTupleWriter > m_metadata_writer; - }; + std::unique_ptr m_file {nullptr}; DatamodelDefinitionCollector m_datamodelCollector{}; + std::set m_categories; + std::map> m_collectionId; + std::map> m_collectionName; + std::map> m_collectionType; + std::map> m_isSubsetCollection; + +}; + } //namespace podio #endif //PODIO_ROOTNTUPLEWRITER_H diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc new file mode 100644 index 000000000..efe96ba31 --- /dev/null +++ b/src/ROOTNTupleReader.cc @@ -0,0 +1,193 @@ +#include "podio/ROOTNTupleReader.h" +#include "podio/CollectionBase.h" +#include "podio/CollectionBuffers.h" +#include "podio/CollectionIDTable.h" +#include "podio/GenericParameters.h" +#include "rootUtils.h" + +// ROOT specific includes +#include "TClass.h" +#include "TFile.h" +#include + +#include "datamodel/ExampleMCData.h" + +namespace podio { + +GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name) { + // Parameter branch is always the last one + // auto& paramBranches = catInfo.branches.back(); + // auto* branch = paramBranches.data; + + GenericParameters params; + // auto* emd = ¶ms; + // branch->SetAddress(&emd); + // branch->GetEntry(catInfo.entry); + return params; +} + +void ROOTNTupleReader::initCategory(const std::string& category) { + std::cout << "initCategory(" << category << ")" << std::endl; + std::cout << "Getting id" << std::endl; + // Assume that the metadata is the same in all files + auto filename = m_filenames[0]; + auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); + m_collectionId[category] = id(0); + + std::cout << "Getting collectionName" << std::endl; + auto collectionName = m_metadata_readers[filename]->GetView>(category + "_name"); + m_collectionName[category] = collectionName(0); + + std::cout << "Getting collectionType" << std::endl; + auto collectionType = m_metadata_readers[filename]->GetView>(root_utils::collInfoName(category)); + m_collectionType[category] = collectionType(0); + + std::cout << "Getting subsetCollection" << std::endl; + auto subsetCollection = m_metadata_readers[filename]->GetView>(category + "_test"); + m_isSubsetCollection[category] = subsetCollection(0); +} + +void ROOTNTupleReader::openFile(const std::string& filename) { + openFiles({filename}); +} + +void ROOTNTupleReader::openFiles(const std::vector& filenames) { + + m_filenames.insert(m_filenames.end(), filenames.begin(), filenames.end()); + for (auto& filename : filenames) { + if (m_metadata_readers.find(filename) == m_metadata_readers.end()) { + m_metadata_readers[filename] = ROOT::Experimental::RNTupleReader::Open(root_utils::metaTreeName, filename); + } + } + + m_metadata = ROOT::Experimental::RNTupleReader::Open(root_utils::metaTreeName, "example_rntuple.root"); + + auto version_view = m_metadata->GetView>(root_utils::versionBranchName); + auto version = version_view(0); + + m_fileVersion = podio::version::Version{version[0], version[1], version[2]}; + std::cout << "Version is " << m_fileVersion.major << " " << m_fileVersion.minor << " " << m_fileVersion.patch << std::endl; + + auto edm_view = m_metadata->GetView>>(root_utils::edmDefBranchName); + auto edm = edm_view(0); + + // m_datamodelHolder = DatamodelDefinitionHolder(std::move(*datamodelDefs)); + + // Do some work up front for setting up categories and setup all the chains + // and record the available categories. The rest of the setup follows on + // demand when the category is first read + // m_availCategories = ::podio::getAvailableCategories2(m_metaChain.get()); + // for (const auto& cat : m_availCategories) { + // auto [it, _] = m_categories.try_emplace(cat, std::make_unique(cat.c_str())); + // for (const auto& fn : filenames) { + // it->second.chain->Add(fn.c_str()); + // } + // } +} + +unsigned ROOTNTupleReader::getEntries(const std::string& name) { + if (m_readers.find(name) == m_readers.end()) { + for (auto& filename : m_filenames) { + m_readers[name].emplace_back(ROOT::Experimental::RNTupleReader::Open(name, filename)); + } + } + return std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); +} + +std::unique_ptr ROOTNTupleReader::readNextEntry(const std::string& name) { + // auto& catInfo = getCategoryInfo(name); + int current_entry = m_entries[name]; + + return readEntry(name, current_entry); +} + +std::unique_ptr ROOTNTupleReader::readEntry(const std::string& category, const unsigned entNum) { + std::cout << "Calling readEntry " << std::endl; + + if (m_collectionId.find(category) == m_collectionId.end()) { + initCategory(category); + } + + ROOTFrameData::BufferMap buffers; + auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); + + for (int i = 0; i < m_collectionId[category].size(); ++i) { + std::cout << "i = " << i << " " << m_collectionId[category][i] << " " << m_collectionType[category][i] << " " << m_collectionName[category][i] << std::endl; + + const auto collectionClass = TClass::GetClass(m_collectionType[category][i].c_str()); + + auto collection = + std::unique_ptr(static_cast(collectionClass->New())); + + const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; + const auto bufferClass = m_isSubsetCollection[category][i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); + + auto collBuffers = podio::CollectionReadBuffers(); + // const bool isSubsetColl = bufferClass == nullptr; + const bool isSubsetColl = bufferClass == nullptr; + if (!isSubsetColl) { + collBuffers.data = bufferClass->New(); + } + collection->setSubsetCollection(isSubsetColl); + + auto tmpBuffers = collection->createBuffers(); + collBuffers.createCollection = std::move(tmpBuffers.createCollection); + collBuffers.recast = std::move(tmpBuffers.recast); + + if (auto* refs = tmpBuffers.references) { + collBuffers.references = new podio::CollRefCollection(refs->size()); + } + if (auto* vminfo = tmpBuffers.vectorMembers) { + collBuffers.vectorMembers = new podio::VectorMembersInfo(); + collBuffers.vectorMembers->reserve(vminfo->size()); + + for (const auto& [type, _] : (*vminfo)) { + const auto* vecClass = TClass::GetClass(("vector<" + type + ">").c_str()); + collBuffers.vectorMembers->emplace_back(type, vecClass->New()); + } + } + + if (!isSubsetColl) { + dentry->CaptureValueUnsafe(m_collectionName[category][i], collBuffers.data); + } + if (auto* refCollections = collBuffers.references) { + std::cout << "The number of references is " << refCollections->size() << std::endl; + for (size_t j = 0; j < refCollections->size(); ++j) { + // The unique_ptrs are nullptrs at the beginning, we first initialize + // them and then fill the values with the read data since + refCollections->at(j) = std::make_unique>(); + const auto brName = root_utils::refBranch(m_collectionName[category][i], j); + std::cout << "brName = " << brName << " " << (refCollections->at(j) == nullptr) << std::endl; + dentry->CaptureValueUnsafe(brName, (*refCollections)[j].get()); + } + } + std::cout << "CaptureValueUnsafe done" << std::endl; + buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); + } + m_readers[category][0]->LoadEntry(entNum); + + auto buf = buffers["mcparticles"]; + auto ptr = (std::vector*)(buf.data); + std::cout << "Size of MCData is " << ptr->size(); + + + auto parameters = readEventMetaData(category); + auto table = std::make_shared(); + + auto names = m_collectionName[category]; + auto ids = m_collectionId[category]; + + std::vector> v; + for (int i = 0; i < names.size(); ++i) { + v.emplace_back(std::make_pair(int(ids[i]),std::string(names[i]))); + } + std::sort(v.begin(), v.end()); + + for (auto& [name, id] : v) { + table->add(std::to_string(name)); + } + + return std::make_unique(std::move(buffers), table, std::move(parameters)); +} + +} // namespace podio diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 923f388e6..b947e0c39 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -20,10 +20,14 @@ namespace podio { ROOTNTupleWriter::ROOTNTupleWriter(const std::string& filename) : m_metadata(nullptr), - m_file(new TFile(filename.c_str(),"RECREATE","data file")) + m_writers(), + m_file(new TFile(filename.c_str(),"RECREATE","data file")), + m_categories(), + m_collectionId(), + m_collectionType(), + m_isSubsetCollection() { m_metadata = rnt::RNTupleModel::Create(); - } void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { @@ -40,7 +44,9 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& collections.emplace_back(name, const_cast(coll)); } + bool new_category = false; if (m_writers.find(category) == m_writers.end()) { + new_category = true; auto model = createModels(collections); m_writers[category] = rnt::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); } @@ -106,8 +112,19 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& // } // } // } + + // Not supported + // m_entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); + + if (new_category) { + m_collectionId[category].emplace_back(coll->getID()); + m_collectionName[category].emplace_back(name); + m_collectionType[category].emplace_back(coll->getTypeName()); + m_isSubsetCollection[category].emplace_back(coll->isSubsetCollection()); + } } m_writers[category]->Fill(); + m_categories.insert(category); } std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { @@ -118,6 +135,7 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec if (collBuffers.data) { auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; std::cout << name << " " << collClassName << std::endl; + std::cout << "Making field with name = " << name << " and collClassName = " << collClassName << std::endl; auto field = rnt::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } @@ -127,6 +145,7 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec for (auto& c : (*refColls)) { const auto brName = root_utils::refBranch(name, i); auto collClassName = "vector"; + std::cout << "Making reference field with name = " << brName << " and collClassName = " << collClassName << std::endl; auto field = rnt::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); model->AddField(std::move(field)); ++i; @@ -144,6 +163,9 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec // } // } } + // auto field = rnt::Detail::RFieldBase::Create(root_utils::paramBranchName, ").Unwrap(); + // Not supported + // auto field = model->MakeField(root_utils::paramBranchName); model->Freeze(); return model; } @@ -158,17 +180,28 @@ void ROOTNTupleWriter::finish() { auto edm_field = m_metadata->MakeField>>(root_utils::edmDefBranchName); *edm_field = edmDefinitions; + for (auto& category : m_categories) { + auto idField = m_metadata->MakeField>(root_utils::idTableName(category)); + *idField = m_collectionId[category]; + auto collectionNameField = m_metadata->MakeField>(category + "_name"); + *collectionNameField = m_collectionName[category]; + auto collectionTypeField = m_metadata->MakeField>(root_utils::collInfoName(category)); + *collectionTypeField = m_collectionType[category]; + auto subsetCollectionField = m_metadata->MakeField>(category + "_test"); + *subsetCollectionField = m_isSubsetCollection[category]; + } + m_metadata->Freeze(); - m_metadata_writer = rnt::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); + m_metadataWriter = rnt::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); - m_metadata_writer->Fill(); + m_metadataWriter->Fill(); m_file->Write(); // All the tuple writers have to be deleted before the file so that they flush // unwritten output m_writers.clear(); - m_metadata_writer.reset(); + m_metadataWriter.reset(); } } //namespace podio From 69d50c83c285e1104f378e2123a659cb6bb4182e Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 20 Mar 2023 13:22:53 +0100 Subject: [PATCH 03/42] Add compilation instructions for RNTuple --- CMakeLists.txt | 7 ++++++- src/CMakeLists.txt | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de56ea75a..cad3686f9 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,12 +68,17 @@ ADD_CLANG_TIDY() option(CREATE_DOC "Whether or not to create doxygen doc target." OFF) option(ENABLE_SIO "Build SIO I/O support" OFF) option(PODIO_RELAX_PYVER "Do not require exact python version match with ROOT" OFF) +option(ENABLE_RNTUPLE "Build with support for the new ROOT NTtuple format" OFF) #--- Declare ROOT dependency --------------------------------------------------- list(APPEND CMAKE_PREFIX_PATH $ENV{ROOTSYS}) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) -find_package(ROOT REQUIRED COMPONENTS RIO Tree) +if(NOT ENABLE_RNTUPLE) + find_package(ROOT REQUIRED COMPONENTS RIO Tree) +else() + find_package(ROOT REQUIRED COMPONENTS RIO Tree ROOTNTuple) +endif() # Check that root is compiled with a modern enough c++ standard get_target_property(ROOT_COMPILE_FEATURES ROOT::Core INTERFACE_COMPILE_FEATURES) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f71864e3..d5d26d1c8 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,9 @@ FUNCTION(PODIO_ADD_LIB_AND_DICT libname headers sources selection ) $ $) target_link_libraries(${dictname} PUBLIC podio::${libname} podio::podio ROOT::Core ROOT::Tree) + if(ENABLE_RNTUPLE) + target_link_libraries(${dictname} PUBLIC ROOT::ROOTNTuple) + endif() PODIO_GENERATE_DICTIONARY(${dictname} ${headers} SELECTION ${selection} OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}${dictname}${CMAKE_SHARED_LIBRARY_SUFFIX} ) @@ -82,15 +85,32 @@ SET(root_sources ROOTFrameReader.cc ROOTLegacyReader.cc ) +if(ENABLE_RNTUPLE) + set(root_sources + ${root_sources} + ROOTNTupleReader.cc + ROOTNTupleWriter.cc + ) +endif() SET(root_headers ${CMAKE_SOURCE_DIR}/include/podio/ROOTFrameReader.h ${CMAKE_SOURCE_DIR}/include/podio/ROOTLegacyReader.h ${CMAKE_SOURCE_DIR}/include/podio/ROOTFrameWriter.h ) +if(ENABLE_RNTUPLE) + set(root_headers + ${root_headers} + ${CMAKE_SOURCE_DIR}/include/podio/ROOTNTupleReader.h + ${CMAKE_SOURCE_DIR}/include/podio/ROOTNTupleWriter.h + ) +endif() PODIO_ADD_LIB_AND_DICT(podioRootIO "${root_headers}" "${root_sources}" root_selection.xml) target_link_libraries(podioRootIO PUBLIC podio::podio ROOT::Core ROOT::RIO ROOT::Tree) +if(ENABLE_RNTUPLE) + target_link_libraries(podioRootIO PUBLIC ROOT::ROOTNTuple) +endif() # --- Python EventStore for enabling (legacy) python bindings From f4cc9237bf9e44c3d65a8349d9c01db7368569d5 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 20 Mar 2023 13:33:58 +0100 Subject: [PATCH 04/42] Add tests --- tests/root_io/CMakeLists.txt | 10 ++++++++++ tests/root_io/read_rntuple.cpp | 6 ++++++ tests/root_io/write_rntuple.cpp | 6 ++++++ 3 files changed, 22 insertions(+) create mode 100644 tests/root_io/read_rntuple.cpp create mode 100644 tests/root_io/write_rntuple.cpp diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index d0d8b21c4..ad1537c23 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -12,6 +12,13 @@ set(root_dependent_tests read_frame_legacy_root.cpp read_frame_root_multiple.cpp ) +if(ENABLE_RNTUPLE) + set(root_dependent_tests + ${root_dependent_tests} + write_rntuple.cpp + read_rntuple.cpp + ) +endif() set(root_libs TestDataModelDict ExtensionDataModelDict podio::podioRootIO) foreach( sourcefile ${root_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${root_libs}") @@ -27,6 +34,9 @@ set_property(TEST read_frame_legacy_root PROPERTY DEPENDS write) set_property(TEST read_timed PROPERTY DEPENDS write_timed) set_property(TEST read_frame_root PROPERTY DEPENDS write_frame_root) set_property(TEST read_frame_root_multiple PROPERTY DEPENDS write_frame_root) +if(ENABLE_RNTUPLE) + set_property(TEST read_rntuple PROPERTY DEPENDS write_rntuple) +endif() add_test(NAME check_benchmark_outputs COMMAND check_benchmark_outputs write_benchmark_root.root read_benchmark_root.root) set_property(TEST check_benchmark_outputs PROPERTY DEPENDS read_timed write_timed) diff --git a/tests/root_io/read_rntuple.cpp b/tests/root_io/read_rntuple.cpp new file mode 100644 index 000000000..3d66dede8 --- /dev/null +++ b/tests/root_io/read_rntuple.cpp @@ -0,0 +1,6 @@ +#include "podio/ROOTNTupleReader.h" +#include "read_frame.h" + +int main() { + return read_frames("example_rntuple.root"); +} diff --git a/tests/root_io/write_rntuple.cpp b/tests/root_io/write_rntuple.cpp new file mode 100644 index 000000000..a0a4b70d6 --- /dev/null +++ b/tests/root_io/write_rntuple.cpp @@ -0,0 +1,6 @@ +#include "podio/ROOTNTupleWriter.h" +#include "write_frame.h" + +int main(){ + write_frames("example_rntuple.root"); +} From 4dbbf7f58cc4b45f82e34d8fb88af38ece1d0500 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 08:02:51 +0100 Subject: [PATCH 05/42] Fix the reader and writer so that they pass most of the tests --- src/ROOTNTupleReader.cc | 106 +++++++++++++++++++++++++++++++++------- src/ROOTNTupleWriter.cc | 5 ++ 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index efe96ba31..75ac135f7 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -8,9 +8,11 @@ // ROOT specific includes #include "TClass.h" #include "TFile.h" +#include #include #include "datamodel/ExampleMCData.h" +#include "datamodel/ExampleHitData.h" namespace podio { @@ -26,7 +28,10 @@ GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name) { return params; } -void ROOTNTupleReader::initCategory(const std::string& category) { +bool ROOTNTupleReader::initCategory(const std::string& category) { + if (std::find(m_availableCategories.begin(), m_availableCategories.end(), category) == m_availableCategories.end()) { + return false; + } std::cout << "initCategory(" << category << ")" << std::endl; std::cout << "Getting id" << std::endl; // Assume that the metadata is the same in all files @@ -45,6 +50,7 @@ void ROOTNTupleReader::initCategory(const std::string& category) { std::cout << "Getting subsetCollection" << std::endl; auto subsetCollection = m_metadata_readers[filename]->GetView>(category + "_test"); m_isSubsetCollection[category] = subsetCollection(0); + return true; } void ROOTNTupleReader::openFile(const std::string& filename) { @@ -71,6 +77,9 @@ void ROOTNTupleReader::openFiles(const std::vector& filenames) { auto edm_view = m_metadata->GetView>>(root_utils::edmDefBranchName); auto edm = edm_view(0); + auto availableCategoriesField = m_metadata->GetView>("available_categories"); + m_availableCategories = availableCategoriesField(0); + // m_datamodelHolder = DatamodelDefinitionHolder(std::move(*datamodelDefs)); // Do some work up front for setting up categories and setup all the chains @@ -88,29 +97,43 @@ void ROOTNTupleReader::openFiles(const std::vector& filenames) { unsigned ROOTNTupleReader::getEntries(const std::string& name) { if (m_readers.find(name) == m_readers.end()) { for (auto& filename : m_filenames) { - m_readers[name].emplace_back(ROOT::Experimental::RNTupleReader::Open(name, filename)); + try { + m_readers[name].emplace_back(ROOT::Experimental::RNTupleReader::Open(name, filename)); + } + catch (const ROOT::Experimental::RException& e) { + std::cout << "Category " << name << " not found in file " << filename << std::endl; + } } } - return std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); + m_totalEntries[name] = std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); + return m_totalEntries[name]; } std::unique_ptr ROOTNTupleReader::readNextEntry(const std::string& name) { - // auto& catInfo = getCategoryInfo(name); - int current_entry = m_entries[name]; - - return readEntry(name, current_entry); + return readEntry(name, m_entries[name]++); } std::unique_ptr ROOTNTupleReader::readEntry(const std::string& category, const unsigned entNum) { - std::cout << "Calling readEntry " << std::endl; + if (m_totalEntries.find(category) == m_totalEntries.end()) { + getEntries(category); + } + if (entNum >= m_totalEntries[category]) { + return nullptr; + } if (m_collectionId.find(category) == m_collectionId.end()) { - initCategory(category); + if (!initCategory(category)) { + return nullptr; + } } + m_entries[category] = entNum + 1; + ROOTFrameData::BufferMap buffers; auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); + std::map, std::vector*> tmp; + for (int i = 0; i < m_collectionId[category].size(); ++i) { std::cout << "i = " << i << " " << m_collectionId[category][i] << " " << m_collectionType[category][i] << " " << m_collectionName[category][i] << std::endl; @@ -128,6 +151,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca if (!isSubsetColl) { collBuffers.data = bufferClass->New(); } + std::cout << "Setting colluction " << m_collectionName[category][i] << " is subset?: " << m_isSubsetCollection[category][i] << std::endl; collection->setSubsetCollection(isSubsetColl); auto tmpBuffers = collection->createBuffers(); @@ -153,12 +177,18 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca if (auto* refCollections = collBuffers.references) { std::cout << "The number of references is " << refCollections->size() << std::endl; for (size_t j = 0; j < refCollections->size(); ++j) { - // The unique_ptrs are nullptrs at the beginning, we first initialize - // them and then fill the values with the read data since - refCollections->at(j) = std::make_unique>(); + // // The unique_ptrs are nullptrs at the beginning, we first initialize + // // them and then fill the values with the read data since + // refCollections->at(j) = std::make_unique>(); + // const auto brName = root_utils::refBranch(m_collectionName[category][i], j); + // std::cout << "brName = " << brName << " " << (refCollections->at(j) == nullptr) << std::endl; + // dentry->CaptureValueUnsafe(brName, (*refCollections)[j].get()); + + auto vec = new std::vector; const auto brName = root_utils::refBranch(m_collectionName[category][i], j); std::cout << "brName = " << brName << " " << (refCollections->at(j) == nullptr) << std::endl; - dentry->CaptureValueUnsafe(brName, (*refCollections)[j].get()); + dentry->CaptureValueUnsafe(brName, vec); + tmp[{brName, j}] = vec; } } std::cout << "CaptureValueUnsafe done" << std::endl; @@ -166,9 +196,48 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } m_readers[category][0]->LoadEntry(entNum); - auto buf = buffers["mcparticles"]; - auto ptr = (std::vector*)(buf.data); - std::cout << "Size of MCData is " << ptr->size(); + for (int i = 0; i < m_collectionId[category].size(); ++i) { + auto collBuffers = buffers[m_collectionName[category][i]]; + if (auto* refCollections = collBuffers.references) { + for (size_t j = 0; j < refCollections->size(); ++j) { + const auto brName = root_utils::refBranch(m_collectionName[category][i], j); + refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); + } + } + + } + + // auto buf = buffers["mcparticles"]; + // auto ptr = (std::vector*)(buf.data); + // std::cout << "Size of MCData is " << ptr->size() << std::endl; + // for (auto& x : *ptr) { + // std::cout << x.energy << " " << x.PDG << std::endl; + // } + + // auto buf = buffers["hits"]; + // auto ptr = (std::vector*)(buf.data); + // std::cout << "Size of HitData is " << ptr->size() << std::endl; + // for (auto& x : *ptr) { + // std::cout << x.cellID << " " << x.energy << " " << " " << x.x << " " << x.y << " " << x.z << std::endl; + // } + + auto buf = buffers["hitRefs"]; + auto refCollections = buf.references; + for (size_t j = 0; j < refCollections->size(); ++j) { + std::cout << "Size of ObjectID is " << refCollections->at(j)->size() << std::endl; + auto ptr = (std::vector*)refCollections->at(j).get(); + for (auto& x : *ptr) { + std::cout << x.index << " " << x.collectionID << std::endl; + } + } + + // auto nbuf = buffers["hitRefs"]; + // auto nptr = (std::vector>*)(buf.references); + // std::cout << "Size of ObjectID is " << nptr->size() << std::endl; + // for (auto& x : *nptr) { + // auto nnptr = ; + // std::cout << x->index << " " << x->collectionID << std::endl; + // } auto parameters = readEventMetaData(category); @@ -179,12 +248,13 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::vector> v; for (int i = 0; i < names.size(); ++i) { + std::cout << ids[i] << " " << names[i] << std::endl; v.emplace_back(std::make_pair(int(ids[i]),std::string(names[i]))); } std::sort(v.begin(), v.end()); - for (auto& [name, id] : v) { - table->add(std::to_string(name)); + for (auto& [id, name] : v) { + table->add(name); } return std::make_unique(std::move(buffers), table, std::move(parameters)); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index b947e0c39..99c6228a9 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -180,6 +180,11 @@ void ROOTNTupleWriter::finish() { auto edm_field = m_metadata->MakeField>>(root_utils::edmDefBranchName); *edm_field = edmDefinitions; + auto availableCategoriesField = m_metadata->MakeField>("available_categories"); + for (auto& [c, _] : m_collectionId ) { + availableCategoriesField->push_back(c); + } + for (auto& category : m_categories) { auto idField = m_metadata->MakeField>(root_utils::idTableName(category)); *idField = m_collectionId[category]; From 49167c947d49c44fff0fc1457f3ad23fc6dac2f5 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 08:03:26 +0100 Subject: [PATCH 06/42] Commit missing changes in the header --- include/podio/ROOTNTupleReader.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 30db2db2e..6c1d62589 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -93,11 +93,10 @@ class ROOTNTupleReader{ * collection infos and all necessary meta data to be able to read entries * with this name */ - void initCategory(const std::string& category); + bool initCategory(const std::string& category); GenericParameters readEventMetaData(const std::string& name); - std::vector m_availCategories{}; ///< All available categories from this file podio::version::Version m_fileVersion{0, 0, 0}; DatamodelDefinitionHolder m_datamodelHolder{}; @@ -112,6 +111,8 @@ class ROOTNTupleReader{ std::map> m_collectionType; std::map> m_isSubsetCollection; + std::map m_totalEntries; + std::vector m_availableCategories; }; From 4bc7ed7c8a4cfe7bcb3604fe709e08003e4fb44d Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 14:00:58 +0100 Subject: [PATCH 07/42] Add support for Generic Parameters --- include/podio/ROOTNTupleReader.h | 12 +--- include/podio/ROOTNTupleWriter.h | 4 +- src/ROOTNTupleReader.cc | 60 +++++++++++++++---- src/ROOTNTupleWriter.cc | 100 +++++++++++++++++++++++++++++-- 4 files changed, 146 insertions(+), 30 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 6c1d62589..6d9452c26 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -22,17 +21,9 @@ namespace rnt = ROOT::Experimental; // forward declarations class TClass; -class TFile; namespace podio { -namespace detail { - // Information about the data vector as wall as the collection class type - // and the index in the collection branches cache vector - using CollectionInfo = std::tuple; - -} // namespace detail - class EventStore; class CollectionBase; class Registry; @@ -95,9 +86,8 @@ class ROOTNTupleReader{ */ bool initCategory(const std::string& category); - GenericParameters readEventMetaData(const std::string& name); + GenericParameters readEventMetaData(const std::string& name, unsigned entNum); - podio::version::Version m_fileVersion{0, 0, 0}; DatamodelDefinitionHolder m_datamodelHolder{}; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index dc04d6a8b..b1a1b2061 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -2,6 +2,7 @@ #define PODIO_ROOTNTUPLEWRITER_H #include "podio/CollectionBase.h" +#include "podio/GenericParameters.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" #include "TFile.h" @@ -37,7 +38,8 @@ class ROOTNTupleWriter { private: using StoreCollection = std::pair; - std::unique_ptr createModels(const std::vector& collections); + std::unique_ptr createModels(const std::vector& collections, + const podio::GenericParameters& params); std::unique_ptr m_metadata {nullptr}; rnt::REntry* m_entry; diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 75ac135f7..81beba731 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -7,7 +7,6 @@ // ROOT specific includes #include "TClass.h" -#include "TFile.h" #include #include @@ -16,15 +15,47 @@ namespace podio { -GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name) { - // Parameter branch is always the last one - // auto& paramBranches = catInfo.branches.back(); - // auto* branch = paramBranches.data; - + GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { GenericParameters params; - // auto* emd = ¶ms; - // branch->SetAddress(&emd); - // branch->GetEntry(catInfo.entry); + + auto intKeyView = m_readers[name][0]->GetView>("GP_int_keys"); + auto floatKeyView = m_readers[name][0]->GetView>("GP_float_keys"); + auto doubleKeyView = m_readers[name][0]->GetView>("GP_double_keys"); + auto stringKeyView = m_readers[name][0]->GetView>("GP_string_keys"); + + auto intValueView = m_readers[name][0]->GetView>>("GP_int_values"); + auto floatValueView = m_readers[name][0]->GetView>>("GP_float_values"); + auto doubleValueView = m_readers[name][0]->GetView>>("GP_double_values"); + auto stringValueView = m_readers[name][0]->GetView>>("GP_string_values"); + + auto keys = intKeyView(entNum); + auto valuesInt = intValueView(entNum); + for (size_t i = 0; i < keys.size(); ++i) { + params.getIntMap()[keys[i]] = valuesInt[i]; + } + keys = floatKeyView(entNum); + auto valuesFloat = floatValueView(entNum); + for (size_t i = 0; i < keys.size(); ++i) { + params.getFloatMap()[keys[i]] = valuesFloat[i]; + } + keys = doubleKeyView(entNum); + auto valuesDouble = doubleValueView(entNum); + for (size_t i = 0; i < keys.size(); ++i) { + params.getDoubleMap()[keys[i]] = valuesDouble[i]; + } + keys = stringKeyView(entNum); + auto valuesString = stringValueView(entNum); + for (size_t i = 0; i < keys.size(); ++i) { + params.getStringMap()[keys[i]] = valuesString[i]; + } + std::cout << "Size of Float map is " << params.getFloatMap().size() << std::endl; + for (auto& [k, v] : params.getFloatMap()) { + std::cout << k << std::endl; + for (auto& x : v) + std::cout << x << " "; + std::cout << std::endl; + } + return params; } @@ -50,6 +81,7 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { std::cout << "Getting subsetCollection" << std::endl; auto subsetCollection = m_metadata_readers[filename]->GetView>(category + "_test"); m_isSubsetCollection[category] = subsetCollection(0); + return true; } @@ -110,7 +142,7 @@ unsigned ROOTNTupleReader::getEntries(const std::string& name) { } std::unique_ptr ROOTNTupleReader::readNextEntry(const std::string& name) { - return readEntry(name, m_entries[name]++); + return readEntry(name, m_entries[name]); } std::unique_ptr ROOTNTupleReader::readEntry(const std::string& category, const unsigned entNum) { @@ -127,7 +159,8 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } } - m_entries[category] = entNum + 1; + std::cout << "Reading entry with category " << category << " and number " << entNum << std::endl; + m_entries[category] = entNum+1; ROOTFrameData::BufferMap buffers; auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); @@ -151,7 +184,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca if (!isSubsetColl) { collBuffers.data = bufferClass->New(); } - std::cout << "Setting colluction " << m_collectionName[category][i] << " is subset?: " << m_isSubsetCollection[category][i] << std::endl; collection->setSubsetCollection(isSubsetColl); auto tmpBuffers = collection->createBuffers(); @@ -194,6 +226,8 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::cout << "CaptureValueUnsafe done" << std::endl; buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); } + + m_readers[category][0]->LoadEntry(entNum); for (int i = 0; i < m_collectionId[category].size(); ++i) { @@ -240,7 +274,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca // } - auto parameters = readEventMetaData(category); + auto parameters = readEventMetaData(category, entNum); auto table = std::make_shared(); auto names = m_collectionName[category]; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 99c6228a9..a5f068ab2 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -2,6 +2,7 @@ #include #include +#include "podio/GenericParameters.h" #include "rootUtils.h" #include "podio/CollectionBase.h" @@ -47,7 +48,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& bool new_category = false; if (m_writers.find(category) == m_writers.end()) { new_category = true; - auto model = createModels(collections); + auto model = createModels(collections, frame.getParameters()); m_writers[category] = rnt::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); } @@ -116,6 +117,8 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& // Not supported // m_entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); + + if (new_category) { m_collectionId[category].emplace_back(coll->getID()); m_collectionName[category].emplace_back(name); @@ -123,11 +126,59 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& m_isSubsetCollection[category].emplace_back(coll->isSubsetCollection()); } } + + auto params = frame.getParameters(); + auto intMap = params.getIntMap(); + auto floatMap = params.getFloatMap(); + auto doubleMap = params.getDoubleMap(); + auto stringMap = params.getStringMap(); + + auto gpintKeys = m_writers[category]->GetModel()->Get>("GP_int_keys"); + auto gpfloatKeys = m_writers[category]->GetModel()->Get>("GP_float_keys"); + auto gpdoubleKeys = m_writers[category]->GetModel()->Get>("GP_double_keys"); + auto gpstringKeys = m_writers[category]->GetModel()->Get>("GP_string_keys"); + + auto gpintValues = m_writers[category]->GetModel()->Get>>("GP_int_values"); + auto gpfloatValues = m_writers[category]->GetModel()->Get>>("GP_float_values"); + auto gpdoubleValues = m_writers[category]->GetModel()->Get>>("GP_double_values"); + auto gpstringValues = m_writers[category]->GetModel()->Get>>("GP_string_values"); + + gpintKeys->clear(); + gpintValues->clear(); + for (auto& [k, v] : intMap) { + gpintKeys->emplace_back(k); + gpintValues->emplace_back(v); + } + std::cout << "Size of gpfloatKeys before filling " << gpfloatKeys->size() << " " << gpfloatValues->size() << std::endl; + + gpfloatKeys->clear(); + gpfloatValues->clear(); + for (auto& [k, v] : floatMap) { + gpfloatKeys->emplace_back(k); + gpfloatValues->emplace_back(v); + for (auto& x : v) { + std::cout << "floatMap: " << x << std::endl; + } + } + gpdoubleKeys->clear(); + gpdoubleValues->clear(); + for (auto& [k, v] : doubleMap) { + gpdoubleKeys->emplace_back(k); + gpdoubleValues->emplace_back(v); + } + gpstringKeys->clear(); + gpstringValues->clear(); + for (auto& [k, v] : stringMap) { + gpstringKeys->emplace_back(k); + gpstringValues->emplace_back(v); + } + m_writers[category]->Fill(); m_categories.insert(category); } -std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { +std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections, + const podio::GenericParameters& params) { auto model = rnt::RNTupleModel::Create(); for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); @@ -163,9 +214,46 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec // } // } } - // auto field = rnt::Detail::RFieldBase::Create(root_utils::paramBranchName, ").Unwrap(); - // Not supported - // auto field = model->MakeField(root_utils::paramBranchName); + + // gp = Generic Parameters + auto gpintKeys = model->MakeField>("GP_int_keys"); + auto gpfloatKeys = model->MakeField>("GP_float_keys"); + auto gpdoubleKeys = model->MakeField>("GP_double_keys"); + auto gpstringKeys = model->MakeField>("GP_string_keys"); + + auto gpintValues = model->MakeField>>("GP_int_values"); + auto gpfloatValues = model->MakeField>>("GP_float_values"); + auto gpdoubleValues = model->MakeField>>("GP_double_values"); + auto gpstringValues = model->MakeField>>("GP_string_values"); + + // auto intMap = params.getIntMap(); + // auto floatMap = params.getFloatMap(); + // auto doubleMap = params.getDoubleMap(); + // auto stringMap = params.getStringMap(); + + // for (auto& [k, v] : intMap) { + // gpintKeys->emplace_back(k); + // gpintValues->emplace_back(v); + // } + // for (auto& [k, v] : floatMap) { + // gpfloatKeys->emplace_back(k); + // gpfloatValues->emplace_back(v); + // for (auto& x : v) { + // std::cout << "floatMap: " << x << std::endl; + // } + // } + // for (auto& [k, v] : doubleMap) { + // gpdoubleKeys->emplace_back(k); + // gpdoubleValues->emplace_back(v); + // } + // for (auto& [k, v] : stringMap) { + // gpstringKeys->emplace_back(k); + // gpstringValues->emplace_back(v); + // } + + // Not supported by ROOT + // model->MakeField(root_utils::paramBranchName); + model->Freeze(); return model; } @@ -196,6 +284,8 @@ void ROOTNTupleWriter::finish() { *subsetCollectionField = m_isSubsetCollection[category]; } + + m_metadata->Freeze(); m_metadataWriter = rnt::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); From 9fc8da3b57642edebaaa335dd5ea4c4764058389 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 14:15:07 +0100 Subject: [PATCH 08/42] Add an ugly workaround to the unique_ptr issue --- include/podio/CollectionBuffers.h | 1 + include/podio/UserDataCollection.h | 2 +- python/templates/CollectionData.cc.jinja2 | 1 + src/ROOTNTupleWriter.cc | 6 +++--- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/podio/CollectionBuffers.h b/include/podio/CollectionBuffers.h index 8846af162..0dadd1137 100644 --- a/include/podio/CollectionBuffers.h +++ b/include/podio/CollectionBuffers.h @@ -27,6 +27,7 @@ using VectorMembersInfo = std::vector>; */ struct CollectionWriteBuffers { void* data{nullptr}; + void* vecPtr{nullptr}; CollRefCollection* references{nullptr}; VectorMembersInfo* vectorMembers{nullptr}; diff --git a/include/podio/UserDataCollection.h b/include/podio/UserDataCollection.h index b9aefaf40..cc5b7154f 100644 --- a/include/podio/UserDataCollection.h +++ b/include/podio/UserDataCollection.h @@ -123,7 +123,7 @@ class UserDataCollection : public CollectionBase { /// Get the collection buffers for this collection podio::CollectionWriteBuffers getBuffers() override { _vecPtr = &_vec; // Set the pointer to the correct internal vector - return {&_vecPtr, &m_refCollections, &m_vecmem_info}; + return {&_vecPtr, _vecPtr, &m_refCollections, &m_vecmem_info}; } /// check for validity of the container after read diff --git a/python/templates/CollectionData.cc.jinja2 b/python/templates/CollectionData.cc.jinja2 index 3ae5d3a80..3946ad756 100644 --- a/python/templates/CollectionData.cc.jinja2 +++ b/python/templates/CollectionData.cc.jinja2 @@ -92,6 +92,7 @@ podio::CollectionWriteBuffers {{ class_type }}::getCollectionBuffers(bool isSubs return { isSubsetColl ? nullptr : (void*)&m_data, + isSubsetColl ? nullptr : (void*)m_data.get(), &m_refCollections, // only need to store the ObjectIDs of the referenced objects &m_vecmem_info }; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index a5f068ab2..ac6e62b34 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -60,7 +60,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& for (const auto& [name, coll] : collections) { // coll->prepareForWrite(); auto collBuffers = coll->getBuffers(); - if (collBuffers.data) { + if (collBuffers.vecPtr) { std::cout << "Capturing unsafe " << name << std::endl; // auto v = (std::vector*)collBuffers.data; @@ -71,7 +71,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& // std::cout << x.energy << std::endl; // } - m_entry->CaptureValueUnsafe(name, (void*)collBuffers.data); + m_entry->CaptureValueUnsafe(name, (void*)collBuffers.vecPtr); // std::cout << "After capturing " << std::endl; // v = (std::vector*)collBuffers.data; @@ -183,7 +183,7 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); - if (collBuffers.data) { + if (collBuffers.vecPtr) { auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; std::cout << name << " " << collClassName << std::endl; std::cout << "Making field with name = " << name << " and collClassName = " << collClassName << std::endl; From 529dffc5b4119adfc9bd4ab3b382b02b5f59ac1c Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 15:54:50 +0100 Subject: [PATCH 09/42] Read also vector members and remove some comments --- src/ROOTNTupleReader.cc | 11 ++++-- src/ROOTNTupleWriter.cc | 75 ++++++++++++++--------------------------- 2 files changed, 34 insertions(+), 52 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 81beba731..86f71fb02 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -207,7 +207,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca dentry->CaptureValueUnsafe(m_collectionName[category][i], collBuffers.data); } if (auto* refCollections = collBuffers.references) { - std::cout << "The number of references is " << refCollections->size() << std::endl; for (size_t j = 0; j < refCollections->size(); ++j) { // // The unique_ptrs are nullptrs at the beginning, we first initialize // // them and then fill the values with the read data since @@ -218,11 +217,19 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto vec = new std::vector; const auto brName = root_utils::refBranch(m_collectionName[category][i], j); - std::cout << "brName = " << brName << " " << (refCollections->at(j) == nullptr) << std::endl; dentry->CaptureValueUnsafe(brName, vec); tmp[{brName, j}] = vec; } } + + if (auto* vecMembers = collBuffers.vectorMembers) { + for (size_t j = 0; j < vecMembers->size(); ++j) { + const auto typeName = "vector<" + vecMembers->at(j).first + ">"; + const auto brName = root_utils::vecBranch(m_collectionName[category][i], j); + dentry->CaptureValueUnsafe(brName, vecMembers->at(j).second); + } + } + std::cout << "CaptureValueUnsafe done" << std::endl; buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); } diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index ac6e62b34..5eab6493e 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -99,20 +99,21 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& } } - // if (auto vmInfo = collBuffers.vectorMembers) { - // int i = 0; - // for (auto& [type, vec] : (*vmInfo)) { - // const auto typeName = "vector<" + type + ">"; - // const auto brName = root_utils::vecBranch(name, i++); - // std::cout << typeName << " " << brName << std::endl; - // auto ptr = (std::vector*)vec; - // std::cout << "Size is " << ptr->size(); - // if (m_first) { - // // m_entry->CaptureValueUnsafe(brName, (void*) vec); - // m_first = false; - // } - // } - // } + if (auto vmInfo = collBuffers.vectorMembers) { + int i = 0; + for (auto& [type, vec] : (*vmInfo)) { + const auto typeName = "vector<" + type + ">"; + const auto brName = root_utils::vecBranch(name, i++); + std::cout << typeName << " " << brName << std::endl; + auto ptr = *(std::vector**)vec; + std::cout << "Vector: Size is " << ptr->size(); + m_entry->CaptureValueUnsafe(brName, ptr); + // if (m_first) { + // // m_entry->CaptureValueUnsafe(brName, (void*) vec); + // m_first = false; + // } + } + } // Not supported // m_entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); @@ -149,7 +150,6 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& gpintKeys->emplace_back(k); gpintValues->emplace_back(v); } - std::cout << "Size of gpfloatKeys before filling " << gpfloatKeys->size() << " " << gpfloatValues->size() << std::endl; gpfloatKeys->clear(); gpfloatValues->clear(); @@ -203,16 +203,16 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec } } - // if (auto vminfo = collBuffers.vectorMembers) { - // int i = 0; - // for (auto& [type, vec] : (*vminfo)) { - // const auto typeName = "vector<" + type + ">"; - // const auto brName = root_utils::vecBranch(name, i); - // auto field = rnt::Detail::RFieldBase::Create(brName, typeName).Unwrap(); - // model->AddField(std::move(field)); - // ++i; - // } - // } + if (auto vminfo = collBuffers.vectorMembers) { + int i = 0; + for (auto& [type, vec] : (*vminfo)) { + const auto typeName = "vector<" + type + ">"; + const auto brName = root_utils::vecBranch(name, i); + auto field = rnt::Detail::RFieldBase::Create(brName, typeName).Unwrap(); + model->AddField(std::move(field)); + ++i; + } + } } // gp = Generic Parameters @@ -226,31 +226,6 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec auto gpdoubleValues = model->MakeField>>("GP_double_values"); auto gpstringValues = model->MakeField>>("GP_string_values"); - // auto intMap = params.getIntMap(); - // auto floatMap = params.getFloatMap(); - // auto doubleMap = params.getDoubleMap(); - // auto stringMap = params.getStringMap(); - - // for (auto& [k, v] : intMap) { - // gpintKeys->emplace_back(k); - // gpintValues->emplace_back(v); - // } - // for (auto& [k, v] : floatMap) { - // gpfloatKeys->emplace_back(k); - // gpfloatValues->emplace_back(v); - // for (auto& x : v) { - // std::cout << "floatMap: " << x << std::endl; - // } - // } - // for (auto& [k, v] : doubleMap) { - // gpdoubleKeys->emplace_back(k); - // gpdoubleValues->emplace_back(v); - // } - // for (auto& [k, v] : stringMap) { - // gpstringKeys->emplace_back(k); - // gpstringValues->emplace_back(v); - // } - // Not supported by ROOT // model->MakeField(root_utils::paramBranchName); From 8dcdf5a6c9883d2d97a0c1f0556d5be4c747c02c Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 22:04:14 +0100 Subject: [PATCH 10/42] Do a bit of cleanup --- include/podio/ROOTNTupleWriter.h | 18 ++----- src/ROOTNTupleWriter.cc | 89 +++++++++----------------------- 2 files changed, 30 insertions(+), 77 deletions(-) diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index b1a1b2061..e529fcb0f 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -4,9 +4,9 @@ #include "podio/CollectionBase.h" #include "podio/GenericParameters.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" +#include "podio/Frame.h" #include "TFile.h" -#include "podio/Frame.h" #include #include @@ -14,12 +14,6 @@ #include #include #include -#include -#include - -namespace rnt = ROOT::Experimental; - -class TFile; namespace podio { @@ -38,13 +32,11 @@ class ROOTNTupleWriter { private: using StoreCollection = std::pair; - std::unique_ptr createModels(const std::vector& collections, - const podio::GenericParameters& params); + std::unique_ptr createModels(const std::vector& collections); - std::unique_ptr m_metadata {nullptr}; - rnt::REntry* m_entry; - std::map> m_writers; - std::unique_ptr m_metadataWriter {nullptr}; + std::unique_ptr m_metadata {nullptr}; + std::map> m_writers; + std::unique_ptr m_metadataWriter {nullptr}; std::unique_ptr m_file {nullptr}; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 5eab6493e..960047ecf 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -1,21 +1,14 @@ -#include -#include -#include - +#include "podio/ROOTNTupleWriter.h" #include "podio/GenericParameters.h" #include "rootUtils.h" - #include "podio/CollectionBase.h" -#include "podio/EventStore.h" -#include "podio/ROOTWriter.h" #include "podio/podioVersion.h" -// ROOT specifc includes #include "TFile.h" +#include +#include -#include "podio/ROOTNTupleWriter.h" - -#include "datamodel/ExampleMCData.h" +#include namespace podio { @@ -25,10 +18,11 @@ ROOTNTupleWriter::ROOTNTupleWriter(const std::string& filename) : m_file(new TFile(filename.c_str(),"RECREATE","data file")), m_categories(), m_collectionId(), + m_collectionName(), m_collectionType(), m_isSubsetCollection() { - m_metadata = rnt::RNTupleModel::Create(); + m_metadata = ROOT::Experimental::RNTupleModel::Create(); } void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { @@ -48,13 +42,13 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& bool new_category = false; if (m_writers.find(category) == m_writers.end()) { new_category = true; - auto model = createModels(collections, frame.getParameters()); - m_writers[category] = rnt::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); + auto model = createModels(collections); + m_writers[category] = ROOT::Experimental::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); } - m_entry = m_writers[category]->GetModel()->GetDefaultEntry(); + auto entry = m_writers[category]->GetModel()->GetDefaultEntry(); - rnt::RNTupleWriteOptions options; + ROOT::Experimental::RNTupleWriteOptions options; options.SetCompression(ROOT::RCompressionSetting::EDefaults::kUseGeneralPurpose); for (const auto& [name, coll] : collections) { @@ -63,39 +57,14 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& if (collBuffers.vecPtr) { std::cout << "Capturing unsafe " << name << std::endl; - // auto v = (std::vector*)collBuffers.data; - // std::cout << "Size is " << v->size() << std::endl; - // std::cout << (*v)[0].energy << std::endl; - // for (auto& x : *v) { - // std::cout << "Inside loop " << std::endl; - // std::cout << x.energy << std::endl; - // } - - m_entry->CaptureValueUnsafe(name, (void*)collBuffers.vecPtr); - - // std::cout << "After capturing " << std::endl; - // v = (std::vector*)collBuffers.data; - // std::cout << "Size is " << v->size() << std::endl; - // std::cout << (*v)[0].energy << std::endl; - // for (auto& x : *v) { - // std::cout << "Inside loop " << std::endl; - // std::cout << x.energy << std::endl; - // } - - // auto ptr = (std::vector*)collBuffers.data; - // auto start = (char*)ptr->data(); - // for (int i = 0; i < 13; ++i) { - // printf("%x ", start[i]); - // } - // std::cout << std::endl; - + entry->CaptureValueUnsafe(name, (void*)collBuffers.vecPtr); } if (auto refColls = collBuffers.references) { int i = 0; for (auto& c : (*refColls)) { const auto brName = root_utils::refBranch(name, i++); - m_entry->CaptureValueUnsafe(brName, c.get()); + entry->CaptureValueUnsafe(brName, c.get()); } } @@ -107,18 +76,12 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& std::cout << typeName << " " << brName << std::endl; auto ptr = *(std::vector**)vec; std::cout << "Vector: Size is " << ptr->size(); - m_entry->CaptureValueUnsafe(brName, ptr); - // if (m_first) { - // // m_entry->CaptureValueUnsafe(brName, (void*) vec); - // m_first = false; - // } + entry->CaptureValueUnsafe(brName, ptr); } } // Not supported - // m_entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); - - + // entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); if (new_category) { m_collectionId[category].emplace_back(coll->getID()); @@ -177,9 +140,8 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& m_categories.insert(category); } -std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections, - const podio::GenericParameters& params) { - auto model = rnt::RNTupleModel::Create(); +std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { + auto model = ROOT::Experimental::RNTupleModel::Create(); for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); @@ -187,17 +149,17 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; std::cout << name << " " << collClassName << std::endl; std::cout << "Making field with name = " << name << " and collClassName = " << collClassName << std::endl; - auto field = rnt::Detail::RFieldBase::Create(name, collClassName).Unwrap(); + auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } if (auto refColls = collBuffers.references) { int i = 0; - for (auto& c : (*refColls)) { + for (auto& c [[maybe_unused]] : (*refColls)) { const auto brName = root_utils::refBranch(name, i); auto collClassName = "vector"; std::cout << "Making reference field with name = " << brName << " and collClassName = " << collClassName << std::endl; - auto field = rnt::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); + auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); model->AddField(std::move(field)); ++i; } @@ -208,13 +170,17 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec for (auto& [type, vec] : (*vminfo)) { const auto typeName = "vector<" + type + ">"; const auto brName = root_utils::vecBranch(name, i); - auto field = rnt::Detail::RFieldBase::Create(brName, typeName).Unwrap(); + auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, typeName).Unwrap(); model->AddField(std::move(field)); ++i; } } } + // Not supported by ROOT because podio::GenericParameters has map types + // so we have to split them manually + // model->MakeField(root_utils::paramBranchName); + // gp = Generic Parameters auto gpintKeys = model->MakeField>("GP_int_keys"); auto gpfloatKeys = model->MakeField>("GP_float_keys"); @@ -226,9 +192,6 @@ std::unique_ptr ROOTNTupleWriter::createModels(const std::vec auto gpdoubleValues = model->MakeField>>("GP_double_values"); auto gpstringValues = model->MakeField>>("GP_string_values"); - // Not supported by ROOT - // model->MakeField(root_utils::paramBranchName); - model->Freeze(); return model; } @@ -259,10 +222,8 @@ void ROOTNTupleWriter::finish() { *subsetCollectionField = m_isSubsetCollection[category]; } - - m_metadata->Freeze(); - m_metadataWriter = rnt::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); + m_metadataWriter = ROOT::Experimental::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); m_metadataWriter->Fill(); From 27e7b878f6cece7287986ddec3050e89cf4944e8 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Mar 2023 22:22:46 +0100 Subject: [PATCH 11/42] Do more cleanup, also compiler warnings --- include/podio/ROOTNTupleReader.h | 10 +---- src/ROOTNTupleReader.cc | 63 ++------------------------------ src/ROOTNTupleWriter.cc | 1 - 3 files changed, 4 insertions(+), 70 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 6d9452c26..c277c111e 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -18,13 +18,8 @@ #include -namespace rnt = ROOT::Experimental; -// forward declarations -class TClass; - namespace podio { -class EventStore; class CollectionBase; class Registry; class CollectionIDTable; @@ -34,13 +29,11 @@ This class has the function to read available data from disk and to prepare collections and buffers. **/ class ROOTNTupleReader{ - friend EventStore; public: ROOTNTupleReader() = default; ~ROOTNTupleReader() = default; - // non-copyable ROOTNTupleReader(const ROOTNTupleReader&) = delete; ROOTNTupleReader& operator=(const ROOTNTupleReader&) = delete; @@ -76,7 +69,6 @@ class ROOTNTupleReader{ std::unique_ptr m_file; std::unique_ptr m_metadata; - private: /** @@ -101,7 +93,7 @@ class ROOTNTupleReader{ std::map> m_collectionType; std::map> m_isSubsetCollection; - std::map m_totalEntries; + std::map m_totalEntries; std::vector m_availableCategories; }; diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 86f71fb02..e25ceea88 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -5,14 +5,10 @@ #include "podio/GenericParameters.h" #include "rootUtils.h" -// ROOT specific includes #include "TClass.h" #include #include -#include "datamodel/ExampleMCData.h" -#include "datamodel/ExampleHitData.h" - namespace podio { GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { @@ -48,13 +44,6 @@ namespace podio { for (size_t i = 0; i < keys.size(); ++i) { params.getStringMap()[keys[i]] = valuesString[i]; } - std::cout << "Size of Float map is " << params.getFloatMap().size() << std::endl; - for (auto& [k, v] : params.getFloatMap()) { - std::cout << k << std::endl; - for (auto& x : v) - std::cout << x << " "; - std::cout << std::endl; - } return params; } @@ -112,18 +101,6 @@ void ROOTNTupleReader::openFiles(const std::vector& filenames) { auto availableCategoriesField = m_metadata->GetView>("available_categories"); m_availableCategories = availableCategoriesField(0); - // m_datamodelHolder = DatamodelDefinitionHolder(std::move(*datamodelDefs)); - - // Do some work up front for setting up categories and setup all the chains - // and record the available categories. The rest of the setup follows on - // demand when the category is first read - // m_availCategories = ::podio::getAvailableCategories2(m_metaChain.get()); - // for (const auto& cat : m_availCategories) { - // auto [it, _] = m_categories.try_emplace(cat, std::make_unique(cat.c_str())); - // for (const auto& fn : filenames) { - // it->second.chain->Add(fn.c_str()); - // } - // } } unsigned ROOTNTupleReader::getEntries(const std::string& name) { @@ -167,7 +144,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::map, std::vector*> tmp; - for (int i = 0; i < m_collectionId[category].size(); ++i) { + for (size_t i = 0; i < m_collectionId[category].size(); ++i) { std::cout << "i = " << i << " " << m_collectionId[category][i] << " " << m_collectionType[category][i] << " " << m_collectionName[category][i] << std::endl; const auto collectionClass = TClass::GetClass(m_collectionType[category][i].c_str()); @@ -237,7 +214,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca m_readers[category][0]->LoadEntry(entNum); - for (int i = 0; i < m_collectionId[category].size(); ++i) { + for (size_t i = 0; i < m_collectionId[category].size(); ++i) { auto collBuffers = buffers[m_collectionName[category][i]]; if (auto* refCollections = collBuffers.references) { for (size_t j = 0; j < refCollections->size(); ++j) { @@ -248,39 +225,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } - // auto buf = buffers["mcparticles"]; - // auto ptr = (std::vector*)(buf.data); - // std::cout << "Size of MCData is " << ptr->size() << std::endl; - // for (auto& x : *ptr) { - // std::cout << x.energy << " " << x.PDG << std::endl; - // } - - // auto buf = buffers["hits"]; - // auto ptr = (std::vector*)(buf.data); - // std::cout << "Size of HitData is " << ptr->size() << std::endl; - // for (auto& x : *ptr) { - // std::cout << x.cellID << " " << x.energy << " " << " " << x.x << " " << x.y << " " << x.z << std::endl; - // } - - auto buf = buffers["hitRefs"]; - auto refCollections = buf.references; - for (size_t j = 0; j < refCollections->size(); ++j) { - std::cout << "Size of ObjectID is " << refCollections->at(j)->size() << std::endl; - auto ptr = (std::vector*)refCollections->at(j).get(); - for (auto& x : *ptr) { - std::cout << x.index << " " << x.collectionID << std::endl; - } - } - - // auto nbuf = buffers["hitRefs"]; - // auto nptr = (std::vector>*)(buf.references); - // std::cout << "Size of ObjectID is " << nptr->size() << std::endl; - // for (auto& x : *nptr) { - // auto nnptr = ; - // std::cout << x->index << " " << x->collectionID << std::endl; - // } - - auto parameters = readEventMetaData(category, entNum); auto table = std::make_shared(); @@ -288,8 +232,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto ids = m_collectionId[category]; std::vector> v; - for (int i = 0; i < names.size(); ++i) { - std::cout << ids[i] << " " << names[i] << std::endl; + for (size_t i = 0; i < names.size(); ++i) { v.emplace_back(std::make_pair(int(ids[i]),std::string(names[i]))); } std::sort(v.begin(), v.end()); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 960047ecf..2e9199fd2 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -52,7 +52,6 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& options.SetCompression(ROOT::RCompressionSetting::EDefaults::kUseGeneralPurpose); for (const auto& [name, coll] : collections) { - // coll->prepareForWrite(); auto collBuffers = coll->getBuffers(); if (collBuffers.vecPtr) { std::cout << "Capturing unsafe " << name << std::endl; From 5ea4a97ac2c71fb0a525db5fbaf6dcde662efa62 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Mar 2023 10:47:18 +0100 Subject: [PATCH 12/42] Add names in rootUtils.h, fix a few compiler warnings --- src/ROOTNTupleReader.cc | 42 +++++++++++------------ src/ROOTNTupleWriter.cc | 74 +++++++++++++++++++---------------------- src/rootUtils.h | 35 +++++++++++++++++++ 3 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index e25ceea88..2faf979fe 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -11,18 +11,18 @@ namespace podio { - GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { +GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { GenericParameters params; - auto intKeyView = m_readers[name][0]->GetView>("GP_int_keys"); - auto floatKeyView = m_readers[name][0]->GetView>("GP_float_keys"); - auto doubleKeyView = m_readers[name][0]->GetView>("GP_double_keys"); - auto stringKeyView = m_readers[name][0]->GetView>("GP_string_keys"); + auto intKeyView = m_readers[name][0]->GetView>(root_utils::intKey); + auto floatKeyView = m_readers[name][0]->GetView>(root_utils::floatKey); + auto doubleKeyView = m_readers[name][0]->GetView>(root_utils::doubleKey); + auto stringKeyView = m_readers[name][0]->GetView>(root_utils::stringKey); - auto intValueView = m_readers[name][0]->GetView>>("GP_int_values"); - auto floatValueView = m_readers[name][0]->GetView>>("GP_float_values"); - auto doubleValueView = m_readers[name][0]->GetView>>("GP_double_values"); - auto stringValueView = m_readers[name][0]->GetView>>("GP_string_values"); + auto intValueView = m_readers[name][0]->GetView>>(root_utils::intValue); + auto floatValueView = m_readers[name][0]->GetView>>(root_utils::floatValue); + auto doubleValueView = m_readers[name][0]->GetView>>(root_utils::doubleValue); + auto stringValueView = m_readers[name][0]->GetView>>(root_utils::stringValue); auto keys = intKeyView(entNum); auto valuesInt = intValueView(entNum); @@ -52,15 +52,13 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { if (std::find(m_availableCategories.begin(), m_availableCategories.end(), category) == m_availableCategories.end()) { return false; } - std::cout << "initCategory(" << category << ")" << std::endl; - std::cout << "Getting id" << std::endl; // Assume that the metadata is the same in all files auto filename = m_filenames[0]; auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); m_collectionId[category] = id(0); std::cout << "Getting collectionName" << std::endl; - auto collectionName = m_metadata_readers[filename]->GetView>(category + "_name"); + auto collectionName = m_metadata_readers[filename]->GetView>(root_utils::collectionName(category)); m_collectionName[category] = collectionName(0); std::cout << "Getting collectionType" << std::endl; @@ -68,7 +66,7 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { m_collectionType[category] = collectionType(0); std::cout << "Getting subsetCollection" << std::endl; - auto subsetCollection = m_metadata_readers[filename]->GetView>(category + "_test"); + auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); m_isSubsetCollection[category] = subsetCollection(0); return true; @@ -87,18 +85,17 @@ void ROOTNTupleReader::openFiles(const std::vector& filenames) { } } - m_metadata = ROOT::Experimental::RNTupleReader::Open(root_utils::metaTreeName, "example_rntuple.root"); + m_metadata = ROOT::Experimental::RNTupleReader::Open(root_utils::metaTreeName, filenames[0]); - auto version_view = m_metadata->GetView>(root_utils::versionBranchName); - auto version = version_view(0); + auto versionView = m_metadata->GetView>(root_utils::versionBranchName); + auto version = versionView(0); m_fileVersion = podio::version::Version{version[0], version[1], version[2]}; - std::cout << "Version is " << m_fileVersion.major << " " << m_fileVersion.minor << " " << m_fileVersion.patch << std::endl; - auto edm_view = m_metadata->GetView>>(root_utils::edmDefBranchName); - auto edm = edm_view(0); + auto edmView = m_metadata->GetView>>(root_utils::edmDefBranchName); + auto edm = edmView(0); - auto availableCategoriesField = m_metadata->GetView>("available_categories"); + auto availableCategoriesField = m_metadata->GetView>(root_utils::availableCategories); m_availableCategories = availableCategoriesField(0); } @@ -156,7 +153,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca const auto bufferClass = m_isSubsetCollection[category][i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); auto collBuffers = podio::CollectionReadBuffers(); - // const bool isSubsetColl = bufferClass == nullptr; const bool isSubsetColl = bufferClass == nullptr; if (!isSubsetColl) { collBuffers.data = bufferClass->New(); @@ -211,7 +207,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); } - m_readers[category][0]->LoadEntry(entNum); for (size_t i = 0; i < m_collectionId[category].size(); ++i) { @@ -231,12 +226,13 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto names = m_collectionName[category]; auto ids = m_collectionId[category]; + // Sort the ids and collection names to recreate the collection ID table with + // the names following the order of the IDs std::vector> v; for (size_t i = 0; i < names.size(); ++i) { v.emplace_back(std::make_pair(int(ids[i]),std::string(names[i]))); } std::sort(v.begin(), v.end()); - for (auto& [id, name] : v) { table->add(name); } diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 2e9199fd2..bab419580 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -13,18 +13,17 @@ namespace podio { ROOTNTupleWriter::ROOTNTupleWriter(const std::string& filename) : - m_metadata(nullptr), - m_writers(), - m_file(new TFile(filename.c_str(),"RECREATE","data file")), - m_categories(), - m_collectionId(), - m_collectionName(), - m_collectionType(), - m_isSubsetCollection() + m_metadata(ROOT::Experimental::RNTupleModel::Create()), + m_file(new TFile(filename.c_str(), "RECREATE", "data file")) { - m_metadata = ROOT::Experimental::RNTupleModel::Create(); } +ROOTNTupleWriter::~ROOTNTupleWriter() { + if (!m_finished) { + finish(); + } +} + void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { writeFrame(frame, category, frame.getAvailableCollections()); } @@ -54,8 +53,6 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& for (const auto& [name, coll] : collections) { auto collBuffers = coll->getBuffers(); if (collBuffers.vecPtr) { - std::cout << "Capturing unsafe " << name << std::endl; - entry->CaptureValueUnsafe(name, (void*)collBuffers.vecPtr); } @@ -96,15 +93,15 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& auto doubleMap = params.getDoubleMap(); auto stringMap = params.getStringMap(); - auto gpintKeys = m_writers[category]->GetModel()->Get>("GP_int_keys"); - auto gpfloatKeys = m_writers[category]->GetModel()->Get>("GP_float_keys"); - auto gpdoubleKeys = m_writers[category]->GetModel()->Get>("GP_double_keys"); - auto gpstringKeys = m_writers[category]->GetModel()->Get>("GP_string_keys"); + auto gpintKeys = m_writers[category]->GetModel()->Get>(root_utils::intKey); + auto gpfloatKeys = m_writers[category]->GetModel()->Get>(root_utils::floatKey); + auto gpdoubleKeys = m_writers[category]->GetModel()->Get>(root_utils::doubleKey); + auto gpstringKeys = m_writers[category]->GetModel()->Get>(root_utils::stringKey); - auto gpintValues = m_writers[category]->GetModel()->Get>>("GP_int_values"); - auto gpfloatValues = m_writers[category]->GetModel()->Get>>("GP_float_values"); - auto gpdoubleValues = m_writers[category]->GetModel()->Get>>("GP_double_values"); - auto gpstringValues = m_writers[category]->GetModel()->Get>>("GP_string_values"); + auto gpintValues = m_writers[category]->GetModel()->Get>>(root_utils::intValue); + auto gpfloatValues = m_writers[category]->GetModel()->Get>>(root_utils::floatValue); + auto gpdoubleValues = m_writers[category]->GetModel()->Get>>(root_utils::doubleValue); + auto gpstringValues = m_writers[category]->GetModel()->Get>>(root_utils::stringValue); gpintKeys->clear(); gpintValues->clear(); @@ -118,9 +115,6 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& for (auto& [k, v] : floatMap) { gpfloatKeys->emplace_back(k); gpfloatValues->emplace_back(v); - for (auto& x : v) { - std::cout << "floatMap: " << x << std::endl; - } } gpdoubleKeys->clear(); gpdoubleValues->clear(); @@ -181,15 +175,15 @@ std::unique_ptr ROOTNTupleWriter::createModels // model->MakeField(root_utils::paramBranchName); // gp = Generic Parameters - auto gpintKeys = model->MakeField>("GP_int_keys"); - auto gpfloatKeys = model->MakeField>("GP_float_keys"); - auto gpdoubleKeys = model->MakeField>("GP_double_keys"); - auto gpstringKeys = model->MakeField>("GP_string_keys"); + auto gpintKeys = model->MakeField>(root_utils::intKey); + auto gpfloatKeys = model->MakeField>(root_utils::floatKey); + auto gpdoubleKeys = model->MakeField>(root_utils::doubleKey); + auto gpstringKeys = model->MakeField>(root_utils::stringKey); - auto gpintValues = model->MakeField>>("GP_int_values"); - auto gpfloatValues = model->MakeField>>("GP_float_values"); - auto gpdoubleValues = model->MakeField>>("GP_double_values"); - auto gpstringValues = model->MakeField>>("GP_string_values"); + auto gpintValues = model->MakeField>>(root_utils::intValue); + auto gpfloatValues = model->MakeField>>(root_utils::floatValue); + auto gpdoubleValues = model->MakeField>>(root_utils::doubleValue); + auto gpstringValues = model->MakeField>>(root_utils::stringValue); model->Freeze(); return model; @@ -198,31 +192,31 @@ std::unique_ptr ROOTNTupleWriter::createModels void ROOTNTupleWriter::finish() { auto podioVersion = podio::version::build_version; - auto version_field = m_metadata->MakeField>(root_utils::versionBranchName); - *version_field = {podioVersion.major, podioVersion.minor, podioVersion.patch}; + auto versionField = m_metadata->MakeField>(root_utils::versionBranchName); + *versionField = {podioVersion.major, podioVersion.minor, podioVersion.patch}; auto edmDefinitions = m_datamodelCollector.getDatamodelDefinitionsToWrite(); - auto edm_field = m_metadata->MakeField>>(root_utils::edmDefBranchName); - *edm_field = edmDefinitions; + auto edmField = m_metadata->MakeField>>(root_utils::edmDefBranchName); + *edmField = edmDefinitions; - auto availableCategoriesField = m_metadata->MakeField>("available_categories"); - for (auto& [c, _] : m_collectionId ) { + auto availableCategoriesField = m_metadata->MakeField>(root_utils::availableCategories); + for (auto& [c, _] : m_collectionId) { availableCategoriesField->push_back(c); } for (auto& category : m_categories) { auto idField = m_metadata->MakeField>(root_utils::idTableName(category)); *idField = m_collectionId[category]; - auto collectionNameField = m_metadata->MakeField>(category + "_name"); + auto collectionNameField = m_metadata->MakeField>(root_utils::collectionName(category)); *collectionNameField = m_collectionName[category]; auto collectionTypeField = m_metadata->MakeField>(root_utils::collInfoName(category)); *collectionTypeField = m_collectionType[category]; - auto subsetCollectionField = m_metadata->MakeField>(category + "_test"); + auto subsetCollectionField = m_metadata->MakeField>(root_utils::subsetCollection(category)); *subsetCollectionField = m_isSubsetCollection[category]; } m_metadata->Freeze(); - m_metadataWriter = ROOT::Experimental::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file.get(), {}); + m_metadataWriter = ROOT::Experimental::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file, {}); m_metadataWriter->Fill(); @@ -232,6 +226,8 @@ void ROOTNTupleWriter::finish() { // unwritten output m_writers.clear(); m_metadataWriter.reset(); + + m_finished = true; } } //namespace podio diff --git a/src/rootUtils.h b/src/rootUtils.h index 507d24b15..bd78f31f3 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -32,6 +32,41 @@ constexpr static auto metaTreeName = "podio_metadata"; */ constexpr static auto paramBranchName = "PARAMETERS"; +/** + * Names of the fields with the keys and values of the generic parameters for + * the RNTuples until map types are supported + */ +constexpr static auto intKey = "GPIntKeys"; +constexpr static auto floatKey = "GPFloatKeys"; +constexpr static auto doubleKey = "GPDoubleKeys"; +constexpr static auto stringKey = "GPStringKeys"; + +constexpr static auto intValue = "GPIntValues"; +constexpr static auto floatValue = "GPFloatValues"; +constexpr static auto doubleValue = "GPDoubleValues"; +constexpr static auto stringValue = "GPStringValues"; + +/** + * Name of the field with the list of categories for RNTuples + */ +constexpr static auto availableCategories = "availableCategories"; + +/** + * Name of the field with the names of the collections for RNTuples + */ +inline std::string collectionName(const std::string& category) { + return category + "_collectionNames"; +} + +/** + * Name of the field with the flag for subset collections for RNTuples + */ +inline std::string subsetCollection(const std::string& category) { + return category + "_isSubsetCollections"; +} + + + /** * The name of the branch into which we store the build version of podio at the * time of writing the file From bdf43353b47a777559251fc4806e7ae924c27368 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Mar 2023 10:56:40 +0100 Subject: [PATCH 13/42] Add a few minor changes --- src/ROOTNTupleReader.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 2faf979fe..8aab47389 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -54,6 +54,7 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { } // Assume that the metadata is the same in all files auto filename = m_filenames[0]; + auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); m_collectionId[category] = id(0); @@ -229,8 +230,9 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca // Sort the ids and collection names to recreate the collection ID table with // the names following the order of the IDs std::vector> v; + v.reserve(names.size()); for (size_t i = 0; i < names.size(); ++i) { - v.emplace_back(std::make_pair(int(ids[i]),std::string(names[i]))); + v.emplace_back(std::make_pair(int(ids[i]), std::string(names[i]))); } std::sort(v.begin(), v.end()); for (auto& [id, name] : v) { From f119026192e64caf7dee59a98394dc88e0dc8072 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Mar 2023 10:57:04 +0100 Subject: [PATCH 14/42] Add missing changes in the headers --- include/podio/ROOTNTupleReader.h | 44 +++++++++++++------------------- include/podio/ROOTNTupleWriter.h | 22 ++++++++-------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index c277c111e..c7e9a390c 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -7,11 +7,9 @@ #include "podio/podioVersion.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" -#include #include #include #include -#include #include #include @@ -20,10 +18,6 @@ namespace podio { -class CollectionBase; -class Registry; -class CollectionIDTable; -class GenericParameters; /** This class has the function to read available data from disk and to prepare collections and buffers. @@ -65,36 +59,34 @@ class ROOTNTupleReader{ void closeFile(); private: - std::vector> m_files; - std::unique_ptr m_file; - std::unique_ptr m_metadata; - -private: - /** - * Initialze the passed CategoryInfo by setting up the necessary branches, - * collection infos and all necessary meta data to be able to read entries - * with this name + * Initialize the given category by filling the maps with metadata information + * that will be used later */ bool initCategory(const std::string& category); + /** + * Read and reconstruct the generic parameters of the Frame + */ GenericParameters readEventMetaData(const std::string& name, unsigned entNum); - podio::version::Version m_fileVersion{0, 0, 0}; + std::unique_ptr m_metadata{}; + + podio::version::Version m_fileVersion{}; DatamodelDefinitionHolder m_datamodelHolder{}; - std::map>> m_readers; - std::map> m_metadata_readers; - std::vector m_filenames; - std::map m_entries; + std::map>> m_readers{}; + std::map> m_metadata_readers{}; + std::vector m_filenames{}; - std::map> m_collectionId; - std::map> m_collectionName; - std::map> m_collectionType; - std::map> m_isSubsetCollection; + std::map m_entries{}; + std::map m_totalEntries{}; - std::map m_totalEntries; - std::vector m_availableCategories; + std::map> m_collectionId{}; + std::map> m_collectionName{}; + std::map> m_collectionType{}; + std::map> m_isSubsetCollection{}; + std::vector m_availableCategories{}; }; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index e529fcb0f..11a67c837 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -20,7 +20,7 @@ namespace podio { class ROOTNTupleWriter { public: ROOTNTupleWriter(const std::string& filename); - // ~ROOTNTupleWriter(); + ~ROOTNTupleWriter(); ROOTNTupleWriter(const ROOTNTupleWriter&) = delete; ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; @@ -34,19 +34,21 @@ class ROOTNTupleWriter { using StoreCollection = std::pair; std::unique_ptr createModels(const std::vector& collections); - std::unique_ptr m_metadata {nullptr}; - std::map> m_writers; - std::unique_ptr m_metadataWriter {nullptr}; + std::unique_ptr m_metadata{}; + std::map> m_writers{}; + std::unique_ptr m_metadataWriter{}; - std::unique_ptr m_file {nullptr}; + std::unique_ptr m_file{}; DatamodelDefinitionCollector m_datamodelCollector{}; - std::set m_categories; - std::map> m_collectionId; - std::map> m_collectionName; - std::map> m_collectionType; - std::map> m_isSubsetCollection; + std::map> m_collectionId{}; + std::map> m_collectionName{}; + std::map> m_collectionType{}; + std::map> m_isSubsetCollection{}; + std::set m_categories{}; + + bool m_finished{false}; }; From 2aa55efa691468d967d2fc3e31ffe8bf64c864f7 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 11 Apr 2023 10:38:18 +0200 Subject: [PATCH 15/42] Change map -> unordered_map and use append in CMakeLists.txt --- include/podio/ROOTNTupleReader.h | 19 +++++++++---------- include/podio/ROOTNTupleWriter.h | 13 ++++++------- src/CMakeLists.txt | 6 ++---- src/ROOTNTupleReader.cc | 2 +- src/ROOTNTupleWriter.cc | 2 +- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index c7e9a390c..61b1c8dae 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -7,8 +7,7 @@ #include "podio/podioVersion.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" -#include -#include +#include #include #include @@ -75,17 +74,17 @@ class ROOTNTupleReader{ podio::version::Version m_fileVersion{}; DatamodelDefinitionHolder m_datamodelHolder{}; - std::map>> m_readers{}; - std::map> m_metadata_readers{}; + std::unordered_map>> m_readers{}; + std::unordered_map> m_metadata_readers{}; std::vector m_filenames{}; - std::map m_entries{}; - std::map m_totalEntries{}; + std::unordered_map m_entries{}; + std::unordered_map m_totalEntries{}; - std::map> m_collectionId{}; - std::map> m_collectionName{}; - std::map> m_collectionType{}; - std::map> m_isSubsetCollection{}; + std::unordered_map> m_collectionId{}; + std::unordered_map> m_collectionName{}; + std::unordered_map> m_collectionType{}; + std::unordered_map> m_isSubsetCollection{}; std::vector m_availableCategories{}; }; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 11a67c837..83772b026 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -12,8 +12,7 @@ #include #include -#include -#include +#include namespace podio { @@ -35,17 +34,17 @@ class ROOTNTupleWriter { std::unique_ptr createModels(const std::vector& collections); std::unique_ptr m_metadata{}; - std::map> m_writers{}; + std::unordered_map> m_writers{}; std::unique_ptr m_metadataWriter{}; std::unique_ptr m_file{}; DatamodelDefinitionCollector m_datamodelCollector{}; - std::map> m_collectionId{}; - std::map> m_collectionName{}; - std::map> m_collectionType{}; - std::map> m_isSubsetCollection{}; + std::unordered_map> m_collectionId{}; + std::unordered_map> m_collectionName{}; + std::unordered_map> m_collectionType{}; + std::unordered_map> m_isSubsetCollection{}; std::set m_categories{}; bool m_finished{false}; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d5d26d1c8..a22972291 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -86,8 +86,7 @@ SET(root_sources ROOTLegacyReader.cc ) if(ENABLE_RNTUPLE) - set(root_sources - ${root_sources} + list(APPEND root_sources ROOTNTupleReader.cc ROOTNTupleWriter.cc ) @@ -99,8 +98,7 @@ SET(root_headers ${CMAKE_SOURCE_DIR}/include/podio/ROOTFrameWriter.h ) if(ENABLE_RNTUPLE) - set(root_headers - ${root_headers} + list(APPEND root_headers ${CMAKE_SOURCE_DIR}/include/podio/ROOTNTupleReader.h ${CMAKE_SOURCE_DIR}/include/podio/ROOTNTupleWriter.h ) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 8aab47389..940ba71d5 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -67,7 +67,7 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { m_collectionType[category] = collectionType(0); std::cout << "Getting subsetCollection" << std::endl; - auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); + auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); m_isSubsetCollection[category] = subsetCollection(0); return true; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index bab419580..24734be6a 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -211,7 +211,7 @@ void ROOTNTupleWriter::finish() { *collectionNameField = m_collectionName[category]; auto collectionTypeField = m_metadata->MakeField>(root_utils::collInfoName(category)); *collectionTypeField = m_collectionType[category]; - auto subsetCollectionField = m_metadata->MakeField>(root_utils::subsetCollection(category)); + auto subsetCollectionField = m_metadata->MakeField>(root_utils::subsetCollection(category)); *subsetCollectionField = m_isSubsetCollection[category]; } From 5edd00e1e824537458ff2be646744d5333abb445 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 10:52:53 +0200 Subject: [PATCH 16/42] Simplify writing and reading of generic parameters --- include/podio/GenericParameters.h | 1 + include/podio/ROOTNTupleReader.h | 3 ++ include/podio/ROOTNTupleWriter.h | 3 ++ src/ROOTNTupleReader.cc | 66 ++++++++----------------------- src/ROOTNTupleWriter.cc | 56 ++++++++------------------ src/rootUtils.h | 37 +++++++++++++++++ 6 files changed, 77 insertions(+), 89 deletions(-) diff --git a/include/podio/GenericParameters.h b/include/podio/GenericParameters.h index 311b622b5..0d332b402 100644 --- a/include/podio/GenericParameters.h +++ b/include/podio/GenericParameters.h @@ -187,6 +187,7 @@ class GenericParameters { } } +private: /// Get the mutex that guards the map for the given type template std::mutex& getMutex() const { diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 61b1c8dae..266c7df9d 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -69,6 +69,9 @@ class ROOTNTupleReader{ */ GenericParameters readEventMetaData(const std::string& name, unsigned entNum); + template + void readParams(const std::string& name, unsigned entNum, GenericParameters& params); + std::unique_ptr m_metadata{}; podio::version::Version m_fileVersion{}; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 83772b026..340919e1d 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -24,6 +24,9 @@ class ROOTNTupleWriter { ROOTNTupleWriter(const ROOTNTupleWriter&) = delete; ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; + template + void fillParams(const std::string& category, GenericParameters& params); + void writeFrame(const podio::Frame& frame, const std::string& category); void writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite); void finish(); diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 940ba71d5..e06af79b6 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -11,39 +11,26 @@ namespace podio { -GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { - GenericParameters params; - - auto intKeyView = m_readers[name][0]->GetView>(root_utils::intKey); - auto floatKeyView = m_readers[name][0]->GetView>(root_utils::floatKey); - auto doubleKeyView = m_readers[name][0]->GetView>(root_utils::doubleKey); - auto stringKeyView = m_readers[name][0]->GetView>(root_utils::stringKey); +template +void ROOTNTupleReader::readParams(const std::string& name, unsigned entNum, GenericParameters& params) { + auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKey()); + auto valueView = m_readers[name][0]->GetView>>(root_utils::getGPValue()); - auto intValueView = m_readers[name][0]->GetView>>(root_utils::intValue); - auto floatValueView = m_readers[name][0]->GetView>>(root_utils::floatValue); - auto doubleValueView = m_readers[name][0]->GetView>>(root_utils::doubleValue); - auto stringValueView = m_readers[name][0]->GetView>>(root_utils::stringValue); + auto keys = keyView(entNum); + auto values = valueView(entNum); - auto keys = intKeyView(entNum); - auto valuesInt = intValueView(entNum); - for (size_t i = 0; i < keys.size(); ++i) { - params.getIntMap()[keys[i]] = valuesInt[i]; - } - keys = floatKeyView(entNum); - auto valuesFloat = floatValueView(entNum); - for (size_t i = 0; i < keys.size(); ++i) { - params.getFloatMap()[keys[i]] = valuesFloat[i]; - } - keys = doubleKeyView(entNum); - auto valuesDouble = doubleValueView(entNum); - for (size_t i = 0; i < keys.size(); ++i) { - params.getDoubleMap()[keys[i]] = valuesDouble[i]; - } - keys = stringKeyView(entNum); - auto valuesString = stringValueView(entNum); for (size_t i = 0; i < keys.size(); ++i) { - params.getStringMap()[keys[i]] = valuesString[i]; + params.getMap()[keys[i]] = values[i]; } +} + +GenericParameters ROOTNTupleReader::readEventMetaData(const std::string& name, unsigned entNum) { + GenericParameters params; + + readParams(name, entNum, params); + readParams(name, entNum, params); + readParams(name, entNum, params); + readParams(name, entNum, params); return params; } @@ -58,15 +45,12 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); m_collectionId[category] = id(0); - std::cout << "Getting collectionName" << std::endl; auto collectionName = m_metadata_readers[filename]->GetView>(root_utils::collectionName(category)); m_collectionName[category] = collectionName(0); - std::cout << "Getting collectionType" << std::endl; auto collectionType = m_metadata_readers[filename]->GetView>(root_utils::collInfoName(category)); m_collectionType[category] = collectionType(0); - std::cout << "Getting subsetCollection" << std::endl; auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); m_isSubsetCollection[category] = subsetCollection(0); @@ -134,7 +118,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } } - std::cout << "Reading entry with category " << category << " and number " << entNum << std::endl; m_entries[category] = entNum+1; ROOTFrameData::BufferMap buffers; @@ -143,8 +126,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::map, std::vector*> tmp; for (size_t i = 0; i < m_collectionId[category].size(); ++i) { - std::cout << "i = " << i << " " << m_collectionId[category][i] << " " << m_collectionType[category][i] << " " << m_collectionName[category][i] << std::endl; - const auto collectionClass = TClass::GetClass(m_collectionType[category][i].c_str()); auto collection = @@ -204,7 +185,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } } - std::cout << "CaptureValueUnsafe done" << std::endl; buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); } @@ -222,22 +202,10 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } auto parameters = readEventMetaData(category, entNum); - auto table = std::make_shared(); auto names = m_collectionName[category]; auto ids = m_collectionId[category]; - - // Sort the ids and collection names to recreate the collection ID table with - // the names following the order of the IDs - std::vector> v; - v.reserve(names.size()); - for (size_t i = 0; i < names.size(); ++i) { - v.emplace_back(std::make_pair(int(ids[i]), std::string(names[i]))); - } - std::sort(v.begin(), v.end()); - for (auto& [id, name] : v) { - table->add(name); - } + auto table = std::make_shared(ids, names); return std::make_unique(std::move(buffers), table, std::move(parameters)); } diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 24734be6a..f6f3bb469 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -24,6 +24,18 @@ ROOTNTupleWriter::~ROOTNTupleWriter() { } } +template +void ROOTNTupleWriter::fillParams(const std::string& category, GenericParameters& params) { + auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKey()); + auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValue()); + gpKeys->clear(); + gpValues->clear(); + for (auto& [k, v] : params.getMap()) { + gpKeys->emplace_back(k); + gpValues->emplace_back(v); + } +} + void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { writeFrame(frame, category, frame.getAvailableCollections()); } @@ -88,46 +100,10 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& } auto params = frame.getParameters(); - auto intMap = params.getIntMap(); - auto floatMap = params.getFloatMap(); - auto doubleMap = params.getDoubleMap(); - auto stringMap = params.getStringMap(); - - auto gpintKeys = m_writers[category]->GetModel()->Get>(root_utils::intKey); - auto gpfloatKeys = m_writers[category]->GetModel()->Get>(root_utils::floatKey); - auto gpdoubleKeys = m_writers[category]->GetModel()->Get>(root_utils::doubleKey); - auto gpstringKeys = m_writers[category]->GetModel()->Get>(root_utils::stringKey); - - auto gpintValues = m_writers[category]->GetModel()->Get>>(root_utils::intValue); - auto gpfloatValues = m_writers[category]->GetModel()->Get>>(root_utils::floatValue); - auto gpdoubleValues = m_writers[category]->GetModel()->Get>>(root_utils::doubleValue); - auto gpstringValues = m_writers[category]->GetModel()->Get>>(root_utils::stringValue); - - gpintKeys->clear(); - gpintValues->clear(); - for (auto& [k, v] : intMap) { - gpintKeys->emplace_back(k); - gpintValues->emplace_back(v); - } - - gpfloatKeys->clear(); - gpfloatValues->clear(); - for (auto& [k, v] : floatMap) { - gpfloatKeys->emplace_back(k); - gpfloatValues->emplace_back(v); - } - gpdoubleKeys->clear(); - gpdoubleValues->clear(); - for (auto& [k, v] : doubleMap) { - gpdoubleKeys->emplace_back(k); - gpdoubleValues->emplace_back(v); - } - gpstringKeys->clear(); - gpstringValues->clear(); - for (auto& [k, v] : stringMap) { - gpstringKeys->emplace_back(k); - gpstringValues->emplace_back(v); - } + fillParams(category, params); + fillParams(category, params); + fillParams(category, params); + fillParams(category, params); m_writers[category]->Fill(); m_categories.insert(category); diff --git a/src/rootUtils.h b/src/rootUtils.h index bd78f31f3..a98e86896 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -46,6 +46,43 @@ constexpr static auto floatValue = "GPFloatValues"; constexpr static auto doubleValue = "GPDoubleValues"; constexpr static auto stringValue = "GPStringValues"; +/** + * Get the name of the key depending on the type + */ +template +constexpr auto getGPKey() { + if constexpr (std::is_same::value) { + return intKey; + } else if constexpr (std::is_same::value) { + return floatKey; + } else if constexpr (std::is_same::value) { + return doubleKey; + } else if constexpr (std::is_same::value) { + return stringKey; + } else { + static_assert(sizeof(T) == 0, "Unsupported type for generic parameters"); + } +} + +/** + * Get the name of the value depending on the type + */ +template +constexpr auto getGPValue() { + if constexpr (std::is_same::value) { + return intValue; + } else if constexpr (std::is_same::value) { + return floatValue; + } else if constexpr (std::is_same::value) { + return doubleValue; + } else if constexpr (std::is_same::value) { + return stringValue; + } else { + static_assert(sizeof(T) == 0, "Unsupported type for generic parameters"); + } +} + + /** * Name of the field with the list of categories for RNTuples */ From 4d995e3fb0980c407bd6ea9a269cf2b1a3499ecb Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 11:06:52 +0200 Subject: [PATCH 17/42] Only create the ID table once --- include/podio/ROOTNTupleReader.h | 2 ++ src/ROOTNTupleReader.cc | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 266c7df9d..3591e093f 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -90,6 +90,8 @@ class ROOTNTupleReader{ std::unordered_map> m_isSubsetCollection{}; std::vector m_availableCategories{}; + std::shared_ptr m_table{}; + }; } // namespace podio diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index e06af79b6..94bdc1668 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -202,12 +202,13 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } auto parameters = readEventMetaData(category, entNum); + if (!m_table) { + auto names = m_collectionName[category]; + auto ids = m_collectionId[category]; + m_table = std::make_shared(ids, names); + } - auto names = m_collectionName[category]; - auto ids = m_collectionId[category]; - auto table = std::make_shared(ids, names); - - return std::make_unique(std::move(buffers), table, std::move(parameters)); + return std::make_unique(std::move(buffers), m_table, std::move(parameters)); } } // namespace podio From 647dd87b5d911ca1f125e75e4b4c3da504f5f4f7 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 11:27:22 +0200 Subject: [PATCH 18/42] Add CollectionInfo structs --- include/podio/ROOTNTupleReader.h | 13 ++++++++---- include/podio/ROOTNTupleWriter.h | 13 ++++++++---- src/ROOTNTupleReader.cc | 34 +++++++++++++++----------------- src/ROOTNTupleWriter.cc | 20 +++++++++---------- 4 files changed, 44 insertions(+), 36 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 3591e093f..b903f02df 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -84,10 +84,15 @@ class ROOTNTupleReader{ std::unordered_map m_entries{}; std::unordered_map m_totalEntries{}; - std::unordered_map> m_collectionId{}; - std::unordered_map> m_collectionName{}; - std::unordered_map> m_collectionType{}; - std::unordered_map> m_isSubsetCollection{}; + struct CollectionInfo { + std::vector id{}; + std::vector name{}; + std::vector type{}; + std::vector isSubsetCollection{}; + }; + + std::unordered_map m_collectionInfo{}; + std::vector m_availableCategories{}; std::shared_ptr m_table{}; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 340919e1d..e40ae3af0 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -44,10 +44,15 @@ class ROOTNTupleWriter { DatamodelDefinitionCollector m_datamodelCollector{}; - std::unordered_map> m_collectionId{}; - std::unordered_map> m_collectionName{}; - std::unordered_map> m_collectionType{}; - std::unordered_map> m_isSubsetCollection{}; + struct CollectionInfo { + std::vector id{}; + std::vector name{}; + std::vector type{}; + std::vector isSubsetCollection{}; + }; + + std::unordered_map m_collectionInfo{}; + std::set m_categories{}; bool m_finished{false}; diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 94bdc1668..d39862f78 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -43,16 +43,16 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { auto filename = m_filenames[0]; auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); - m_collectionId[category] = id(0); + m_collectionInfo[category].id = id(0); auto collectionName = m_metadata_readers[filename]->GetView>(root_utils::collectionName(category)); - m_collectionName[category] = collectionName(0); + m_collectionInfo[category].name = collectionName(0); auto collectionType = m_metadata_readers[filename]->GetView>(root_utils::collInfoName(category)); - m_collectionType[category] = collectionType(0); + m_collectionInfo[category].type = collectionType(0); auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); - m_isSubsetCollection[category] = subsetCollection(0); + m_collectionInfo[category].isSubsetCollection = subsetCollection(0); return true; } @@ -112,7 +112,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca return nullptr; } - if (m_collectionId.find(category) == m_collectionId.end()) { + if (m_collectionInfo.find(category) == m_collectionInfo.end()) { if (!initCategory(category)) { return nullptr; } @@ -125,14 +125,14 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::map, std::vector*> tmp; - for (size_t i = 0; i < m_collectionId[category].size(); ++i) { - const auto collectionClass = TClass::GetClass(m_collectionType[category][i].c_str()); + for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { + const auto collectionClass = TClass::GetClass(m_collectionInfo[category].type[i].c_str()); auto collection = std::unique_ptr(static_cast(collectionClass->New())); const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; - const auto bufferClass = m_isSubsetCollection[category][i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); + const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); auto collBuffers = podio::CollectionReadBuffers(); const bool isSubsetColl = bufferClass == nullptr; @@ -159,7 +159,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } if (!isSubsetColl) { - dentry->CaptureValueUnsafe(m_collectionName[category][i], collBuffers.data); + dentry->CaptureValueUnsafe(m_collectionInfo[category].name[i], collBuffers.data); } if (auto* refCollections = collBuffers.references) { for (size_t j = 0; j < refCollections->size(); ++j) { @@ -171,7 +171,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca // dentry->CaptureValueUnsafe(brName, (*refCollections)[j].get()); auto vec = new std::vector; - const auto brName = root_utils::refBranch(m_collectionName[category][i], j); + const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], j); dentry->CaptureValueUnsafe(brName, vec); tmp[{brName, j}] = vec; } @@ -180,21 +180,21 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca if (auto* vecMembers = collBuffers.vectorMembers) { for (size_t j = 0; j < vecMembers->size(); ++j) { const auto typeName = "vector<" + vecMembers->at(j).first + ">"; - const auto brName = root_utils::vecBranch(m_collectionName[category][i], j); + const auto brName = root_utils::vecBranch(m_collectionInfo[category].name[i], j); dentry->CaptureValueUnsafe(brName, vecMembers->at(j).second); } } - buffers.emplace(m_collectionName[category][i], std::move(collBuffers)); + buffers.emplace(m_collectionInfo[category].name[i], std::move(collBuffers)); } m_readers[category][0]->LoadEntry(entNum); - for (size_t i = 0; i < m_collectionId[category].size(); ++i) { - auto collBuffers = buffers[m_collectionName[category][i]]; + for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { + auto collBuffers = buffers[m_collectionInfo[category].name[i]]; if (auto* refCollections = collBuffers.references) { for (size_t j = 0; j < refCollections->size(); ++j) { - const auto brName = root_utils::refBranch(m_collectionName[category][i], j); + const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], j); refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); } } @@ -203,9 +203,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto parameters = readEventMetaData(category, entNum); if (!m_table) { - auto names = m_collectionName[category]; - auto ids = m_collectionId[category]; - m_table = std::make_shared(ids, names); + m_table = std::make_shared(m_collectionInfo[category].id, m_collectionInfo[category].name); } return std::make_unique(std::move(buffers), m_table, std::move(parameters)); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index f6f3bb469..6c6a54faf 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -92,10 +92,10 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& // entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); if (new_category) { - m_collectionId[category].emplace_back(coll->getID()); - m_collectionName[category].emplace_back(name); - m_collectionType[category].emplace_back(coll->getTypeName()); - m_isSubsetCollection[category].emplace_back(coll->isSubsetCollection()); + m_collectionInfo[category].id.emplace_back(coll->getID()); + m_collectionInfo[category].name.emplace_back(name); + m_collectionInfo[category].type.emplace_back(coll->getTypeName()); + m_collectionInfo[category].isSubsetCollection.emplace_back(coll->isSubsetCollection()); } } @@ -176,19 +176,19 @@ void ROOTNTupleWriter::finish() { *edmField = edmDefinitions; auto availableCategoriesField = m_metadata->MakeField>(root_utils::availableCategories); - for (auto& [c, _] : m_collectionId) { + for (auto& [c, _] : m_collectionInfo) { availableCategoriesField->push_back(c); } for (auto& category : m_categories) { auto idField = m_metadata->MakeField>(root_utils::idTableName(category)); - *idField = m_collectionId[category]; + *idField = m_collectionInfo[category].id; auto collectionNameField = m_metadata->MakeField>(root_utils::collectionName(category)); - *collectionNameField = m_collectionName[category]; + *collectionNameField = m_collectionInfo[category].name; auto collectionTypeField = m_metadata->MakeField>(root_utils::collInfoName(category)); - *collectionTypeField = m_collectionType[category]; + *collectionTypeField = m_collectionInfo[category].type; auto subsetCollectionField = m_metadata->MakeField>(root_utils::subsetCollection(category)); - *subsetCollectionField = m_isSubsetCollection[category]; + *subsetCollectionField = m_collectionInfo[category].isSubsetCollection; } m_metadata->Freeze(); @@ -198,7 +198,7 @@ void ROOTNTupleWriter::finish() { m_file->Write(); - // All the tuple writers have to be deleted before the file so that they flush + // All the tuple writers must be deleted before the file so that they flush // unwritten output m_writers.clear(); m_metadataWriter.reset(); From 8fee8b37fc01fb701d5d6b8720dd634d90afd9a2 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 11:33:44 +0200 Subject: [PATCH 19/42] Add a ROOT version check --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index cad3686f9..39586a9f5 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,8 @@ if(NOT ENABLE_RNTUPLE) find_package(ROOT REQUIRED COMPONENTS RIO Tree) else() find_package(ROOT REQUIRED COMPONENTS RIO Tree ROOTNTuple) + if(${ROOT_VERSION} VERSION_LESS 6.28.02) + message(FATAL_ERROR "You are trying to build podio with support for the new ROOT NTuple format, but your ROOT version is too old. Please update ROOT to at least version 6.28.02") endif() # Check that root is compiled with a modern enough c++ standard From a1d97babcbe21c85c13d11a5aff6972eb6c0a17f Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 11:39:48 +0200 Subject: [PATCH 20/42] Add missing endif() --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39586a9f5..35c521882 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,7 @@ else() find_package(ROOT REQUIRED COMPONENTS RIO Tree ROOTNTuple) if(${ROOT_VERSION} VERSION_LESS 6.28.02) message(FATAL_ERROR "You are trying to build podio with support for the new ROOT NTuple format, but your ROOT version is too old. Please update ROOT to at least version 6.28.02") + endif() endif() # Check that root is compiled with a modern enough c++ standard From d8ab05aed423e53f588fbe0672213b79f967504b Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 12:53:36 +0200 Subject: [PATCH 21/42] Add Name at the end of some names --- src/ROOTNTupleReader.cc | 4 ++-- src/ROOTNTupleWriter.cc | 4 ++-- src/rootUtils.h | 36 ++++++++++++++++++------------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index d39862f78..608530104 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -13,8 +13,8 @@ namespace podio { template void ROOTNTupleReader::readParams(const std::string& name, unsigned entNum, GenericParameters& params) { - auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKey()); - auto valueView = m_readers[name][0]->GetView>>(root_utils::getGPValue()); + auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKeyName()); + auto valueView = m_readers[name][0]->GetView>>(root_utils::getGPValueName()); auto keys = keyView(entNum); auto values = valueView(entNum); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 6c6a54faf..deb429b90 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -26,8 +26,8 @@ ROOTNTupleWriter::~ROOTNTupleWriter() { template void ROOTNTupleWriter::fillParams(const std::string& category, GenericParameters& params) { - auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKey()); - auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValue()); + auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKeyName()); + auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValueName()); gpKeys->clear(); gpValues->clear(); for (auto& [k, v] : params.getMap()) { diff --git a/src/rootUtils.h b/src/rootUtils.h index a98e86896..294079f88 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -36,29 +36,29 @@ constexpr static auto paramBranchName = "PARAMETERS"; * Names of the fields with the keys and values of the generic parameters for * the RNTuples until map types are supported */ -constexpr static auto intKey = "GPIntKeys"; -constexpr static auto floatKey = "GPFloatKeys"; -constexpr static auto doubleKey = "GPDoubleKeys"; -constexpr static auto stringKey = "GPStringKeys"; +constexpr static auto intKeyName = "GPIntKeys"; +constexpr static auto floatKeyName = "GPFloatKeys"; +constexpr static auto doubleKeyName = "GPDoubleKeys"; +constexpr static auto stringKeyName = "GPStringKeys"; -constexpr static auto intValue = "GPIntValues"; -constexpr static auto floatValue = "GPFloatValues"; -constexpr static auto doubleValue = "GPDoubleValues"; -constexpr static auto stringValue = "GPStringValues"; +constexpr static auto intValueName = "GPIntValues"; +constexpr static auto floatValueName = "GPFloatValues"; +constexpr static auto doubleValueName = "GPDoubleValues"; +constexpr static auto stringValueName = "GPStringValues"; /** * Get the name of the key depending on the type */ template -constexpr auto getGPKey() { +constexpr auto getGPKeyName() { if constexpr (std::is_same::value) { - return intKey; + return intKeyName; } else if constexpr (std::is_same::value) { - return floatKey; + return floatKeyName; } else if constexpr (std::is_same::value) { - return doubleKey; + return doubleKeyName; } else if constexpr (std::is_same::value) { - return stringKey; + return stringKeyName; } else { static_assert(sizeof(T) == 0, "Unsupported type for generic parameters"); } @@ -68,15 +68,15 @@ constexpr auto getGPKey() { * Get the name of the value depending on the type */ template -constexpr auto getGPValue() { +constexpr auto getGPValueName() { if constexpr (std::is_same::value) { - return intValue; + return intValueName; } else if constexpr (std::is_same::value) { - return floatValue; + return floatValueName; } else if constexpr (std::is_same::value) { - return doubleValue; + return doubleValueName; } else if constexpr (std::is_same::value) { - return stringValue; + return stringValueName; } else { static_assert(sizeof(T) == 0, "Unsupported type for generic parameters"); } From be74a357377a55f7ee46be8964ece88b8832f3ed Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 12:55:24 +0200 Subject: [PATCH 22/42] Add missing Name at the end --- src/ROOTNTupleWriter.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index deb429b90..5bc12bd9b 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -151,15 +151,15 @@ std::unique_ptr ROOTNTupleWriter::createModels // model->MakeField(root_utils::paramBranchName); // gp = Generic Parameters - auto gpintKeys = model->MakeField>(root_utils::intKey); - auto gpfloatKeys = model->MakeField>(root_utils::floatKey); - auto gpdoubleKeys = model->MakeField>(root_utils::doubleKey); - auto gpstringKeys = model->MakeField>(root_utils::stringKey); - - auto gpintValues = model->MakeField>>(root_utils::intValue); - auto gpfloatValues = model->MakeField>>(root_utils::floatValue); - auto gpdoubleValues = model->MakeField>>(root_utils::doubleValue); - auto gpstringValues = model->MakeField>>(root_utils::stringValue); + auto gpintKeys = model->MakeField>(root_utils::intKeyName); + auto gpfloatKeys = model->MakeField>(root_utils::floatKeyName); + auto gpdoubleKeys = model->MakeField>(root_utils::doubleKeyName); + auto gpstringKeys = model->MakeField>(root_utils::stringKeyName); + + auto gpintValues = model->MakeField>>(root_utils::intValueName); + auto gpfloatValues = model->MakeField>>(root_utils::floatValueName); + auto gpdoubleValues = model->MakeField>>(root_utils::doubleValueName); + auto gpstringValues = model->MakeField>>(root_utils::stringValueName); model->Freeze(); return model; From 53e1dd05946bf3667f6e3eb3d7481045b0cb0599 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 13:23:13 +0200 Subject: [PATCH 23/42] Cast to rvalue --- src/ROOTNTupleReader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 608530104..f80cf03dc 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -20,7 +20,7 @@ void ROOTNTupleReader::readParams(const std::string& name, unsigned entNum, Gene auto values = valueView(entNum); for (size_t i = 0; i < keys.size(); ++i) { - params.getMap()[keys[i]] = values[i]; + params.getMap()[keys[i]] = std::move(values[i]); } } From 7216c2dde35984af56eadc2445e401f0faa19acf Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 13:33:22 +0200 Subject: [PATCH 24/42] Cache entries and reserve --- src/ROOTNTupleReader.cc | 2 +- src/ROOTNTupleWriter.cc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index f80cf03dc..ccfd3187d 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -95,8 +95,8 @@ unsigned ROOTNTupleReader::getEntries(const std::string& name) { std::cout << "Category " << name << " not found in file " << filename << std::endl; } } + m_totalEntries[name] = std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); } - m_totalEntries[name] = std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); return m_totalEntries[name]; } diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 5bc12bd9b..644239c35 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -29,7 +29,9 @@ void ROOTNTupleWriter::fillParams(const std::string& category, GenericParameters auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKeyName()); auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValueName()); gpKeys->clear(); + gpKeys->reserve(params.getMap().size()); gpValues->clear(); + gpValues->reserve(params.getMap().size()); for (auto& [k, v] : params.getMap()) { gpKeys->emplace_back(k); gpValues->emplace_back(v); From 01ccc6409c5b5db83f6df7fd53c450adfa74c4d0 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 14:14:00 +0200 Subject: [PATCH 25/42] Add comment and remove old comments --- src/ROOTNTupleReader.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index ccfd3187d..fbbfc8877 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -123,6 +123,8 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca ROOTFrameData::BufferMap buffers; auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); + // This map is needed to keep the pointers to the vectors that + // will be written later alive std::map, std::vector*> tmp; for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { @@ -163,13 +165,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } if (auto* refCollections = collBuffers.references) { for (size_t j = 0; j < refCollections->size(); ++j) { - // // The unique_ptrs are nullptrs at the beginning, we first initialize - // // them and then fill the values with the read data since - // refCollections->at(j) = std::make_unique>(); - // const auto brName = root_utils::refBranch(m_collectionName[category][i], j); - // std::cout << "brName = " << brName << " " << (refCollections->at(j) == nullptr) << std::endl; - // dentry->CaptureValueUnsafe(brName, (*refCollections)[j].get()); - auto vec = new std::vector; const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], j); dentry->CaptureValueUnsafe(brName, vec); From 09cbd61c44d44c5fbcd32e3c01eab2d05f77bdb2 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 17 Apr 2023 14:30:17 +0200 Subject: [PATCH 26/42] Remove a few couts --- src/ROOTNTupleWriter.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 644239c35..79a64a632 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -83,9 +83,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& for (auto& [type, vec] : (*vmInfo)) { const auto typeName = "vector<" + type + ">"; const auto brName = root_utils::vecBranch(name, i++); - std::cout << typeName << " " << brName << std::endl; auto ptr = *(std::vector**)vec; - std::cout << "Vector: Size is " << ptr->size(); entry->CaptureValueUnsafe(brName, ptr); } } @@ -118,8 +116,6 @@ std::unique_ptr ROOTNTupleWriter::createModels if (collBuffers.vecPtr) { auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; - std::cout << name << " " << collClassName << std::endl; - std::cout << "Making field with name = " << name << " and collClassName = " << collClassName << std::endl; auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } @@ -129,7 +125,6 @@ std::unique_ptr ROOTNTupleWriter::createModels for (auto& c [[maybe_unused]] : (*refColls)) { const auto brName = root_utils::refBranch(name, i); auto collClassName = "vector"; - std::cout << "Making reference field with name = " << brName << " and collClassName = " << collClassName << std::endl; auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); model->AddField(std::move(field)); ++i; From 52b419db8e6f4516232bc280811486f17822ea4b Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 18 Apr 2023 11:09:06 +0200 Subject: [PATCH 27/42] Remove intermediate variables and use std::move --- src/ROOTNTupleReader.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index fbbfc8877..9cef9ea89 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -16,11 +16,8 @@ void ROOTNTupleReader::readParams(const std::string& name, unsigned entNum, Gene auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKeyName()); auto valueView = m_readers[name][0]->GetView>>(root_utils::getGPValueName()); - auto keys = keyView(entNum); - auto values = valueView(entNum); - - for (size_t i = 0; i < keys.size(); ++i) { - params.getMap()[keys[i]] = std::move(values[i]); + for (size_t i = 0; i < keyView(entNum).size(); ++i) { + params.getMap().emplace(std::move(keyView(entNum)[i]), std::move(valueView(entNum)[i])); } } From 7379adee955595d12e6365a3eccea1840e936f06 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 19 Apr 2023 14:38:45 +0200 Subject: [PATCH 28/42] Run clang-format --- include/podio/ROOTNTupleReader.h | 10 ++++------ include/podio/ROOTNTupleWriter.h | 12 +++++------- src/ROOTNTupleReader.cc | 28 +++++++++++++++------------- src/ROOTNTupleWriter.cc | 31 +++++++++++++++++-------------- src/rootUtils.h | 3 --- 5 files changed, 41 insertions(+), 43 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index b903f02df..b964d2a85 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -1,27 +1,26 @@ #ifndef PODIO_ROOTNTUPLEREADER_H #define PODIO_ROOTNTUPLEREADER_H -#include "podio/ROOTFrameData.h" #include "podio/CollectionBranches.h" #include "podio/ICollectionProvider.h" +#include "podio/ROOTFrameData.h" #include "podio/podioVersion.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" -#include #include +#include #include #include #include - namespace podio { /** This class has the function to read available data from disk and to prepare collections and buffers. **/ -class ROOTNTupleReader{ +class ROOTNTupleReader { public: ROOTNTupleReader() = default; @@ -69,7 +68,7 @@ class ROOTNTupleReader{ */ GenericParameters readEventMetaData(const std::string& name, unsigned entNum); - template + template void readParams(const std::string& name, unsigned entNum, GenericParameters& params); std::unique_ptr m_metadata{}; @@ -96,7 +95,6 @@ class ROOTNTupleReader{ std::vector m_availableCategories{}; std::shared_ptr m_table{}; - }; } // namespace podio diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index e40ae3af0..2341ca7cb 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -2,17 +2,17 @@ #define PODIO_ROOTNTUPLEWRITER_H #include "podio/CollectionBase.h" +#include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" -#include "podio/Frame.h" #include "TFile.h" #include #include #include -#include #include +#include namespace podio { @@ -24,7 +24,7 @@ class ROOTNTupleWriter { ROOTNTupleWriter(const ROOTNTupleWriter&) = delete; ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; - template + template void fillParams(const std::string& category, GenericParameters& params); void writeFrame(const podio::Frame& frame, const std::string& category); @@ -32,7 +32,6 @@ class ROOTNTupleWriter { void finish(); private: - using StoreCollection = std::pair; std::unique_ptr createModels(const std::vector& collections); @@ -56,9 +55,8 @@ class ROOTNTupleWriter { std::set m_categories{}; bool m_finished{false}; - }; -} //namespace podio +} // namespace podio -#endif //PODIO_ROOTNTUPLEWRITER_H +#endif // PODIO_ROOTNTUPLEWRITER_H diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 9cef9ea89..514740336 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -11,9 +11,9 @@ namespace podio { -template +template void ROOTNTupleReader::readParams(const std::string& name, unsigned entNum, GenericParameters& params) { - auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKeyName()); + auto keyView = m_readers[name][0]->GetView>(root_utils::getGPKeyName()); auto valueView = m_readers[name][0]->GetView>>(root_utils::getGPValueName()); for (size_t i = 0; i < keyView(entNum).size(); ++i) { @@ -42,13 +42,16 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); m_collectionInfo[category].id = id(0); - auto collectionName = m_metadata_readers[filename]->GetView>(root_utils::collectionName(category)); + auto collectionName = + m_metadata_readers[filename]->GetView>(root_utils::collectionName(category)); m_collectionInfo[category].name = collectionName(0); - auto collectionType = m_metadata_readers[filename]->GetView>(root_utils::collInfoName(category)); + auto collectionType = + m_metadata_readers[filename]->GetView>(root_utils::collInfoName(category)); m_collectionInfo[category].type = collectionType(0); - - auto subsetCollection = m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); + + auto subsetCollection = + m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); m_collectionInfo[category].isSubsetCollection = subsetCollection(0); return true; @@ -79,7 +82,6 @@ void ROOTNTupleReader::openFiles(const std::vector& filenames) { auto availableCategoriesField = m_metadata->GetView>(root_utils::availableCategories); m_availableCategories = availableCategoriesField(0); - } unsigned ROOTNTupleReader::getEntries(const std::string& name) { @@ -87,12 +89,12 @@ unsigned ROOTNTupleReader::getEntries(const std::string& name) { for (auto& filename : m_filenames) { try { m_readers[name].emplace_back(ROOT::Experimental::RNTupleReader::Open(name, filename)); - } - catch (const ROOT::Experimental::RException& e) { + } catch (const ROOT::Experimental::RException& e) { std::cout << "Category " << name << " not found in file " << filename << std::endl; } } - m_totalEntries[name] = std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, [](int total, auto& reader) {return total + reader->GetNEntries();}); + m_totalEntries[name] = std::accumulate(m_readers[name].begin(), m_readers[name].end(), 0, + [](int total, auto& reader) { return total + reader->GetNEntries(); }); } return m_totalEntries[name]; } @@ -115,7 +117,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } } - m_entries[category] = entNum+1; + m_entries[category] = entNum + 1; ROOTFrameData::BufferMap buffers; auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); @@ -131,7 +133,8 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::unique_ptr(static_cast(collectionClass->New())); const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; - const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); + const auto bufferClass = + m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); auto collBuffers = podio::CollectionReadBuffers(); const bool isSubsetColl = bufferClass == nullptr; @@ -190,7 +193,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); } } - } auto parameters = readEventMetaData(category, entNum); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 79a64a632..2bb573c9c 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -1,8 +1,8 @@ #include "podio/ROOTNTupleWriter.h" -#include "podio/GenericParameters.h" -#include "rootUtils.h" #include "podio/CollectionBase.h" +#include "podio/GenericParameters.h" #include "podio/podioVersion.h" +#include "rootUtils.h" #include "TFile.h" #include @@ -13,10 +13,9 @@ namespace podio { ROOTNTupleWriter::ROOTNTupleWriter(const std::string& filename) : - m_metadata(ROOT::Experimental::RNTupleModel::Create()), - m_file(new TFile(filename.c_str(), "RECREATE", "data file")) - { - } + m_metadata(ROOT::Experimental::RNTupleModel::Create()), + m_file(new TFile(filename.c_str(), "RECREATE", "data file")) { +} ROOTNTupleWriter::~ROOTNTupleWriter() { if (!m_finished) { @@ -24,7 +23,7 @@ ROOTNTupleWriter::~ROOTNTupleWriter() { } } -template +template void ROOTNTupleWriter::fillParams(const std::string& category, GenericParameters& params) { auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKeyName()); auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValueName()); @@ -89,7 +88,8 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& } // Not supported - // entry->CaptureValueUnsafe(root_utils::paramBranchName, &const_cast(frame.getParameters())); + // entry->CaptureValueUnsafe(root_utils::paramBranchName, + // &const_cast(frame.getParameters())); if (new_category) { m_collectionInfo[category].id.emplace_back(coll->getID()); @@ -109,13 +109,14 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& m_categories.insert(category); } -std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { +std::unique_ptr +ROOTNTupleWriter::createModels(const std::vector& collections) { auto model = ROOT::Experimental::RNTupleModel::Create(); for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); if (collBuffers.vecPtr) { - auto collClassName = "std::vector<" + coll->getDataTypeName() +">"; + auto collClassName = "std::vector<" + coll->getDataTypeName() + ">"; auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } @@ -142,7 +143,7 @@ std::unique_ptr ROOTNTupleWriter::createModels } } } - + // Not supported by ROOT because podio::GenericParameters has map types // so we have to split them manually // model->MakeField(root_utils::paramBranchName); @@ -169,7 +170,8 @@ void ROOTNTupleWriter::finish() { *versionField = {podioVersion.major, podioVersion.minor, podioVersion.patch}; auto edmDefinitions = m_datamodelCollector.getDatamodelDefinitionsToWrite(); - auto edmField = m_metadata->MakeField>>(root_utils::edmDefBranchName); + auto edmField = + m_metadata->MakeField>>(root_utils::edmDefBranchName); *edmField = edmDefinitions; auto availableCategoriesField = m_metadata->MakeField>(root_utils::availableCategories); @@ -189,7 +191,8 @@ void ROOTNTupleWriter::finish() { } m_metadata->Freeze(); - m_metadataWriter = ROOT::Experimental::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file, {}); + m_metadataWriter = + ROOT::Experimental::RNTupleWriter::Append(std::move(m_metadata), root_utils::metaTreeName, *m_file, {}); m_metadataWriter->Fill(); @@ -203,4 +206,4 @@ void ROOTNTupleWriter::finish() { m_finished = true; } -} //namespace podio +} // namespace podio diff --git a/src/rootUtils.h b/src/rootUtils.h index 294079f88..523d5c228 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -82,7 +82,6 @@ constexpr auto getGPValueName() { } } - /** * Name of the field with the list of categories for RNTuples */ @@ -102,8 +101,6 @@ inline std::string subsetCollection(const std::string& category) { return category + "_isSubsetCollections"; } - - /** * The name of the branch into which we store the build version of podio at the * time of writing the file From 027925b2d010664dff9db61d8689d3c5da24b7ab Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 19 Apr 2023 14:53:27 +0200 Subject: [PATCH 29/42] Use clang-format on tests too --- tests/root_io/read_rntuple.cpp | 2 +- tests/root_io/write_rntuple.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/root_io/read_rntuple.cpp b/tests/root_io/read_rntuple.cpp index 3d66dede8..59688b2f2 100644 --- a/tests/root_io/read_rntuple.cpp +++ b/tests/root_io/read_rntuple.cpp @@ -2,5 +2,5 @@ #include "read_frame.h" int main() { - return read_frames("example_rntuple.root"); + return read_frames("example_rntuple.root"); } diff --git a/tests/root_io/write_rntuple.cpp b/tests/root_io/write_rntuple.cpp index a0a4b70d6..ce7810c53 100644 --- a/tests/root_io/write_rntuple.cpp +++ b/tests/root_io/write_rntuple.cpp @@ -1,6 +1,6 @@ #include "podio/ROOTNTupleWriter.h" #include "write_frame.h" -int main(){ +int main() { write_frames("example_rntuple.root"); } From 2629d66f15fe4bde6a7128450c55c480da020792 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 25 Apr 2023 13:27:46 +0200 Subject: [PATCH 30/42] Enable RNTuple I/O in Key4hep CI --- .github/workflows/key4hep.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/key4hep.yml b/.github/workflows/key4hep.yml index 92240b93e..43f69ef67 100644 --- a/.github/workflows/key4hep.yml +++ b/.github/workflows/key4hep.yml @@ -30,7 +30,8 @@ jobs: -DCMAKE_INSTALL_PREFIX=../install \ -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \ - -DUSE_EXTERNAL_CATCH2=AUTO \ + -DUSE_EXTERNAL_CATCH2=ON \ + -DENABLE_RNTUPLE=ON \ -G Ninja .. echo "::endgroup::" echo "::group::Build" From f8c58d1cc992118decca8fbdcea09f81e7ed4b55 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 25 Apr 2023 13:32:00 +0200 Subject: [PATCH 31/42] Check if dev3 workflows come with recent enough ROOT --- .github/workflows/ubuntu.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 16ae05bee..9814ca7cb 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -30,9 +30,10 @@ jobs: -DCMAKE_INSTALL_PREFIX=../install \ -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \ - -DUSE_EXTERNAL_CATCH2=OFF \ - -DPODIO_SET_RPATH=ON \ - -G Ninja .. + -DUSE_EXTERNAL_CATCH2=OFF \ + -DPODIO_SET_RPATH=ON \ + -DENABLE_RNTUPLE=ON \ + -G Ninja .. echo "::endgroup::" echo "::group::Build" ninja -k0 From a4ecb552fef65b36a02f45a460904cbe8d15a6f2 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 25 Apr 2023 16:09:06 +0200 Subject: [PATCH 32/42] Change MakeField to the new signature --- src/ROOTNTupleWriter.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 2bb573c9c..362fb3d82 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -180,13 +180,13 @@ void ROOTNTupleWriter::finish() { } for (auto& category : m_categories) { - auto idField = m_metadata->MakeField>(root_utils::idTableName(category)); + auto idField = m_metadata->MakeField>({root_utils::idTableName(category)}); *idField = m_collectionInfo[category].id; - auto collectionNameField = m_metadata->MakeField>(root_utils::collectionName(category)); + auto collectionNameField = m_metadata->MakeField>({root_utils::collectionName(category)}); *collectionNameField = m_collectionInfo[category].name; - auto collectionTypeField = m_metadata->MakeField>(root_utils::collInfoName(category)); + auto collectionTypeField = m_metadata->MakeField>({root_utils::collInfoName(category)}); *collectionTypeField = m_collectionInfo[category].type; - auto subsetCollectionField = m_metadata->MakeField>(root_utils::subsetCollection(category)); + auto subsetCollectionField = m_metadata->MakeField>({root_utils::subsetCollection(category)}); *subsetCollectionField = m_collectionInfo[category].isSubsetCollection; } From beaf236b6ad4235f5f85d6800ace9e80143af55c Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Sun, 14 May 2023 19:29:41 +0200 Subject: [PATCH 33/42] Update the RNTuple reader and writer to use the buffer factory --- include/podio/ROOTNTupleReader.h | 2 ++ include/podio/ROOTNTupleWriter.h | 2 ++ src/ROOTNTupleReader.cc | 31 ++++++++++++------------------- src/ROOTNTupleWriter.cc | 4 ++++ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index b964d2a85..9454d453a 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -4,6 +4,7 @@ #include "podio/CollectionBranches.h" #include "podio/ICollectionProvider.h" #include "podio/ROOTFrameData.h" +#include "podio/SchemaEvolution.h" #include "podio/podioVersion.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" @@ -88,6 +89,7 @@ class ROOTNTupleReader { std::vector name{}; std::vector type{}; std::vector isSubsetCollection{}; + std::vector schemaVersion{}; }; std::unordered_map m_collectionInfo{}; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 2341ca7cb..9b954c3cb 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -4,6 +4,7 @@ #include "podio/CollectionBase.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" +#include "podio/SchemaEvolution.h" #include "podio/utilities/DatamodelRegistryIOHelpers.h" #include "TFile.h" @@ -48,6 +49,7 @@ class ROOTNTupleWriter { std::vector name{}; std::vector type{}; std::vector isSubsetCollection{}; + std::vector schemaVersion{}; }; std::unordered_map m_collectionInfo{}; diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 514740336..24b864dfc 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -1,5 +1,6 @@ #include "podio/ROOTNTupleReader.h" #include "podio/CollectionBase.h" +#include "podio/CollectionBufferFactory.h" #include "podio/CollectionBuffers.h" #include "podio/CollectionIDTable.h" #include "podio/GenericParameters.h" @@ -54,6 +55,9 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { m_metadata_readers[filename]->GetView>(root_utils::subsetCollection(category)); m_collectionInfo[category].isSubsetCollection = subsetCollection(0); + auto schemaVersion = m_metadata_readers[filename]->GetView>("schemaVersion_" + category); + m_collectionInfo[category].schemaVersion = schemaVersion(0); + return true; } @@ -136,28 +140,17 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); - auto collBuffers = podio::CollectionReadBuffers(); const bool isSubsetColl = bufferClass == nullptr; - if (!isSubsetColl) { - collBuffers.data = bufferClass->New(); - } - collection->setSubsetCollection(isSubsetColl); - auto tmpBuffers = collection->createBuffers(); - collBuffers.createCollection = std::move(tmpBuffers.createCollection); - collBuffers.recast = std::move(tmpBuffers.recast); + const auto& bufferFactory = podio::CollectionBufferFactory::instance(); + auto maybeBuffers = bufferFactory.createBuffers(m_collectionInfo[category].type[i], m_collectionInfo[category].schemaVersion[i], isSubsetColl); + auto collBuffers = maybeBuffers.value_or(podio::CollectionReadBuffers{}); - if (auto* refs = tmpBuffers.references) { - collBuffers.references = new podio::CollRefCollection(refs->size()); - } - if (auto* vminfo = tmpBuffers.vectorMembers) { - collBuffers.vectorMembers = new podio::VectorMembersInfo(); - collBuffers.vectorMembers->reserve(vminfo->size()); - - for (const auto& [type, _] : (*vminfo)) { - const auto* vecClass = TClass::GetClass(("vector<" + type + ">").c_str()); - collBuffers.vectorMembers->emplace_back(type, vecClass->New()); - } + if (!maybeBuffers) { + std::cout << "WARNING: Buffers couldn't be created for collection " << m_collectionInfo[category].name[i] + << " of type " << m_collectionInfo[category].type[i] << " and schema version " << m_collectionInfo[category].schemaVersion[i] + << std::endl; + return nullptr; } if (!isSubsetColl) { diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 362fb3d82..e22f25d64 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -1,6 +1,7 @@ #include "podio/ROOTNTupleWriter.h" #include "podio/CollectionBase.h" #include "podio/GenericParameters.h" +#include "podio/SchemaEvolution.h" #include "podio/podioVersion.h" #include "rootUtils.h" @@ -96,6 +97,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& m_collectionInfo[category].name.emplace_back(name); m_collectionInfo[category].type.emplace_back(coll->getTypeName()); m_collectionInfo[category].isSubsetCollection.emplace_back(coll->isSubsetCollection()); + m_collectionInfo[category].schemaVersion.emplace_back(coll->getSchemaVersion()); } } @@ -188,6 +190,8 @@ void ROOTNTupleWriter::finish() { *collectionTypeField = m_collectionInfo[category].type; auto subsetCollectionField = m_metadata->MakeField>({root_utils::subsetCollection(category)}); *subsetCollectionField = m_collectionInfo[category].isSubsetCollection; + auto schemaVersionField = m_metadata->MakeField>({"schemaVersion_" + category}); + *schemaVersionField = m_collectionInfo[category].schemaVersion; } m_metadata->Freeze(); From ab5cfb213f0db5e60a94db6c86cf2f865b433831 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 15 May 2023 08:04:48 +0200 Subject: [PATCH 34/42] Run clang-format --- src/ROOTNTupleReader.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 24b864dfc..2af5f18d4 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -143,13 +143,14 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca const bool isSubsetColl = bufferClass == nullptr; const auto& bufferFactory = podio::CollectionBufferFactory::instance(); - auto maybeBuffers = bufferFactory.createBuffers(m_collectionInfo[category].type[i], m_collectionInfo[category].schemaVersion[i], isSubsetColl); + auto maybeBuffers = bufferFactory.createBuffers(m_collectionInfo[category].type[i], + m_collectionInfo[category].schemaVersion[i], isSubsetColl); auto collBuffers = maybeBuffers.value_or(podio::CollectionReadBuffers{}); if (!maybeBuffers) { - std::cout << "WARNING: Buffers couldn't be created for collection " << m_collectionInfo[category].name[i] - << " of type " << m_collectionInfo[category].type[i] << " and schema version " << m_collectionInfo[category].schemaVersion[i] - << std::endl; + std::cout << "WARNING: Buffers couldn't be created for collection " << m_collectionInfo[category].name[i] + << " of type " << m_collectionInfo[category].type[i] << " and schema version " + << m_collectionInfo[category].schemaVersion[i] << std::endl; return nullptr; } From 1cd727ece0b6f6dc8f3851183d540ed22a419f2a Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 30 May 2023 10:16:57 +0200 Subject: [PATCH 35/42] Update the RNTuple writer to use a bare model --- include/podio/ROOTNTupleWriter.h | 14 ++++++- src/ROOTNTupleWriter.cc | 72 ++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 9b954c3cb..3e29ef6ac 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -26,7 +26,7 @@ class ROOTNTupleWriter { ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; template - void fillParams(const std::string& category, GenericParameters& params); + void fillParams(GenericParameters& params, ROOT::Experimental::REntry *entry); void writeFrame(const podio::Frame& frame, const std::string& category); void writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite); @@ -57,6 +57,18 @@ class ROOTNTupleWriter { std::set m_categories{}; bool m_finished{false}; + + std::vector m_intkeys{}, m_floatkeys{}, m_doublekeys{}, m_stringkeys{}; + + std::vector> m_intvalues{}; + std::vector> m_floatvalues{}; + std::vector> m_doublevalues{}; + std::vector> m_stringvalues{}; + + template + std::pair&, std::vector>&> + getKeyValueVectors(); + }; } // namespace podio diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index e22f25d64..f49595b2a 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -6,6 +6,7 @@ #include "rootUtils.h" #include "TFile.h" +#include #include #include @@ -25,19 +26,39 @@ ROOTNTupleWriter::~ROOTNTupleWriter() { } template -void ROOTNTupleWriter::fillParams(const std::string& category, GenericParameters& params) { - auto gpKeys = m_writers[category]->GetModel()->Get>(root_utils::getGPKeyName()); - auto gpValues = m_writers[category]->GetModel()->Get>>(root_utils::getGPValueName()); - gpKeys->clear(); - gpKeys->reserve(params.getMap().size()); - gpValues->clear(); - gpValues->reserve(params.getMap().size()); - for (auto& [k, v] : params.getMap()) { - gpKeys->emplace_back(k); - gpValues->emplace_back(v); +std::pair&, std::vector>&> +ROOTNTupleWriter::getKeyValueVectors() { + if constexpr (std::is_same_v) { + return {m_intkeys, m_intvalues}; + } else if constexpr (std::is_same_v) { + return {m_floatkeys, m_floatvalues}; + } else if constexpr (std::is_same_v) { + return {m_doublekeys, m_doublevalues}; + } else if constexpr (std::is_same_v) { + return {m_stringkeys, m_stringvalues}; + } else { + throw std::runtime_error("Unknown type"); } } +template +void ROOTNTupleWriter::fillParams(GenericParameters& params, ROOT::Experimental::REntry *entry) { + auto [key, value] = getKeyValueVectors(); + entry->CaptureValueUnsafe(root_utils::getGPKeyName(), &key); + entry->CaptureValueUnsafe(root_utils::getGPValueName(), &value); + + key.clear(); + key.reserve(params.getMap().size()); + value.clear(); + value.reserve(params.getMap().size()); + + for (auto& [kk, vv] : params.getMap()) { + key.emplace_back(kk); + value.emplace_back(vv); + } + +} + void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { writeFrame(frame, category, frame.getAvailableCollections()); } @@ -59,7 +80,7 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& m_writers[category] = ROOT::Experimental::RNTupleWriter::Append(std::move(model), category, *m_file.get(), {}); } - auto entry = m_writers[category]->GetModel()->GetDefaultEntry(); + auto entry = m_writers[category]->GetModel()->CreateBareEntry(); ROOT::Experimental::RNTupleWriteOptions options; options.SetCompression(ROOT::RCompressionSetting::EDefaults::kUseGeneralPurpose); @@ -102,18 +123,18 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& } auto params = frame.getParameters(); - fillParams(category, params); - fillParams(category, params); - fillParams(category, params); - fillParams(category, params); + fillParams(params, entry.get()); + fillParams(params, entry.get()); + fillParams(params, entry.get()); + fillParams(params, entry.get()); - m_writers[category]->Fill(); + m_writers[category]->Fill(*entry); m_categories.insert(category); } std::unique_ptr ROOTNTupleWriter::createModels(const std::vector& collections) { - auto model = ROOT::Experimental::RNTupleModel::Create(); + auto model = ROOT::Experimental::RNTupleModel::CreateBare(); for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); @@ -150,16 +171,15 @@ ROOTNTupleWriter::createModels(const std::vector& collections) // so we have to split them manually // model->MakeField(root_utils::paramBranchName); - // gp = Generic Parameters - auto gpintKeys = model->MakeField>(root_utils::intKeyName); - auto gpfloatKeys = model->MakeField>(root_utils::floatKeyName); - auto gpdoubleKeys = model->MakeField>(root_utils::doubleKeyName); - auto gpstringKeys = model->MakeField>(root_utils::stringKeyName); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intKeyName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatKeyName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleKeyName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringKeyName, "std::vector>").Unwrap()); - auto gpintValues = model->MakeField>>(root_utils::intValueName); - auto gpfloatValues = model->MakeField>>(root_utils::floatValueName); - auto gpdoubleValues = model->MakeField>>(root_utils::doubleValueName); - auto gpstringValues = model->MakeField>>(root_utils::stringValueName); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intValueName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatValueName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleValueName, "std::vector>").Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringValueName, "std::vector>").Unwrap()); model->Freeze(); return model; From f34ebc7b7fccb5f88d7ef74fbd946d2567321e9c Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 30 May 2023 11:33:43 +0200 Subject: [PATCH 36/42] Add friends for Generic Parameters --- include/podio/GenericParameters.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/podio/GenericParameters.h b/include/podio/GenericParameters.h index 0d332b402..eeee3696f 100644 --- a/include/podio/GenericParameters.h +++ b/include/podio/GenericParameters.h @@ -18,6 +18,11 @@ class write_device; using version_type = uint32_t; // from sio/definitions } // namespace sio +namespace podio { +class ROOTNTupleReader; +class ROOTNTupleWriter; +} // namespace podio + #define DEPR_NON_TEMPLATE \ [[deprecated("Non-templated access will be removed. Switch to templated access functionality")]] @@ -145,6 +150,8 @@ class GenericParameters { friend void writeGenericParameters(sio::write_device& device, const GenericParameters& parameters); friend void readGenericParameters(sio::read_device& device, GenericParameters& parameters, sio::version_type version); + friend ROOTNTupleReader; + friend ROOTNTupleWriter; /// Get a reference to the internal map for a given type template From 659e051fb08f74ff1673e85b5433b0d684267000 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 27 Jun 2023 09:13:00 +0200 Subject: [PATCH 37/42] Update changes after the changes in the collectionID and string_view --- include/podio/ROOTNTupleReader.h | 2 +- include/podio/ROOTNTupleWriter.h | 2 +- src/ROOTNTupleReader.cc | 7 ++++--- src/ROOTNTupleWriter.cc | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/podio/ROOTNTupleReader.h b/include/podio/ROOTNTupleReader.h index 9454d453a..a25f66f2a 100644 --- a/include/podio/ROOTNTupleReader.h +++ b/include/podio/ROOTNTupleReader.h @@ -85,7 +85,7 @@ class ROOTNTupleReader { std::unordered_map m_totalEntries{}; struct CollectionInfo { - std::vector id{}; + std::vector id{}; std::vector name{}; std::vector type{}; std::vector isSubsetCollection{}; diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index 3e29ef6ac..ae8f605f3 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -45,7 +45,7 @@ class ROOTNTupleWriter { DatamodelDefinitionCollector m_datamodelCollector{}; struct CollectionInfo { - std::vector id{}; + std::vector id{}; std::vector name{}; std::vector type{}; std::vector isSubsetCollection{}; diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 2af5f18d4..b01063e57 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -40,7 +40,7 @@ bool ROOTNTupleReader::initCategory(const std::string& category) { // Assume that the metadata is the same in all files auto filename = m_filenames[0]; - auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); + auto id = m_metadata_readers[filename]->GetView>(root_utils::idTableName(category)); m_collectionInfo[category].id = id(0); auto collectionName = @@ -136,9 +136,10 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto collection = std::unique_ptr(static_cast(collectionClass->New())); - const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; + // const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; + const auto bufferClassName = collection->getTypeName(); const auto bufferClass = - m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(bufferClassName.c_str()); + m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(std::string(bufferClassName).c_str()); const bool isSubsetColl = bufferClass == nullptr; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index f49595b2a..7a15611d9 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -139,7 +139,7 @@ ROOTNTupleWriter::createModels(const std::vector& collections) const auto collBuffers = coll->getBuffers(); if (collBuffers.vecPtr) { - auto collClassName = "std::vector<" + coll->getDataTypeName() + ">"; + auto collClassName = "std::vector<" + std::string(coll->getDataTypeName()) + ">"; auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } @@ -202,7 +202,7 @@ void ROOTNTupleWriter::finish() { } for (auto& category : m_categories) { - auto idField = m_metadata->MakeField>({root_utils::idTableName(category)}); + auto idField = m_metadata->MakeField>({root_utils::idTableName(category)}); *idField = m_collectionInfo[category].id; auto collectionNameField = m_metadata->MakeField>({root_utils::collectionName(category)}); *collectionNameField = m_collectionInfo[category].name; From a471949b1a79b4b46e69e8cfb127804c7df3a0c0 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 28 Jun 2023 21:34:29 +0200 Subject: [PATCH 38/42] Run clang-format --- include/podio/ROOTNTupleWriter.h | 6 ++---- src/ROOTNTupleReader.cc | 5 +++-- src/ROOTNTupleWriter.cc | 36 ++++++++++++++++++++------------ 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/include/podio/ROOTNTupleWriter.h b/include/podio/ROOTNTupleWriter.h index ae8f605f3..0f6f6d466 100644 --- a/include/podio/ROOTNTupleWriter.h +++ b/include/podio/ROOTNTupleWriter.h @@ -26,7 +26,7 @@ class ROOTNTupleWriter { ROOTNTupleWriter& operator=(const ROOTNTupleWriter&) = delete; template - void fillParams(GenericParameters& params, ROOT::Experimental::REntry *entry); + void fillParams(GenericParameters& params, ROOT::Experimental::REntry* entry); void writeFrame(const podio::Frame& frame, const std::string& category); void writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite); @@ -66,9 +66,7 @@ class ROOTNTupleWriter { std::vector> m_stringvalues{}; template - std::pair&, std::vector>&> - getKeyValueVectors(); - + std::pair&, std::vector>&> getKeyValueVectors(); }; } // namespace podio diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index b01063e57..74f05bbef 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -138,8 +138,9 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca // const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; const auto bufferClassName = collection->getTypeName(); - const auto bufferClass = - m_collectionInfo[category].isSubsetCollection[i] ? nullptr : TClass::GetClass(std::string(bufferClassName).c_str()); + const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] + ? nullptr + : TClass::GetClass(std::string(bufferClassName).c_str()); const bool isSubsetColl = bufferClass == nullptr; diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 7a15611d9..cb50e7686 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -26,8 +26,7 @@ ROOTNTupleWriter::~ROOTNTupleWriter() { } template -std::pair&, std::vector>&> -ROOTNTupleWriter::getKeyValueVectors() { +std::pair&, std::vector>&> ROOTNTupleWriter::getKeyValueVectors() { if constexpr (std::is_same_v) { return {m_intkeys, m_intvalues}; } else if constexpr (std::is_same_v) { @@ -42,7 +41,7 @@ ROOTNTupleWriter::getKeyValueVectors() { } template -void ROOTNTupleWriter::fillParams(GenericParameters& params, ROOT::Experimental::REntry *entry) { +void ROOTNTupleWriter::fillParams(GenericParameters& params, ROOT::Experimental::REntry* entry) { auto [key, value] = getKeyValueVectors(); entry->CaptureValueUnsafe(root_utils::getGPKeyName(), &key); entry->CaptureValueUnsafe(root_utils::getGPValueName(), &value); @@ -56,7 +55,6 @@ void ROOTNTupleWriter::fillParams(GenericParameters& params, ROOT::Experimental: key.emplace_back(kk); value.emplace_back(vv); } - } void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& category) { @@ -171,15 +169,27 @@ ROOTNTupleWriter::createModels(const std::vector& collections) // so we have to split them manually // model->MakeField(root_utils::paramBranchName); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intKeyName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatKeyName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleKeyName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringKeyName, "std::vector>").Unwrap()); - - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intValueName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatValueName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleValueName, "std::vector>").Unwrap()); - model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringValueName, "std::vector>").Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intKeyName, "std::vector>").Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatKeyName, "std::vector>").Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleKeyName, "std::vector>").Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringKeyName, "std::vector>").Unwrap()); + + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::intValueName, "std::vector>") + .Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::floatValueName, "std::vector>") + .Unwrap()); + model->AddField( + ROOT::Experimental::Detail::RFieldBase::Create(root_utils::doubleValueName, "std::vector>") + .Unwrap()); + model->AddField(ROOT::Experimental::Detail::RFieldBase::Create(root_utils::stringValueName, + "std::vector>") + .Unwrap()); model->Freeze(); return model; From 178e6a14a9fb2f468f2bbc83af88de1d4d41c478 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Mon, 10 Jul 2023 22:27:09 +0200 Subject: [PATCH 39/42] Update the reader and writer to conform to #405 --- src/ROOTNTupleReader.cc | 50 ++++++++++++++++++------- src/ROOTNTupleWriter.cc | 81 ++++++++++++++++++++++++++--------------- 2 files changed, 87 insertions(+), 44 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 74f05bbef..718d725ff 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -3,6 +3,7 @@ #include "podio/CollectionBufferFactory.h" #include "podio/CollectionBuffers.h" #include "podio/CollectionIDTable.h" +#include "podio/DatamodelRegistry.h" #include "podio/GenericParameters.h" #include "rootUtils.h" @@ -136,7 +137,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto collection = std::unique_ptr(static_cast(collectionClass->New())); - // const std::string bufferClassName = "std::vector<" + collection->getDataTypeName() + ">"; const auto bufferClassName = collection->getTypeName(); const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] ? nullptr @@ -156,23 +156,27 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca return nullptr; } - if (!isSubsetColl) { + if (isSubsetColl) { + auto brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); + auto vec = new std::vector; + dentry->CaptureValueUnsafe(brName, vec); + tmp[{brName, 0}] = vec; + } else { dentry->CaptureValueUnsafe(m_collectionInfo[category].name[i], collBuffers.data); - } - if (auto* refCollections = collBuffers.references) { - for (size_t j = 0; j < refCollections->size(); ++j) { + + const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(collection->getTypeName()); + for (size_t j = 0; j < relVecNames.relations.size(); ++j) { + const auto relName = relVecNames.relations[j]; auto vec = new std::vector; - const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], j); + const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], relName); dentry->CaptureValueUnsafe(brName, vec); tmp[{brName, j}] = vec; } - } - if (auto* vecMembers = collBuffers.vectorMembers) { - for (size_t j = 0; j < vecMembers->size(); ++j) { - const auto typeName = "vector<" + vecMembers->at(j).first + ">"; - const auto brName = root_utils::vecBranch(m_collectionInfo[category].name[i], j); - dentry->CaptureValueUnsafe(brName, vecMembers->at(j).second); + for (size_t j = 0; j < relVecNames.vectorMembers.size(); ++j) { + const auto vecName = relVecNames.vectorMembers[j]; + const auto brName = root_utils::vecBranch(m_collectionInfo[category].name[i], vecName); + dentry->CaptureValueUnsafe(brName, collBuffers.vectorMembers->at(j).second); } } @@ -182,11 +186,29 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca m_readers[category][0]->LoadEntry(entNum); for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { + const auto collectionClass = TClass::GetClass(m_collectionInfo[category].type[i].c_str()); + auto collection = + std::unique_ptr(static_cast(collectionClass->New())); + const auto bufferClassName = collection->getTypeName(); + const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] + ? nullptr + : TClass::GetClass(std::string(bufferClassName).c_str()); auto collBuffers = buffers[m_collectionInfo[category].name[i]]; + const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(collection->getTypeName()); if (auto* refCollections = collBuffers.references) { - for (size_t j = 0; j < refCollections->size(); ++j) { - const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], j); + size_t maxj = refCollections->size(); + if (bufferClass == nullptr) { + maxj = 1; + } + for (size_t j = 0; j < maxj; ++j) { + if (bufferClass != nullptr) { + auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], relVecNames.relations[j]); refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); + } + if (bufferClass == nullptr) { + auto brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); + refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); + } } } } diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index cb50e7686..1c497396b 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -1,6 +1,7 @@ #include "podio/ROOTNTupleWriter.h" #include "podio/CollectionBase.h" #include "podio/GenericParameters.h" +#include "podio/DatamodelRegistry.h" #include "podio/SchemaEvolution.h" #include "podio/podioVersion.h" #include "rootUtils.h" @@ -89,21 +90,31 @@ void ROOTNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& entry->CaptureValueUnsafe(name, (void*)collBuffers.vecPtr); } - if (auto refColls = collBuffers.references) { - int i = 0; - for (auto& c : (*refColls)) { - const auto brName = root_utils::refBranch(name, i++); - entry->CaptureValueUnsafe(brName, c.get()); + if (coll->isSubsetCollection()) { + auto& refColl = (*collBuffers.references)[0]; + const auto brName = root_utils::subsetBranch(name); + entry->CaptureValueUnsafe(brName, refColl.get()); + } else { + + const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(coll->getValueTypeName()); + if (auto refColls = collBuffers.references) { + int i = 0; + for (auto& c : (*refColls)) { + const auto brName = root_utils::refBranch(name, relVecNames.relations[i]); + entry->CaptureValueUnsafe(brName, c.get()); + ++i; + } } - } - if (auto vmInfo = collBuffers.vectorMembers) { - int i = 0; - for (auto& [type, vec] : (*vmInfo)) { - const auto typeName = "vector<" + type + ">"; - const auto brName = root_utils::vecBranch(name, i++); - auto ptr = *(std::vector**)vec; - entry->CaptureValueUnsafe(brName, ptr); + if (auto vmInfo = collBuffers.vectorMembers) { + int i = 0; + for (auto& [type, vec] : (*vmInfo)) { + const auto typeName = "vector<" + type + ">"; + const auto brName = root_utils::vecBranch(name, relVecNames.vectorMembers[i]); + auto ptr = *(std::vector**)vec; + entry->CaptureValueUnsafe(brName, ptr); + ++i; + } } } @@ -136,31 +147,41 @@ ROOTNTupleWriter::createModels(const std::vector& collections) for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); + if (collBuffers.vecPtr) { auto collClassName = "std::vector<" + std::string(coll->getDataTypeName()) + ">"; auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); model->AddField(std::move(field)); } - if (auto refColls = collBuffers.references) { - int i = 0; - for (auto& c [[maybe_unused]] : (*refColls)) { - const auto brName = root_utils::refBranch(name, i); - auto collClassName = "vector"; - auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); - model->AddField(std::move(field)); - ++i; + if (coll->isSubsetCollection()) { + const auto brName = root_utils::subsetBranch(name); + auto collClassName = "vector"; + auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); + model->AddField(std::move(field)); + } else { + + const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(coll->getValueTypeName()); + if (auto refColls = collBuffers.references) { + int i = 0; + for (auto& c [[maybe_unused]] : (*refColls)) { + const auto brName = root_utils::refBranch(name, relVecNames.relations[i]); + auto collClassName = "vector"; + auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, collClassName).Unwrap(); + model->AddField(std::move(field)); + ++i; + } } - } - if (auto vminfo = collBuffers.vectorMembers) { - int i = 0; - for (auto& [type, vec] : (*vminfo)) { - const auto typeName = "vector<" + type + ">"; - const auto brName = root_utils::vecBranch(name, i); - auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, typeName).Unwrap(); - model->AddField(std::move(field)); - ++i; + if (auto vminfo = collBuffers.vectorMembers) { + int i = 0; + for (auto& [type, vec] : (*vminfo)) { + const auto typeName = "vector<" + type + ">"; + const auto brName = root_utils::vecBranch(name, relVecNames.vectorMembers[i]); + auto field = ROOT::Experimental::Detail::RFieldBase::Create(brName, typeName).Unwrap(); + model->AddField(std::move(field)); + ++i; + } } } } From 5be72230f4175263ba1b90c725ca5e4c4bd7393b Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 11 Jul 2023 08:38:30 +0200 Subject: [PATCH 40/42] Reorganize and clean up code in the reader --- src/ROOTNTupleReader.cc | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 718d725ff..bdc6498db 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -137,16 +137,10 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto collection = std::unique_ptr(static_cast(collectionClass->New())); - const auto bufferClassName = collection->getTypeName(); - const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] - ? nullptr - : TClass::GetClass(std::string(bufferClassName).c_str()); - - const bool isSubsetColl = bufferClass == nullptr; - const auto& bufferFactory = podio::CollectionBufferFactory::instance(); auto maybeBuffers = bufferFactory.createBuffers(m_collectionInfo[category].type[i], - m_collectionInfo[category].schemaVersion[i], isSubsetColl); + m_collectionInfo[category].schemaVersion[i], + m_collectionInfo[category].isSubsetCollection[i]); auto collBuffers = maybeBuffers.value_or(podio::CollectionReadBuffers{}); if (!maybeBuffers) { @@ -156,7 +150,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca return nullptr; } - if (isSubsetColl) { + if (m_collectionInfo[category].isSubsetCollection[i]) { auto brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); auto vec = new std::vector; dentry->CaptureValueUnsafe(brName, vec); @@ -189,26 +183,27 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca const auto collectionClass = TClass::GetClass(m_collectionInfo[category].type[i].c_str()); auto collection = std::unique_ptr(static_cast(collectionClass->New())); - const auto bufferClassName = collection->getTypeName(); - const auto bufferClass = m_collectionInfo[category].isSubsetCollection[i] - ? nullptr - : TClass::GetClass(std::string(bufferClassName).c_str()); + auto collBuffers = buffers[m_collectionInfo[category].name[i]]; const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(collection->getTypeName()); if (auto* refCollections = collBuffers.references) { - size_t maxj = refCollections->size(); - if (bufferClass == nullptr) { + size_t maxj; + // If it is a subset collection, only one reference collection is filled + if (m_collectionInfo[category].isSubsetCollection[i]) { maxj = 1; } + else { + maxj = refCollections->size(); + } for (size_t j = 0; j < maxj; ++j) { - if (bufferClass != nullptr) { - auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], relVecNames.relations[j]); - refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); + std::string brName; + if (m_collectionInfo[category].isSubsetCollection[i]) { + brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); } - if (bufferClass == nullptr) { - auto brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); - refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); + else { + brName = root_utils::refBranch(m_collectionInfo[category].name[i], relVecNames.relations[j]); } + refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); } } } From ec5bcf3e52bc61d6e01269f82902005438b37353 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 11 Jul 2023 08:40:20 +0200 Subject: [PATCH 41/42] Run clang-format --- src/ROOTNTupleReader.cc | 16 +++++++--------- src/ROOTNTupleWriter.cc | 3 +-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index bdc6498db..65214a9b2 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -138,9 +138,9 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca std::unique_ptr(static_cast(collectionClass->New())); const auto& bufferFactory = podio::CollectionBufferFactory::instance(); - auto maybeBuffers = bufferFactory.createBuffers(m_collectionInfo[category].type[i], - m_collectionInfo[category].schemaVersion[i], - m_collectionInfo[category].isSubsetCollection[i]); + auto maybeBuffers = + bufferFactory.createBuffers(m_collectionInfo[category].type[i], m_collectionInfo[category].schemaVersion[i], + m_collectionInfo[category].isSubsetCollection[i]); auto collBuffers = maybeBuffers.value_or(podio::CollectionReadBuffers{}); if (!maybeBuffers) { @@ -155,7 +155,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto vec = new std::vector; dentry->CaptureValueUnsafe(brName, vec); tmp[{brName, 0}] = vec; - } else { + } else { dentry->CaptureValueUnsafe(m_collectionInfo[category].name[i], collBuffers.data); const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(collection->getTypeName()); @@ -168,7 +168,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca } for (size_t j = 0; j < relVecNames.vectorMembers.size(); ++j) { - const auto vecName = relVecNames.vectorMembers[j]; + const auto vecName = relVecNames.vectorMembers[j]; const auto brName = root_utils::vecBranch(m_collectionInfo[category].name[i], vecName); dentry->CaptureValueUnsafe(brName, collBuffers.vectorMembers->at(j).second); } @@ -191,16 +191,14 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca // If it is a subset collection, only one reference collection is filled if (m_collectionInfo[category].isSubsetCollection[i]) { maxj = 1; - } - else { + } else { maxj = refCollections->size(); } for (size_t j = 0; j < maxj; ++j) { std::string brName; if (m_collectionInfo[category].isSubsetCollection[i]) { brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); - } - else { + } else { brName = root_utils::refBranch(m_collectionInfo[category].name[i], relVecNames.relations[j]); } refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); diff --git a/src/ROOTNTupleWriter.cc b/src/ROOTNTupleWriter.cc index 1c497396b..741af53b3 100644 --- a/src/ROOTNTupleWriter.cc +++ b/src/ROOTNTupleWriter.cc @@ -1,7 +1,7 @@ #include "podio/ROOTNTupleWriter.h" #include "podio/CollectionBase.h" -#include "podio/GenericParameters.h" #include "podio/DatamodelRegistry.h" +#include "podio/GenericParameters.h" #include "podio/SchemaEvolution.h" #include "podio/podioVersion.h" #include "rootUtils.h" @@ -147,7 +147,6 @@ ROOTNTupleWriter::createModels(const std::vector& collections) for (auto& [name, coll] : collections) { const auto collBuffers = coll->getBuffers(); - if (collBuffers.vecPtr) { auto collClassName = "std::vector<" + std::string(coll->getDataTypeName()) + ">"; auto field = ROOT::Experimental::Detail::RFieldBase::Create(name, collClassName).Unwrap(); From b0f654c9604dc8650a555f6d96871799b3f32385 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 11 Jul 2023 08:55:10 +0200 Subject: [PATCH 42/42] Simplify how the references are filled --- src/ROOTNTupleReader.cc | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/src/ROOTNTupleReader.cc b/src/ROOTNTupleReader.cc index 65214a9b2..299f88da3 100644 --- a/src/ROOTNTupleReader.cc +++ b/src/ROOTNTupleReader.cc @@ -127,10 +127,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca ROOTFrameData::BufferMap buffers; auto dentry = m_readers[category][0]->GetModel()->GetDefaultEntry(); - // This map is needed to keep the pointers to the vectors that - // will be written later alive - std::map, std::vector*> tmp; - for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { const auto collectionClass = TClass::GetClass(m_collectionInfo[category].type[i].c_str()); @@ -154,7 +150,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); auto vec = new std::vector; dentry->CaptureValueUnsafe(brName, vec); - tmp[{brName, 0}] = vec; + collBuffers.references->at(0) = std::unique_ptr>(vec); } else { dentry->CaptureValueUnsafe(m_collectionInfo[category].name[i], collBuffers.data); @@ -164,7 +160,7 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca auto vec = new std::vector; const auto brName = root_utils::refBranch(m_collectionInfo[category].name[i], relName); dentry->CaptureValueUnsafe(brName, vec); - tmp[{brName, j}] = vec; + collBuffers.references->at(j) = std::unique_ptr>(vec); } for (size_t j = 0; j < relVecNames.vectorMembers.size(); ++j) { @@ -179,33 +175,6 @@ std::unique_ptr ROOTNTupleReader::readEntry(const std::string& ca m_readers[category][0]->LoadEntry(entNum); - for (size_t i = 0; i < m_collectionInfo[category].id.size(); ++i) { - const auto collectionClass = TClass::GetClass(m_collectionInfo[category].type[i].c_str()); - auto collection = - std::unique_ptr(static_cast(collectionClass->New())); - - auto collBuffers = buffers[m_collectionInfo[category].name[i]]; - const auto relVecNames = podio::DatamodelRegistry::instance().getRelationNames(collection->getTypeName()); - if (auto* refCollections = collBuffers.references) { - size_t maxj; - // If it is a subset collection, only one reference collection is filled - if (m_collectionInfo[category].isSubsetCollection[i]) { - maxj = 1; - } else { - maxj = refCollections->size(); - } - for (size_t j = 0; j < maxj; ++j) { - std::string brName; - if (m_collectionInfo[category].isSubsetCollection[i]) { - brName = root_utils::subsetBranch(m_collectionInfo[category].name[i]); - } else { - brName = root_utils::refBranch(m_collectionInfo[category].name[i], relVecNames.relations[j]); - } - refCollections->at(j) = std::unique_ptr>(tmp[{brName, j}]); - } - } - } - auto parameters = readEventMetaData(category, entNum); if (!m_table) { m_table = std::make_shared(m_collectionInfo[category].id, m_collectionInfo[category].name);