From a0bf553ed5d3be01dc9befccf24877fb74d4255c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 14 Nov 2023 08:06:29 +0100
Subject: [PATCH 001/127] Add an IOSvc, an algorithm for reading and writing

---
 k4FWCore/components/IIOSvc.h                  |  50 ++++++
 k4FWCore/components/IOSvc.cpp                 |  65 ++++++++
 k4FWCore/components/IOSvc.h                   |  94 +++++++++++
 k4FWCore/components/Reader.cpp                | 154 ++++++++++++++++++
 k4FWCore/components/SaveFile.cpp              | 128 +++++++++++++++
 .../options/AvalancheSchedulerSimpleTest.py   |  73 +++++++++
 6 files changed, 564 insertions(+)
 create mode 100644 k4FWCore/components/IIOSvc.h
 create mode 100644 k4FWCore/components/IOSvc.cpp
 create mode 100644 k4FWCore/components/IOSvc.h
 create mode 100644 k4FWCore/components/Reader.cpp
 create mode 100644 k4FWCore/components/SaveFile.cpp
 create mode 100644 test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
new file mode 100644
index 00000000..922daf3d
--- /dev/null
+++ b/k4FWCore/components/IIOSvc.h
@@ -0,0 +1,50 @@
+/*****************************************************************************\
+* (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration      *
+*                                                                             *
+* This software is distributed under the terms of the GNU General Public      *
+* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   *
+*                                                                             *
+* In applying this licence, CERN does not waive the privileges and immunities *
+* granted to it by virtue of its status as an Intergovernmental Organization  *
+* or submit itself to any jurisdiction.                                       *
+\*****************************************************************************/
+#pragma once
+
+#include "GaudiKernel/IInterface.h"
+
+#include "podio/CollectionBase.h"
+#include "podio/ROOTFrameWriter.h"
+
+#include <exception>
+#include <memory>
+#include <vector>
+
+#pragma once
+
+/**
+ * The interface implemented by any class making IO and reading RawEvent Data
+ */
+class IIOSvc : virtual public IInterface {
+
+public:
+  struct EndOfInput : std::logic_error {
+    EndOfInput() : logic_error( "Reached end of input while more data were expected" ){};
+  };
+
+public:
+  /// InterfaceID
+  DeclareInterfaceID( IIOSvc, 1, 0 );
+
+  /**
+   * get next event from input
+   * @return a pair RawEvent, shared_ptr<Buffer> where the second one holds the data pointed to
+   * by the first one
+   * @throws IIOSvc::EndOfInput
+   */
+  virtual std::vector<std::shared_ptr<podio::CollectionBase>> next( ) = 0;
+  virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
+
+  virtual std::shared_ptr<podio::ROOTFrameWriter> getWriter() = 0;
+  virtual void deleteWriter() = 0;
+  virtual void deleteReader() = 0;
+};
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
new file mode 100644
index 00000000..64270a63
--- /dev/null
+++ b/k4FWCore/components/IOSvc.cpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "IOSvc.h"
+
+#include "edm4hep/MCParticleCollection.h"
+#include "podio/Frame.h"
+#include "podio/FrameCategories.h"
+
+#include "k4FWCore/KeepDropSwitch.h"
+
+#include <mutex>
+#include <thread>
+#include <tuple>
+
+StatusCode IOSvc::initialize() {
+  m_reader = std::make_unique<podio::ROOTFrameReader>();
+  m_reader->openFiles(m_fileNames);
+
+  m_switch = KeepDropSwitch(m_outputCommands);
+
+  return Service::initialize();
+}
+
+StatusCode IOSvc::finalize() {
+
+  return Service::finalize();
+}
+
+std::vector<std::shared_ptr<podio::CollectionBase>> IOSvc::next() {
+  info() << "IOSvc::next()" << endmsg;
+  podio::Frame* frame;
+  {
+    std::scoped_lock<std::mutex> lock(m_changeBufferLock);
+    frame = new podio::Frame(std::move(m_reader->readNextEntry(podio::Category::Event)));
+  }
+
+  std::vector<std::shared_ptr<podio::CollectionBase>> collections;
+
+  for (const auto& name : m_collectionNames) {
+    info() << "Collection name: " << name << endmsg;
+    auto ptr = const_cast<podio::CollectionBase*>(frame->get(name));
+    collections.push_back(std::shared_ptr<podio::CollectionBase>(ptr));
+  }
+
+  return collections;
+}
+
+DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
new file mode 100644
index 00000000..d4e70c97
--- /dev/null
+++ b/k4FWCore/components/IOSvc.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_IOSVC_H
+#define FWCORE_IOSVC_H
+
+#include "Gaudi/Property.h"
+#include "GaudiKernel/Service.h"
+
+#include "podio/ROOTFrameReader.h"
+#include "podio/ROOTFrameWriter.h"
+#include "podio/ROOTNTupleReader.h"
+#include "podio/ROOTNTupleWriter.h"
+
+#include "k4FWCore/KeepDropSwitch.h"
+
+#include "IIOSvc.h"
+
+#include <string>
+#include <type_traits>
+#include <vector>
+
+class IOSvc : public extends<Service, IIOSvc> {
+  using extends::extends;
+public:
+  // Gaudi doesn't run the destructor of the Services so we have to
+  // manually ask for the writer to be deleted so it will call finish()
+  ~IOSvc() override = default;
+
+  StatusCode initialize() override;
+  StatusCode finalize() override;
+
+  std::vector<std::shared_ptr<podio::CollectionBase>> next() override;
+
+  std::shared_ptr<std::vector<std::string>> getCollectionNames() const override {
+    return std::make_shared<std::vector<std::string>>(m_collectionNames);
+  }
+
+protected:
+  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
+  Gaudi::Property<std::vector<std::string>> m_fileNames{this, "FileNames", {}, "List of files to read"};
+  Gaudi::Property<unsigned int> m_bufferNbEvents{
+      this, "BufferNbEvents", 20000,
+      "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
+  Gaudi::Property<std::vector<std::string>> m_input{this, "Input", {}, "List of inputs"};
+  Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
+  Gaudi::Property<std::string> m_output{this, "Output", "output.root", "Output file name"};
+  Gaudi::Property<std::vector<std::string>> m_outputCommands{
+      this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
+
+  /// lock for handling the change of buffer
+  std::mutex m_changeBufferLock;
+
+  KeepDropSwitch m_switch;
+
+  // To be changed by a base class to allow to use different readers
+  std::unique_ptr<podio::ROOTFrameReader> m_reader{nullptr};
+  std::shared_ptr<podio::ROOTFrameWriter> m_writer{nullptr};
+
+  std::shared_ptr<podio::ROOTFrameWriter> getWriter() override {
+    if (!m_writer) {
+      m_writer = std::shared_ptr<podio::ROOTFrameWriter>(new podio::ROOTFrameWriter(m_output));
+    }
+    return m_writer;
+  }
+
+  void deleteWriter() override {
+    if (m_writer) {
+      m_writer = nullptr;
+    }
+  }
+  void deleteReader() override {
+    if (m_reader) {
+      m_reader = nullptr;
+    }
+  }
+};
+
+#endif
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
new file mode 100644
index 00000000..0f8cdb8b
--- /dev/null
+++ b/k4FWCore/components/Reader.cpp
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+#include "GaudiKernel/FunctionalFilterDecision.h"
+#include "GaudiKernel/StatusCode.h"
+
+#include "IIOSvc.h"
+
+#include <memory>
+
+namespace Gaudi::Functional {
+template <typename Container>
+using vector_of_ = std::vector<Container>;
+template <typename Container>
+using vector_of_optional_ = std::vector<std::optional<Container>>;
+
+namespace details {
+
+  template <typename Signature, typename Traits_>
+  class MyTransformer;
+
+  template <typename Out, typename Traits_>
+  class MyTransformer<vector_of_<Out>(), Traits_> : public BaseClass_t<Traits_> {
+    using base_class = BaseClass_t<Traits_>;
+    static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
+
+  public:
+    using KeyValues = std::pair<std::string, std::vector<std::string>>;
+
+    MyTransformer(std::string name, ISvcLocator* locator, const KeyValues& outputs) :
+        base_class(std::move(name), locator),
+        m_outputLocations(
+            this, outputs.first, details::to_DataObjID(outputs.second),
+            [this](Gaudi::Details::PropertyBase&) {
+              this->m_outputs = details::make_vector_of_handles<decltype(this->m_outputs)>(this, m_outputLocations);
+              if constexpr (details::is_optional_v<Out>) { // handle constructor does not (yet) allow to
+                                                           // set optional flag... so
+                                                           // do it explicitly here...
+                std::for_each(this->m_outputs.begin(), this->m_outputs.end(), [](auto& h) { h.setOptional(true); });
+              }
+            },
+            Gaudi::Details::Property::ImmediatelyInvokeHandler{true}) {
+    }
+
+    // derived classes can NOT implement execute
+    StatusCode execute(const EventContext&) const override final {
+      Gaudi::Algorithm::info() << "MyTransformer::execute()" << endmsg;
+      try {
+        // TODO:FIXME: how does operator() know the number and order of expected outputs?
+        auto out = (*this)();
+        if (out.size() != m_outputs.size()) {
+          throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
+                                   " containers, got " + std::to_string(out.size()) + " instead",
+                               this->name(), StatusCode::FAILURE);
+        }
+        for (unsigned i = 0; i != out.size(); ++i) {
+          Gaudi::Algorithm::info() << "MyTransformer::execute() : putting " << m_outputs[i].fullKey() << endmsg;
+          details::put(m_outputs[i], std::move(out[i]));
+        }
+        return FilterDecision::PASSED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // TODO/FIXME: how does the callee know in which order to produce the outputs?
+    //             (note: 'missing' items can be specified by making Out an std::optional<Out>,
+    //              and only those entries which contain an Out are stored)
+    virtual vector_of_<Out> operator()() const = 0;
+
+  private:
+    // if In is a pointer, it signals optional (as opposed to mandatory) input
+    template <typename T>
+    using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+    // std::vector<InputHandle_t<In>>          m_inputs;         //   and make the handles properties instead...
+    Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
+    // TODO/FIXME: replace vector of DataObjID property + call-back with a
+    //             vector<handle> property ... as soon as declareProperty can deal with that.
+    template <typename T>
+    using OutputHandle = details::OutputHandle_t<Traits_, details::remove_optional_t<T>>;
+    std::vector<OutputHandle<Out>> m_outputs;
+    Gaudi::Property<std::vector<DataObjID>> m_outputLocations; // TODO/FIXME  for now: use a call-back to update the
+                                                               // actual handles!
+  };
+
+} // namespace details
+
+} // namespace Gaudi::Functional
+
+template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+using MyTransformer =
+    Gaudi::Functional::details::MyTransformer<Signature, Traits_>; // details::isLegacy<Traits_>>;
+
+// class Reader final : public MyTransformer<std::vector<std::shared_ptr<podio::CollectionBase>>()> {
+class Reader final : public MyTransformer<std::vector<std::shared_ptr<podio::CollectionBase>>()> {
+public:
+  Reader(const std::string& name, ISvcLocator* svcLoc) :
+      MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticles"}}) {
+  }
+
+  // Gaudi doesn't run the destructor of the Services so we have to
+  // manually ask for the reader to be deleted so it will call finish()
+  ~Reader() {
+    iosvc->deleteReader();
+  }
+
+  std::shared_ptr<std::vector<std::string>> m_collectionNames;
+
+  ServiceHandle<IIOSvc> iosvc{this, "IOSvc", "IOSvc"};
+
+  StatusCode initialize() override {
+    if (!iosvc.isValid()) {
+      error() << "Unable to locate IIOSvc interface" << endmsg;
+      return StatusCode::FAILURE;
+    }
+
+    m_collectionNames = iosvc->getCollectionNames();
+    return StatusCode::SUCCESS;
+  }
+
+  StatusCode finalize() override {
+    if (iosvc) {
+      iosvc->deleteReader();
+    }
+    return StatusCode::SUCCESS;
+  }
+
+  std::vector<std::shared_ptr<podio::CollectionBase>> operator()() const override {
+    info() << "Reader::operator()" << endmsg;
+    auto val = iosvc->next();
+    info() << "Number of collections " << val.size() << endmsg;
+    return val;
+  }
+};
+
+DECLARE_COMPONENT(Reader)
diff --git a/k4FWCore/components/SaveFile.cpp b/k4FWCore/components/SaveFile.cpp
new file mode 100644
index 00000000..8bb3257c
--- /dev/null
+++ b/k4FWCore/components/SaveFile.cpp
@@ -0,0 +1,128 @@
+/* * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Functional/Consumer.h"
+#include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/IDataManagerSvc.h"
+#include "GaudiKernel/IDataProviderSvc.h"
+#include "GaudiKernel/IHiveWhiteBoard.h"
+#include "GaudiKernel/SmartDataPtr.h"
+#include "GaudiKernel/StatusCode.h"
+
+#include "podio/Frame.h"
+
+#include "edm4hep/MCParticleCollection.h"
+
+#include "IIOSvc.h"
+
+#include <memory>
+
+class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
+public:
+  SaveToFile(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) {
+  }
+
+  mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
+
+  SmartIF<IIOSvc> iosvc;
+  SmartIF<IDataProviderSvc> datasvc;
+  mutable bool m_first = false;
+
+  StatusCode initialize() override {
+    iosvc = service("IOSvc", true);
+    if (!iosvc) {
+      error() << "Unable to locate IIOSvc interface" << endmsg;
+      return StatusCode::FAILURE;
+    }
+
+    datasvc = service("EventDataSvc", true);
+    if (!datasvc) {
+      error() << "Unable to locate IDataSvc interface" << endmsg;
+      return StatusCode::FAILURE;
+    }
+
+    return StatusCode::SUCCESS;
+  }
+
+  StatusCode finalize() override {
+    iosvc->deleteWriter();
+    return StatusCode::SUCCESS;
+  }
+
+  void getOutputCollections(IRegistry* pObj) const {
+    SmartIF<IDataManagerSvc> m_mgr;
+    m_mgr = eventSvc();
+    info() << "pObj = " << pObj << endmsg;
+    if (!pObj) {
+      return;
+    }
+    auto mgr = eventSvc().as<IDataManagerSvc>();
+    if (!mgr) {
+      return;
+    }
+    std::vector<IRegistry*> leaves;
+    StatusCode sc = m_mgr->objectLeaves(pObj, leaves);
+    if (!sc.isSuccess()) {
+      return;
+    }
+    for (auto& pReg : leaves) {
+      info() << "Found leaf: " << pReg->name() << endmsg;
+      /// We are only interested in leaves with an object
+      if (!pReg->address() || !pReg->object())
+        continue;
+      m_OutputNames.value().push_back(pReg->name());
+      const std::string& id = pReg->identifier();
+    }
+  }
+
+  void operator()() const override {
+
+    if (m_first) {
+      SmartDataPtr<DataObject> root(eventSvc(), "/Event");
+      if (!root) {
+        error() << "Failed to retrieve root object /Event" << endmsg;
+        return;
+      }
+      getOutputCollections(root->registry());
+      m_first = false;
+    }
+
+    podio::Frame frame;
+    for (auto& coll : m_OutputNames) {
+      DataObject* p;
+      auto code = datasvc->retrieveObject("/Event/" + coll, p);
+      if (code.isFailure()) {
+        error() << "Failed to retrieve collection " << coll << endmsg;
+        return;
+      }
+      // We take ownership back from the store
+      code = datasvc->unregisterObject(p);
+      if (code.isFailure()) {
+        error() << "Failed to unregister collection " << coll << endmsg;
+        return;
+      }
+      const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+      std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
+      frame.put(std::move(uptr), coll);
+    }
+
+    iosvc->getWriter()->writeFrame(frame, podio::Category::Event);
+  }
+};
+
+DECLARE_COMPONENT(SaveToFile)
diff --git a/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py b/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
new file mode 100644
index 00000000..d7b84d5a
--- /dev/null
+++ b/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
@@ -0,0 +1,73 @@
+# Test that we can get reproducible random numbers in a multithreaded environment
+# given that the seed is the same (obtained, for example, with the UniqueIDGenSvc)
+
+from Configurables import AlgResourcePool, AvalancheSchedulerSvc
+from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard
+from Configurables import ApplicationMgr, Gaudi__Sequencer
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, SaveToFile, StoreSnifferAlg
+from Gaudi.Configuration import INFO, DEBUG, WARNING
+
+evtslots = 2
+threads = 2
+# -------------------------------------------------------------------------------
+
+# The configuration of the whiteboard ------------------------------------------
+# It is useful to call it EventDataSvc to replace the usual data service with
+# the whiteboard transparently.
+
+whiteboard = HiveWhiteBoard("EventDataSvc",
+                            EventSlots=evtslots,
+                            ForceLeaves=True,
+                            )
+
+# Event Loop Manager -----------------------------------------------------------
+# It's called slim since it has less functionalities overall than the good-old
+# event loop manager. Here we just set its outputlevel to DEBUG.
+
+slimeventloopmgr = HiveSlimEventLoopMgr(
+    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+)
+
+# AvalancheScheduler -----------------------------------------------------------
+# We just decide how many algorithms in flight we want to have and how many
+# threads in the pool. The default value is -1, which is for TBB equivalent
+# to take over the whole machine.
+
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+
+# Algo Resource Pool -----------------------------------------------------------
+# Nothing special here, we just set the debug level.
+AlgResourcePool(OutputLevel=DEBUG)
+
+a1 = ExampleFunctionalProducer("First")
+a2 = ExampleFunctionalConsumer("Consumer")
+# a2 = ExampleFunctionalProducer("Second", OutputCollection="MySecondCollection")
+# a3 = ExampleFunctionalProducer("Third", OutputCollection="MyThirdCollection")
+
+
+svc = IOSvc("IOSvc")
+svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/sim1.root']
+svc.CollectionNames = ['MCParticle', 'EventHeader']
+
+io = Reader("Reader",
+           OutputLocations=['MCParticles', 'EventHeader'])
+io.Cardinality = 1
+
+save = SaveToFile("SaveToFile")
+save.CollectionNames = ['MCParticles', 'EventHeader']
+save.Cardinality = 1
+
+node = Gaudi__Sequencer("Node", Members=[io, save], Sequential=True, OutputLevel=INFO)
+
+app = ApplicationMgr(
+    EvtMax=2,
+    EvtSel="NONE",
+    ExtSvc=[whiteboard],
+    EventLoop=slimeventloopmgr,
+    TopAlg=[node],
+    MessageSvcType="InertMessageSvc",
+    OutputLevel=INFO,
+)
+
+
+print(app)

From 462af5f10ab105679a098c904b878ebd81160ec9 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 25 Mar 2024 11:47:08 +0100
Subject: [PATCH 002/127] Working example with new Consumers

---
 k4FWCore/components/IIOSvc.h                  |  2 +-
 k4FWCore/components/IOSvc.cpp                 | 47 +++++++++---
 k4FWCore/components/IOSvc.h                   | 20 ++++--
 k4FWCore/components/Reader.cpp                | 71 +++++++++++--------
 k4FWCore/components/SaveFile.cpp              | 28 ++++++--
 .../components/ExampleFunctionalConsumer.cpp  | 56 +++++++++++++--
 .../ExampleFunctionalConsumerMultiple.cpp     | 10 +--
 .../ExampleFunctionalProducerMultiple.cpp     | 44 +++++++-----
 8 files changed, 205 insertions(+), 73 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 922daf3d..18a043a3 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -41,7 +41,7 @@ class IIOSvc : virtual public IInterface {
    * by the first one
    * @throws IIOSvc::EndOfInput
    */
-  virtual std::vector<std::shared_ptr<podio::CollectionBase>> next( ) = 0;
+  virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> next( ) = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
   virtual std::shared_ptr<podio::ROOTFrameWriter> getWriter() = 0;
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 64270a63..7f23f78e 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -25,6 +25,8 @@
 
 #include "k4FWCore/KeepDropSwitch.h"
 
+#include "GaudiKernel/AnyDataWrapper.h"
+
 #include <mutex>
 #include <thread>
 #include <tuple>
@@ -35,31 +37,58 @@ StatusCode IOSvc::initialize() {
 
   m_switch = KeepDropSwitch(m_outputCommands);
 
-  return Service::initialize();
-}
+  m_incidentSvc = service("IncidentSvc");
+  m_incidentSvc->addListener(this, IncidentType::EndEvent);
 
-StatusCode IOSvc::finalize() {
+  m_dataSvc = service("EventDataSvc");
 
-  return Service::finalize();
+  return Service::initialize();
 }
 
-std::vector<std::shared_ptr<podio::CollectionBase>> IOSvc::next() {
+StatusCode IOSvc::finalize() { return Service::finalize(); }
+
+std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> IOSvc::next() {
   info() << "IOSvc::next()" << endmsg;
-  podio::Frame* frame;
+  podio::Frame frame;
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
-    frame = new podio::Frame(std::move(m_reader->readNextEntry(podio::Category::Event)));
+    frame = podio::Frame(std::move(m_reader->readNextEntry(podio::Category::Event)));
   }
 
   std::vector<std::shared_ptr<podio::CollectionBase>> collections;
 
   for (const auto& name : m_collectionNames) {
     info() << "Collection name: " << name << endmsg;
-    auto ptr = const_cast<podio::CollectionBase*>(frame->get(name));
+    auto ptr = const_cast<podio::CollectionBase*>(frame.get(name));
     collections.push_back(std::shared_ptr<podio::CollectionBase>(ptr));
   }
 
-  return collections;
+  return std::make_tuple(collections, std::move(frame));
+}
+
+// After every event if there is still a frame in the TES
+// that means it hasn't been written so the collections inside the Frame
+// should be removed so that they are deleted when the Frame is deleted
+void IOSvc::handle( const Incident& incident ) {
+
+  DataObject *p;
+  auto code = m_dataSvc->retrieveObject("/Event/Frame", p);
+  if (code.isFailure()) {
+    return;
+  }
+
+  auto frame = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+  for (const auto& coll : frame->getData().getAvailableCollections()) {
+    DataObject *collPtr;
+    code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);
+    if (code.isSuccess()) {
+      info() << "Removing collection: " << coll << endmsg;
+      m_dataSvc->unregisterObject(collPtr);
+    }
+  }
+
+  // code = m_dataSvc->unregisterObject(p);
+  
 }
 
 DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index d4e70c97..71513abd 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -21,11 +21,16 @@
 
 #include "Gaudi/Property.h"
 #include "GaudiKernel/Service.h"
+#include "GaudiKernel/IIncidentListener.h"
+#include "GaudiKernel/IIncidentSvc.h"
+#include "GaudiKernel/IDataProviderSvc.h"
 
 #include "podio/ROOTFrameReader.h"
 #include "podio/ROOTFrameWriter.h"
-#include "podio/ROOTNTupleReader.h"
-#include "podio/ROOTNTupleWriter.h"
+#include "podio/ROOTRNTupleReader.h"
+#include "podio/ROOTRNTupleWriter.h"
+#include "podio/IROOTFrameReader.h"
+#include "podio/IROOTFrameWriter.h"
 
 #include "k4FWCore/KeepDropSwitch.h"
 
@@ -35,7 +40,7 @@
 #include <type_traits>
 #include <vector>
 
-class IOSvc : public extends<Service, IIOSvc> {
+class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   using extends::extends;
 public:
   // Gaudi doesn't run the destructor of the Services so we have to
@@ -45,7 +50,7 @@ class IOSvc : public extends<Service, IIOSvc> {
   StatusCode initialize() override;
   StatusCode finalize() override;
 
-  std::vector<std::shared_ptr<podio::CollectionBase>> next() override;
+  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> next() override;
 
   std::shared_ptr<std::vector<std::string>> getCollectionNames() const override {
     return std::make_shared<std::vector<std::string>>(m_collectionNames);
@@ -68,8 +73,7 @@ class IOSvc : public extends<Service, IIOSvc> {
 
   KeepDropSwitch m_switch;
 
-  // To be changed by a base class to allow to use different readers
-  std::unique_ptr<podio::ROOTFrameReader> m_reader{nullptr};
+  std::unique_ptr<podio::IROOTFrameReader> m_reader{nullptr};
   std::shared_ptr<podio::ROOTFrameWriter> m_writer{nullptr};
 
   std::shared_ptr<podio::ROOTFrameWriter> getWriter() override {
@@ -89,6 +93,10 @@ class IOSvc : public extends<Service, IIOSvc> {
       m_reader = nullptr;
     }
   }
+
+  SmartIF<IDataProviderSvc> m_dataSvc;
+  SmartIF<IIncidentSvc> m_incidentSvc;
+  void handle(const Incident& incident) override;
 };
 
 #endif
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 0f8cdb8b..21fe231a 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -23,8 +23,15 @@
 
 #include "IIOSvc.h"
 
+#include "podio/CollectionBase.h"
+#include "podio/Frame.h"
+
+#include <GaudiKernel/AnyDataWrapper.h>
+#include <GaudiKernel/IDataManagerSvc.h>
 #include <memory>
 
+using podio::CollectionBase;
+
 namespace Gaudi::Functional {
 template <typename Container>
 using vector_of_ = std::vector<Container>;
@@ -33,11 +40,9 @@ using vector_of_optional_ = std::vector<std::optional<Container>>;
 
 namespace details {
 
-  template <typename Signature, typename Traits_>
-  class MyTransformer;
-
-  template <typename Out, typename Traits_>
-  class MyTransformer<vector_of_<Out>(), Traits_> : public BaseClass_t<Traits_> {
+  class MyTransformer : public BaseClass_t<Gaudi::Functional::Traits::useDefaults> {
+    using Traits_ = Gaudi::Functional::Traits::useDefaults;
+    using Out = std::shared_ptr<podio::CollectionBase>;
     using base_class = BaseClass_t<Traits_>;
     static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
 
@@ -50,18 +55,18 @@ namespace details {
             this, outputs.first, details::to_DataObjID(outputs.second),
             [this](Gaudi::Details::PropertyBase&) {
               this->m_outputs = details::make_vector_of_handles<decltype(this->m_outputs)>(this, m_outputLocations);
-              if constexpr (details::is_optional_v<Out>) { // handle constructor does not (yet) allow to
-                                                           // set optional flag... so
-                                                           // do it explicitly here...
-                std::for_each(this->m_outputs.begin(), this->m_outputs.end(), [](auto& h) { h.setOptional(true); });
-              }
+              // if constexpr (details::is_optional_v<Out>) { // handle constructor does not (yet) allow to
+              //                                              // set optional flag... so
+              //                                              // do it explicitly here...
+              //   std::for_each(this->m_outputs.begin(), this->m_outputs.end(), [](auto& h) { h.setOptional(true); });
+              // }
             },
-            Gaudi::Details::Property::ImmediatelyInvokeHandler{true}) {
-    }
+            Gaudi::Details::Property::ImmediatelyInvokeHandler{true})
+    {}
 
     // derived classes can NOT implement execute
     StatusCode execute(const EventContext&) const override final {
-      Gaudi::Algorithm::info() << "MyTransformer::execute()" << endmsg;
+      // Gaudi::Algorithm::info() << "MyTransformer::execute()" << endmsg;
       try {
         // TODO:FIXME: how does operator() know the number and order of expected outputs?
         auto out = (*this)();
@@ -71,9 +76,10 @@ namespace details {
                                this->name(), StatusCode::FAILURE);
         }
         for (unsigned i = 0; i != out.size(); ++i) {
-          Gaudi::Algorithm::info() << "MyTransformer::execute() : putting " << m_outputs[i].fullKey() << endmsg;
+          // Gaudi::Algorithm::info() << "MyTransformer::execute() : putting " << m_outputs[i].fullKey() << endmsg;
           details::put(m_outputs[i], std::move(out[i]));
         }
+        // details::put(m_frame[0], std::move(frame));
         return FilterDecision::PASSED;
       } catch (GaudiException& e) {
         (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
@@ -88,32 +94,33 @@ namespace details {
 
   private:
     // if In is a pointer, it signals optional (as opposed to mandatory) input
-    template <typename T>
-    using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
-    // std::vector<InputHandle_t<In>>          m_inputs;         //   and make the handles properties instead...
-    Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
+    // template <typename T>
+    // using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+    // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
     // TODO/FIXME: replace vector of DataObjID property + call-back with a
     //             vector<handle> property ... as soon as declareProperty can deal with that.
     template <typename T>
     using OutputHandle = details::OutputHandle_t<Traits_, details::remove_optional_t<T>>;
     std::vector<OutputHandle<Out>> m_outputs;
-    Gaudi::Property<std::vector<DataObjID>> m_outputLocations; // TODO/FIXME  for now: use a call-back to update the
-                                                               // actual handles!
+    Gaudi::Property<std::vector<DataObjID>> m_outputLocations;
+    // Gaudi::Property<std::vector<DataObjID>> m_outputFrameLocations;
+    // std::vector<OutputHandle<std::shared_ptr<CollectionBase>>> m_frame{};
+
   };
 
 } // namespace details
 
 } // namespace Gaudi::Functional
 
-template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-using MyTransformer =
-    Gaudi::Functional::details::MyTransformer<Signature, Traits_>; // details::isLegacy<Traits_>>;
+using Gaudi::Functional::details::MyTransformer;
 
-// class Reader final : public MyTransformer<std::vector<std::shared_ptr<podio::CollectionBase>>()> {
-class Reader final : public MyTransformer<std::vector<std::shared_ptr<podio::CollectionBase>>()> {
+class Reader final : public MyTransformer {
+// class Reader final : public MyTransformer<std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::shared_ptr<CollectionBase>>()> {
 public:
   Reader(const std::string& name, ISvcLocator* svcLoc) :
-      MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticles"}}) {
+    MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticles"}}
+                  
+                  ) {
   }
 
   // Gaudi doesn't run the destructor of the Services so we have to
@@ -146,8 +153,16 @@ class Reader final : public MyTransformer<std::vector<std::shared_ptr<podio::Col
   std::vector<std::shared_ptr<podio::CollectionBase>> operator()() const override {
     info() << "Reader::operator()" << endmsg;
     auto val = iosvc->next();
-    info() << "Number of collections " << val.size() << endmsg;
-    return val;
+
+    auto eds = eventSvc().as<IDataProviderSvc>();
+    auto frame = std::move(std::get<1>(val));
+
+    // We'll hand the ownership of 
+    auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
+    auto code = eds->registerObject("/Event/Frame", tmp);
+
+
+    return std::get<0>(val);
   }
 };
 
diff --git a/k4FWCore/components/SaveFile.cpp b/k4FWCore/components/SaveFile.cpp
index 8bb3257c..2d80ce71 100644
--- a/k4FWCore/components/SaveFile.cpp
+++ b/k4FWCore/components/SaveFile.cpp
@@ -31,6 +31,7 @@
 #include "IIOSvc.h"
 
 #include <memory>
+#include <thread>
 
 class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
 public:
@@ -38,6 +39,7 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
   }
 
   mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
+  mutable std::set<std::string> m_availableCollections;
 
   SmartIF<IIOSvc> iosvc;
   SmartIF<IDataProviderSvc> datasvc;
@@ -85,7 +87,7 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
       /// We are only interested in leaves with an object
       if (!pReg->address() || !pReg->object())
         continue;
-      m_OutputNames.value().push_back(pReg->name());
+      m_availableCollections.insert(pReg->name());
       const std::string& id = pReg->identifier();
     }
   }
@@ -99,10 +101,17 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
         return;
       }
       getOutputCollections(root->registry());
+
+      // iosvc->getCollsToWrite(m_availableCollections);
+
       m_first = false;
     }
 
-    podio::Frame frame;
+    DataObject *p;
+    // Get back ownership of the original frame that doesn't have to be deleted manually
+    auto code = datasvc->retrieveObject("/Event/Frame", p);
+    code = datasvc->unregisterObject(p);
+    auto ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
     for (auto& coll : m_OutputNames) {
       DataObject* p;
       auto code = datasvc->retrieveObject("/Event/" + coll, p);
@@ -117,11 +126,22 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
         return;
       }
       const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+      if (!collection) {
+        error() << "Failed to cast collection " << coll << endmsg;
+        return;
+      }
+      else {
+        info() << "collection = " << collection << endmsg;
+        // info() << "Collection " << coll << " has " << collection->getData()->size() << " elements" << endmsg;
+      }
       std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
-      frame.put(std::move(uptr), coll);
+      ptr->getData().put(std::move(uptr), "MCParticless");
     }
 
-    iosvc->getWriter()->writeFrame(frame, podio::Category::Event);
+    ptr->addRef();
+    iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event);
+
+
   }
 };
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index 1f5e9458..b2bcf0f7 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -27,20 +27,30 @@
 
 #include <stdexcept>
 #include <string>
+#include <thread>
 
-struct ExampleFunctionalConsumer final
-    : Gaudi::Functional::Consumer<void(const edm4hep::MCParticleCollection& input), BaseClass_t> {
+#include "k4FWCore/Consumer.h"
+
+
+struct ExampleFunctionalConsumer final : Gaudi::Functional::CConsumer<void(const edm4hep::MCParticleCollection& input)> {
+// struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const edm4hep::MCParticleCollection& input), BaseClass_t> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the input collection
   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+      : CConsumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as the collections
   // we get from the input
   void operator()(const edm4hep::MCParticleCollection& input) const override {
+  // void operator()(const edm4hep::MCParticleCollection& input) const override {
+  // void operator()(const edm4hep::MCParticleCollection& input) const override {
+    info() << "ExampleFunctionalConsumer::operator()" << endmsg;
+    info() << "Size of collection " << input.size() << endmsg;
+    // std::this_thread::sleep_for(std::chrono::milliseconds(2000));
     int i = 0;
     for (const auto& particle : input) {
+      info() << particle.getPDG() << endmsg;
       if ((particle.getPDG() != 1 + i + m_possibleOffset) ||
           (particle.getGeneratorStatus() != 2 + i + m_possibleOffset) ||
           (particle.getSimulatorStatus() != 3 + i + m_possibleOffset) ||
@@ -52,7 +62,7 @@ struct ExampleFunctionalConsumer final
               << ", " << 5 + i + m_possibleOffset << ", " << 6 + i + m_possibleOffset << " got " << particle.getPDG()
               << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
               << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
-        throw std::runtime_error(error.str());
+        // throw std::runtime_error(error.str());
       }
       i++;
     }
@@ -61,4 +71,42 @@ struct ExampleFunctionalConsumer final
   Gaudi::Property<int> m_possibleOffset{this, "PossibleOffset", 0, "Possible offset in the values data"};
 };
 
+
+// struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const std::shared_ptr<podio::CollectionBase>& input)> {
+// // struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const edm4hep::MCParticleCollection& input), BaseClass_t> {
+//   // The pair in KeyValue can be changed from python and it corresponds
+//   // to the name of the input collection
+//   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
+//       : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+
+//   // This is the function that will be called to transform the data
+//   // Note that the function has to be const, as well as the collections
+//   // we get from the input
+//   void operator()(const std::shared_ptr<podio::CollectionBase>& input) const override {
+//   // void operator()(const edm4hep::MCParticleCollection& input) const override {
+//     info() << "ExampleFunctionalConsumer::operator()" << endmsg;
+//     info() << "Size of collection " << input->size() << endmsg;
+//     // std::this_thread::sleep_for(std::chrono::milliseconds(2000));
+//     int i = 0;
+//     for (const auto& particle : static_cast<const edm4hep::MCParticleCollection&>(*input)) {
+//       if ((particle.getPDG() != 1 + i + m_possibleOffset) ||
+//           (particle.getGeneratorStatus() != 2 + i + m_possibleOffset) ||
+//           (particle.getSimulatorStatus() != 3 + i + m_possibleOffset) ||
+//           (particle.getCharge() != 4 + i + m_possibleOffset) || (particle.getTime() != 5 + i + m_possibleOffset) ||
+//           (particle.getMass() != 6 + i + m_possibleOffset)) {
+//         std::stringstream error;
+//         error << "Wrong data in MCParticle collection, expected " << 1 + i + m_possibleOffset << ", "
+//               << 2 + i + m_possibleOffset << ", " << 3 + i + m_possibleOffset << ", " << 4 + i + m_possibleOffset
+//               << ", " << 5 + i + m_possibleOffset << ", " << 6 + i + m_possibleOffset << " got " << particle.getPDG()
+//               << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
+//               << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
+//         // throw std::runtime_error(error.str());
+//       }
+//       i++;
+//     }
+//   }
+
+//   Gaudi::Property<int> m_possibleOffset{this, "PossibleOffset", 0, "Possible offset in the values data"};
+// };
+
 DECLARE_COMPONENT(ExampleFunctionalConsumer)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
index 440d8f85..02109493 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
@@ -35,6 +35,7 @@ namespace edm4hep {
 
 // Define BaseClass_t
 #include "k4FWCore/BaseClass.h"
+#include "k4FWCore/Consumer.h"
 
 #include <sstream>
 #include <stdexcept>
@@ -48,13 +49,12 @@ using TrackerHitColl    = edm4hep::TrackerHit3DCollection;
 using TrackColl         = edm4hep::TrackCollection;
 
 struct ExampleFunctionalConsumerMultiple final
-    : Gaudi::Functional::Consumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
-                                       const TrackerHitColl&, const TrackColl&),
-                                  BaseClass_t> {
+    : Gaudi::Functional::CConsumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
+                                       const TrackerHitColl&, const TrackColl&)> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the input collection
   ExampleFunctionalConsumerMultiple(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc,
+      : CConsumer(name, svcLoc,
                  {
                      KeyValue("InputCollectionFloat", "VectorFloat"),
                      KeyValue("InputCollectionParticles", "MCParticles1"),
@@ -68,6 +68,7 @@ struct ExampleFunctionalConsumerMultiple final
   // we get from the input
   void operator()(const FloatColl& floatVector, const ParticleColl& particles, const SimTrackerHitColl& simTrackerHits,
                   const TrackerHitColl& trackerHits, const TrackColl& tracks) const override {
+    info() << "ExampleFunctionalConsumerMultiple called" << endmsg;
     if (floatVector.size() != 3) {
       throw std::runtime_error("Wrong size of floatVector collection, expected 3, got " +
                                std::to_string(floatVector.size()) + "");
@@ -79,6 +80,7 @@ struct ExampleFunctionalConsumerMultiple final
       throw std::runtime_error(error.str());
     }
 
+    info() << "VectorFloat collection is ok" << endmsg;
     auto p4 = particles.momentum()[0];
     if ((p4.x != m_magicNumberOffset + 5) || (p4.y != m_magicNumberOffset + 6) || (p4.z != m_magicNumberOffset + 7) ||
         (particles[0].getMass() != m_magicNumberOffset + 8)) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index 70659cce..ae14422f 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -42,10 +42,11 @@ using Particle      = edm4hep::MCParticleCollection;
 using SimTrackerHit = edm4hep::SimTrackerHitCollection;
 using TrackerHit    = edm4hep::TrackerHit3DCollection;
 using Track         = edm4hep::TrackCollection;
+using Ptr = std::shared_ptr<podio::CollectionBase>;
 
 struct ExampleFunctionalProducerMultiple final
-    : Gaudi::Functional::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>(),
-                                  BaseClass_t> {
+    // : Gaudi::Functional::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>(),
+  : Gaudi::Functional::Producer<std::tuple<Ptr, Ptr, Ptr, Ptr, Ptr, Ptr>()> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the output collections
   ExampleFunctionalProducerMultiple(const std::string& name, ISvcLocator* svcLoc)
@@ -57,31 +58,34 @@ struct ExampleFunctionalProducerMultiple final
              KeyValue("OutputCollectionTrackerHits", "TrackerHits"), KeyValue("OutputCollectionTracks", "Tracks")}) {}
 
   // This is the function that will be called to produce the data
-  std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track> operator()() const override {
+  std::tuple<Ptr, Ptr, Ptr, Ptr, Ptr, Ptr> operator()() const override {
     // The following was copied and adapted from the
     // k4FWCoreTest_CreateExampleEventData test
 
-    auto floatVector = podio::UserDataCollection<float>();
-    floatVector.push_back(125.);
-    floatVector.push_back(25.);
-    floatVector.push_back(m_event);
+    auto floatVector = new podio::UserDataCollection<float>();
+    floatVector->push_back(125.);
+    floatVector->push_back(25.);
+    floatVector->push_back(m_event);
 
     auto particles = edm4hep::MCParticleCollection();
     auto particle  = particles.create();
-    particle.setMomentum({m_magicNumberOffset + m_event + 5.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0});
+    auto& p4        = particle.getMomentum();
+    p4.x            = m_magicNumberOffset + m_event + 5;
+    p4.y            = m_magicNumberOffset + 6;
+    p4.z            = m_magicNumberOffset + 7;
     particle.setMass(m_magicNumberOffset + m_event + 8);
 
-    auto simTrackerHits = edm4hep::SimTrackerHitCollection();
-    auto hit            = simTrackerHits.create();
+    auto simTrackerHits = new edm4hep::SimTrackerHitCollection();
+    auto hit            = simTrackerHits->create();
     hit.setPosition({3, 4, 5});
 
-    auto trackerHits = edm4hep::TrackerHit3DCollection();
-    auto trackerHit  = trackerHits.create();
+    auto trackerHits = new edm4hep::TrackerHit3DCollection();
+    auto trackerHit  = trackerHits->create();
     trackerHit.setPosition({3, 4, 5});
 
-    auto tracks = edm4hep::TrackCollection();
-    auto track  = tracks.create();
-    auto track2 = tracks.create();
+    auto tracks = new edm4hep::TrackCollection();
+    auto track  = tracks->create();
+    auto track2 = tracks->create();
     // set members
     track.setType(1);
     track.setChi2(2.1);
@@ -96,8 +100,14 @@ struct ExampleFunctionalProducerMultiple final
     track.addToTrackerHits(trackerHit);
     track.addToTracks(track2);
 
-    return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits),
-                           std::move(trackerHits), std::move(tracks));
+    // return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits),
+    //                        std::move(trackerHits), std::move(tracks));
+    return std::make_tuple(std::shared_ptr<podio::CollectionBase>(floatVector),
+                            std::shared_ptr<podio::CollectionBase>(particles),
+                            std::shared_ptr<podio::CollectionBase>(new Particle()),
+                            std::shared_ptr<podio::CollectionBase>(simTrackerHits),
+                            std::shared_ptr<podio::CollectionBase>(trackerHits),
+                            std::shared_ptr<podio::CollectionBase>(tracks));
   }
 
 private:

From 64026efd0571091e4354b64f2f6eb81e11409c7c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 25 Mar 2024 11:48:20 +0100
Subject: [PATCH 003/127] Make changes, everything working except reading from
 multiple files

---
 k4FWCore/components/IOSvc.cpp                 |  27 ++-
 k4FWCore/components/IOSvc.h                   |  13 +-
 k4FWCore/components/Reader.cpp                |  20 +-
 k4FWCore/components/SaveFile.cpp              |  43 ++--
 k4FWCore/include/k4FWCore/BaseClass.h         |   2 +-
 k4FWCore/include/k4FWCore/Consumer.h          |  60 +++++
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |  66 ++++++
 k4FWCore/include/k4FWCore/Producer.h          |  51 +++++
 k4FWCore/include/k4FWCore/Transformer.h       | 213 ++++++++++++++++++
 test/k4FWCoreTest/CMakeLists.txt              |  13 +-
 .../ExampleFunctionalConsumerMemory.py        |  41 ++++
 ...ExampleFunctionalConsumerMultipleMemory.py |  53 +++++
 ...oducer.py => ExampleFunctionalProducer.py} |   0
 ...y => ExampleFunctionalProducerMultiple.py} |   0
 .../options/ExampleFunctionalsFile.py         |  57 +++++
 .../options/ExampleFunctionalsFileMultiple.py |  58 +++++
 ...sformer.py => ExampleFunctionalsMemory.py} |   0
 .../ExampleFunctionalsMultipleMemory.py       |  56 +++++
 .../options/runEventHeaderCheck.py            |  15 +-
 .../components/ExampleEventHeaderConsumer.cpp |   4 +-
 .../components/ExampleFunctionalConsumer.cpp  |  80 ++-----
 .../ExampleFunctionalConsumerMultiple.cpp     |  38 ++--
 .../components/ExampleFunctionalProducer.cpp  |   5 +-
 .../ExampleFunctionalProducerMultiple.cpp     |  49 ++--
 .../ExampleFunctionalTransformer.cpp          |  22 +-
 .../ExampleFunctionalTransformerMultiple.cpp  |  27 +--
 26 files changed, 817 insertions(+), 196 deletions(-)
 create mode 100644 k4FWCore/include/k4FWCore/Consumer.h
 create mode 100644 k4FWCore/include/k4FWCore/FunctionalUtils.h
 create mode 100644 k4FWCore/include/k4FWCore/Producer.h
 create mode 100644 k4FWCore/include/k4FWCore/Transformer.h
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
 rename test/k4FWCoreTest/options/{runExampleFunctionalProducer.py => ExampleFunctionalProducer.py} (100%)
 rename test/k4FWCoreTest/options/{runExampleFunctionalProducerMultiple.py => ExampleFunctionalProducerMultiple.py} (100%)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalsFile.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
 rename test/k4FWCoreTest/options/{runExampleFunctionalTransformer.py => ExampleFunctionalsMemory.py} (100%)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 7f23f78e..ed0911ed 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -32,8 +32,10 @@
 #include <tuple>
 
 StatusCode IOSvc::initialize() {
-  m_reader = std::make_unique<podio::ROOTFrameReader>();
-  m_reader->openFiles(m_fileNames);
+  if (!m_readingFileNames.empty()) {
+    m_reader = std::make_unique<podio::ROOTFrameReader>();
+    m_reader->openFiles(m_readingFileNames);
+  }
 
   m_switch = KeepDropSwitch(m_outputCommands);
 
@@ -53,6 +55,9 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> IO
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
     frame = podio::Frame(std::move(m_reader->readNextEntry(podio::Category::Event)));
+    if (m_collectionNames.empty()) {
+        m_collectionNames = frame.getAvailableCollections();
+    }
   }
 
   std::vector<std::shared_ptr<podio::CollectionBase>> collections;
@@ -70,9 +75,8 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> IO
 // that means it hasn't been written so the collections inside the Frame
 // should be removed so that they are deleted when the Frame is deleted
 void IOSvc::handle( const Incident& incident ) {
-
   DataObject *p;
-  auto code = m_dataSvc->retrieveObject("/Event/Frame", p);
+  auto code = m_dataSvc->retrieveObject("/Event/_Frame", p);
   if (code.isFailure()) {
     return;
   }
@@ -82,13 +86,20 @@ void IOSvc::handle( const Incident& incident ) {
     DataObject *collPtr;
     code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);
     if (code.isSuccess()) {
-      info() << "Removing collection: " << coll << endmsg;
-      m_dataSvc->unregisterObject(collPtr);
+      code = m_dataSvc->unregisterObject(collPtr);
     }
+    // else {
+    //   info() << "Collection not found: " << coll << endmsg;
+    // }
   }
+}
+
+void IOSvc::setReadingCollectionNames(const std::vector<std::string>& names) {
+  m_collectionNames = names;
+}
 
-  // code = m_dataSvc->unregisterObject(p);
-  
+void IOSvc::setReadingFileNames(const std::vector<std::string>& names) {
+  m_readingFileNames = names;
 }
 
 DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 71513abd..e61feba0 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -56,15 +56,20 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
     return std::make_shared<std::vector<std::string>>(m_collectionNames);
   }
 
+  void setReadingCollectionNames(const std::vector<std::string>& names);
+  void setReadingFileNames(const std::vector<std::string>& names);
+
 protected:
-  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
-  Gaudi::Property<std::vector<std::string>> m_fileNames{this, "FileNames", {}, "List of files to read"};
+
   Gaudi::Property<unsigned int> m_bufferNbEvents{
       this, "BufferNbEvents", 20000,
       "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
   Gaudi::Property<std::vector<std::string>> m_input{this, "Input", {}, "List of inputs"};
   Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
-  Gaudi::Property<std::string> m_output{this, "Output", "output.root", "Output file name"};
+
+  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
+  Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
+  Gaudi::Property<std::string> m_writingFileName{this, "output", {}, "List of files to read"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
 
@@ -78,7 +83,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   std::shared_ptr<podio::ROOTFrameWriter> getWriter() override {
     if (!m_writer) {
-      m_writer = std::shared_ptr<podio::ROOTFrameWriter>(new podio::ROOTFrameWriter(m_output));
+      m_writer = std::shared_ptr<podio::ROOTFrameWriter>(new podio::ROOTFrameWriter(m_writingFileName.value()));
     }
     return m_writer;
   }
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 21fe231a..f511b4f4 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -70,14 +70,14 @@ namespace details {
       try {
         // TODO:FIXME: how does operator() know the number and order of expected outputs?
         auto out = (*this)();
-        if (out.size() != m_outputs.size()) {
-          throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
-                                   " containers, got " + std::to_string(out.size()) + " instead",
-                               this->name(), StatusCode::FAILURE);
-        }
+        // if (out.size() != m_outputs.size()) {
+        //   throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
+        //                            " containers, got " + std::to_string(out.size()) + " instead",
+        //                        this->name(), StatusCode::FAILURE);
+        // }
         for (unsigned i = 0; i != out.size(); ++i) {
           // Gaudi::Algorithm::info() << "MyTransformer::execute() : putting " << m_outputs[i].fullKey() << endmsg;
-          details::put(m_outputs[i], std::move(out[i]));
+          details::put(m_outputs[0], std::move(out[i]));
         }
         // details::put(m_frame[0], std::move(frame));
         return FilterDecision::PASSED;
@@ -118,7 +118,7 @@ class Reader final : public MyTransformer {
 // class Reader final : public MyTransformer<std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::shared_ptr<CollectionBase>>()> {
 public:
   Reader(const std::string& name, ISvcLocator* svcLoc) :
-    MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticles"}}
+    MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticless"}}
                   
                   ) {
   }
@@ -129,8 +129,6 @@ class Reader final : public MyTransformer {
     iosvc->deleteReader();
   }
 
-  std::shared_ptr<std::vector<std::string>> m_collectionNames;
-
   ServiceHandle<IIOSvc> iosvc{this, "IOSvc", "IOSvc"};
 
   StatusCode initialize() override {
@@ -139,7 +137,6 @@ class Reader final : public MyTransformer {
       return StatusCode::FAILURE;
     }
 
-    m_collectionNames = iosvc->getCollectionNames();
     return StatusCode::SUCCESS;
   }
 
@@ -159,11 +156,12 @@ class Reader final : public MyTransformer {
 
     // We'll hand the ownership of 
     auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
-    auto code = eds->registerObject("/Event/Frame", tmp);
+    auto code = eds->registerObject("/Event/_Frame", tmp);
 
 
     return std::get<0>(val);
   }
+
 };
 
 DECLARE_COMPONENT(Reader)
diff --git a/k4FWCore/components/SaveFile.cpp b/k4FWCore/components/SaveFile.cpp
index 2d80ce71..135b1ece 100644
--- a/k4FWCore/components/SaveFile.cpp
+++ b/k4FWCore/components/SaveFile.cpp
@@ -43,7 +43,7 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
 
   SmartIF<IIOSvc> iosvc;
   SmartIF<IDataProviderSvc> datasvc;
-  mutable bool m_first = false;
+  mutable bool m_first {true};
 
   StatusCode initialize() override {
     iosvc = service("IOSvc", true);
@@ -71,48 +71,58 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
     m_mgr = eventSvc();
     info() << "pObj = " << pObj << endmsg;
     if (!pObj) {
+      error() << "Failed to retrieve root object" << endmsg;
       return;
     }
     auto mgr = eventSvc().as<IDataManagerSvc>();
     if (!mgr) {
+      error() << "Failed to retrieve IDataManagerSvc" << endmsg;
       return;
     }
     std::vector<IRegistry*> leaves;
     StatusCode sc = m_mgr->objectLeaves(pObj, leaves);
     if (!sc.isSuccess()) {
+      error() << "Failed to retrieve object leaves" << endmsg;
       return;
     }
     for (auto& pReg : leaves) {
-      info() << "Found leaf: " << pReg->name() << endmsg;
-      /// We are only interested in leaves with an object
-      if (!pReg->address() || !pReg->object())
+      if (pReg->name() == "/_Frame") {
         continue;
-      m_availableCollections.insert(pReg->name());
+      }
+      // info() << "Found leaf: " << pReg->name() << endmsg;
+      /// We are only interested in leaves with an object
+      // if (!pReg->address() || !pReg->object()) {
+      //   info() << "Leaf " << pReg->name() << " has no object" << endmsg;
+      //   continue;
+      // }
+      m_availableCollections.insert(pReg->name().substr(1, pReg->name().size() - 1));
       const std::string& id = pReg->identifier();
     }
   }
 
   void operator()() const override {
-
     if (m_first) {
       SmartDataPtr<DataObject> root(eventSvc(), "/Event");
       if (!root) {
-        error() << "Failed to retrieve root object /Event" << endmsg;
+        info() << "Failed to retrieve root object /Event" << endmsg;
         return;
       }
       getOutputCollections(root->registry());
 
-      // iosvc->getCollsToWrite(m_availableCollections);
-
       m_first = false;
     }
 
     DataObject *p;
-    // Get back ownership of the original frame that doesn't have to be deleted manually
     auto code = datasvc->retrieveObject("/Event/Frame", p);
-    code = datasvc->unregisterObject(p);
-    auto ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
-    for (auto& coll : m_OutputNames) {
+    AnyDataWrapper<podio::Frame>* ptr;
+    if (code.isSuccess()) {
+      code = datasvc->unregisterObject(p);
+      ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+    }
+    else {
+      ptr = new AnyDataWrapper<podio::Frame>(podio::Frame());
+    }
+    for (auto& coll : m_availableCollections) {
       DataObject* p;
       auto code = datasvc->retrieveObject("/Event/" + coll, p);
       if (code.isFailure()) {
@@ -131,17 +141,16 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
         return;
       }
       else {
-        info() << "collection = " << collection << endmsg;
+        // info() << "collection = " << collection << endmsg;
         // info() << "Collection " << coll << " has " << collection->getData()->size() << " elements" << endmsg;
       }
       std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
-      ptr->getData().put(std::move(uptr), "MCParticless");
+      ptr->getData().put(std::move(uptr), coll);
     }
 
-    ptr->addRef();
+    // ptr->addRef();
     iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event);
 
-
   }
 };
 
diff --git a/k4FWCore/include/k4FWCore/BaseClass.h b/k4FWCore/include/k4FWCore/BaseClass.h
index 4d58f8d3..02732d60 100644
--- a/k4FWCore/include/k4FWCore/BaseClass.h
+++ b/k4FWCore/include/k4FWCore/BaseClass.h
@@ -26,7 +26,7 @@
 
 // Base class used for the Traits template argument of the
 // Gaudi::Functional algorithms
-struct BaseClass_t {
+struct [[deprecated("Functional algorithms using the BaseClass.h header are deprecated and will be removed in the future")]] BaseClass_t {
   template <typename T> using InputHandle  = DataObjectReadHandle<DataWrapper<T>>;
   template <typename T> using OutputHandle = DataObjectWriteHandle<DataWrapper<T>>;
 
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
new file mode 100644
index 00000000..ff78ef0c
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -0,0 +1,60 @@
+/***********************************************************************************\
+* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
+*                                                                                   *
+* This software is distributed under the terms of the Apache version 2 licence,     *
+* copied verbatim in the file "LICENSE".                                            *
+*                                                                                   *
+* In applying this licence, CERN does not waive the privileges and immunities       *
+* granted to it by virtue of its status as an Intergovernmental Organization        *
+* or submit itself to any jurisdiction.                                             *
+\***********************************************************************************/
+#pragma once
+
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+#include <GaudiKernel/FunctionalFilterDecision.h>
+
+#include "podio/CollectionBase.h"
+
+#include "k4FWCore/FunctionalUtils.h"
+
+#include <type_traits>
+#include <utility>
+
+namespace k4FWCore {
+
+namespace details {
+
+  template <typename Signature, typename Traits_>
+  struct Consumer;
+
+  template <typename... In, typename Traits_>
+  struct Consumer<void(const In&...), Traits_>
+      : Gaudi::Functional::details::DataHandleMixin<
+            std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
+            Traits_> {
+    using Gaudi::Functional::details::DataHandleMixin<
+        std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
+        Traits_>::DataHandleMixin;
+
+    // derived classes are NOT allowed to implement execute ...
+    StatusCode execute(const EventContext& ctx) const override final {
+      try {
+        filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+        return Gaudi::Functional::FilterDecision::PASSED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // ... instead, they must implement the following operator
+    virtual void operator()(const In&...) const = 0;
+  };
+
+} // namespace details
+
+template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+using Consumer = details::Consumer<Signature, Traits_>;
+
+} // namespace k4FWCore
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
new file mode 100644
index 00000000..1e61b1c2
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -0,0 +1,66 @@
+#pragma once
+
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+
+#include "podio/CollectionBase.h"
+
+#include <tuple>
+#include <type_traits>
+
+namespace k4FWCore {
+
+namespace details {
+
+  template <typename T, typename P, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, P>, int> = 0>
+  const auto& transformIfDerivedFromBase(const P& arg) {
+    return arg;
+  }
+  template <typename T, typename P, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, P>, int> = 0>
+  const auto& transformIfDerivedFromBase(const P& arg) {
+    return static_cast<const T&>(*arg);
+  }
+
+
+  template <typename T>
+  std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(const T& arg) {
+      // Transformation logic for types derived from podio::CollectionBase
+      // For example:
+      // return /* Transformation logic for derived types */;
+      // Replace /* Transformation logic for derived types */ with your specific transformation logic
+    return std::shared_ptr<podio::CollectionBase>(arg);
+  }
+  // Transformation function for types not derived from podio::CollectionBase
+  template <typename T>
+  std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
+      // Default: no transformation
+      // return arg;
+      return std::shared_ptr<podio::CollectionBase>(arg);
+  }
+
+
+  template <typename... In>
+  struct filter_evtcontext_tt {
+
+    static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
+                  "EventContext can only appear as first argument");
+
+    template <typename Algorithm, typename Handles>
+    static auto apply(const Algorithm& algo, Handles& handles) {
+      return std::apply(
+          [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
+    }
+
+    template <typename Algorithm, typename Handles>
+    static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+      // return std::apply( [&]( const auto&... handle ) {
+      // printType(*get( handle, algo, ctx )...);
+      //   return algo( static_cast<const
+      // edm4hep::MCParticleCollection&>(*get( handle, algo, ctx ))... ); }, handles );
+      return std::apply(
+          [&](const auto&... handle) { return algo(transformIfDerivedFromBase<In>(get(handle, algo, ctx))...); },
+          handles);
+    }
+  };
+} // namespace details
+} // namespace k4FWCore
diff --git a/k4FWCore/include/k4FWCore/Producer.h b/k4FWCore/include/k4FWCore/Producer.h
new file mode 100644
index 00000000..efa50c95
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/Producer.h
@@ -0,0 +1,51 @@
+/***********************************************************************************\
+* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
+*                                                                                   *
+* This software is distributed under the terms of the Apache version 2 licence,     *
+* copied verbatim in the file "LICENSE".                                            *
+*                                                                                   *
+* In applying this licence, CERN does not waive the privileges and immunities       *
+* granted to it by virtue of its status as an Intergovernmental Organization        *
+* or submit itself to any jurisdiction.                                             *
+\***********************************************************************************/
+#pragma once
+
+#include "FunctionalUtils.h"
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+#include <GaudiKernel/CommonMessaging.h>
+#include <GaudiKernel/FunctionalFilterDecision.h>
+
+#include "podio/CollectionBase.h"
+
+#include "k4FWCore/FunctionalUtils.h"
+#include "k4FWCore/Transformer.h"
+
+#include <type_traits>
+#include <utility>
+
+
+
+namespace k4FWCore {
+
+  namespace details {
+
+    template <typename Signature, typename Traits_>
+    struct Producer;
+
+    template <typename... Out, typename Traits_>
+    struct Producer<std::tuple<Out...>(), Traits_> : MultiTransformer<std::tuple<Out...>(), Traits_> {
+      using MultiTransformer<std::tuple<Out...>(), Traits_>::MultiTransformer;
+    };
+
+    template <typename Out, typename Traits_>
+    struct Producer<Out(), Traits_> : Transformer<Out(), Traits_> {
+      using Transformer<Out(), Traits_>::Transformer;
+    };
+
+  } // namespace details
+
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using Producer = details::Producer<Signature, Traits_>;
+
+} // namespace k4FWCore
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
new file mode 100644
index 00000000..03bfa3ec
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -0,0 +1,213 @@
+/***********************************************************************************\
+* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
+*                                                                                   *
+* This software is distributed under the terms of the Apache version 2 licence,     *
+* copied verbatim in the file "LICENSE".                                            *
+*                                                                                   *
+* In applying this licence, CERN does not waive the privileges and immunities       *
+* granted to it by virtue of its status as an Intergovernmental Organization        *
+* or submit itself to any jurisdiction.                                             *
+\***********************************************************************************/
+#pragma once
+
+#include "FunctionalUtils.h"
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+#include <GaudiKernel/CommonMessaging.h>
+#include <GaudiKernel/FunctionalFilterDecision.h>
+
+#include "podio/CollectionBase.h"
+
+#include "k4FWCore/FunctionalUtils.h"
+
+#include <type_traits>
+#include <utility>
+
+namespace k4FWCore {
+
+namespace details {
+
+  template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+  auto transformm(T&& arg) {
+    // return arg;
+    return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
+  }
+  template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+  auto transformm(T&& arg) {
+    // return arg;
+    std::cout << "Calling static_cast<const T&>(*arg) (transformm)" << std::endl;
+    return static_cast<const T&>(*arg);
+  }
+
+
+  template <typename Signature, typename Traits_>
+  struct Transformer;
+
+  template <typename Out, typename... In, typename Traits_>
+  struct Transformer<Out(const In&...), Traits_>
+      : Gaudi::Functional::details::DataHandleMixin<std::tuple<decltype(transformType(std::declval<Out>()))>,
+                                                    Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
+    using Gaudi::Functional::details::DataHandleMixin<
+        std::tuple<decltype(transformType(std::declval<Out>()))>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_>::DataHandleMixin;
+
+    // derived classes can NOT implement execute
+    StatusCode execute(const EventContext& ctx) const override final {
+      try {
+        if constexpr (sizeof...(In) == 0) {
+          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), transformm((*this)()));
+        } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
+          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), (*this)(ctx));
+        } else {
+          Gaudi::Functional::details::put(std::get<0>(this->m_outputs),
+              transformm(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
+                                              );
+        }
+        return Gaudi::Functional::FilterDecision::PASSED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // instead they MUST implement this operator
+    virtual Out operator()(const In&...) const = 0;
+  };
+
+  //
+  // general N -> M algorithms
+  //
+  template <typename Signature, typename Traits_>
+  struct MultiTransformer;
+
+  template <typename... Out, typename... In, typename Traits_>
+  struct MultiTransformer<std::tuple<Out...>(const In&...), Traits_>
+      : Gaudi::Functional::details::DataHandleMixin<std::tuple<decltype(transformType(std::declval<Out>()))...>,
+                                                    Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
+    using Gaudi::Functional::details::DataHandleMixin<
+        std::tuple<decltype(transformType(std::declval<Out>()))...>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_>::DataHandleMixin;
+
+    // derived classes can NOT implement execute
+    StatusCode execute(const EventContext& ctx) const override final {
+      try {
+        GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN
+        std::apply(
+            [this, &ctx](auto&... ohandle) {
+              if constexpr (sizeof...(In) == 0) {
+                std::apply(
+                    [&ohandle...](auto&&... data) {
+                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                    },
+                    (*this)());
+              } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
+                std::apply(
+                    [&ohandle...](auto&&... data) {
+                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                    },
+                    (*this)(ctx));
+              } else {
+                std::apply(
+                    [&ohandle...](auto&&... data) {
+                      // (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
+                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                    },
+                    filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs));
+              }
+            },
+            this->m_outputs);
+        GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
+        return Gaudi::Functional::FilterDecision::PASSED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // instead they MUST implement this operator
+    virtual std::tuple<Out...> operator()(const In&...) const = 0;
+  };
+
+  //
+  // general N -> M algorithms with filter functionality
+  //
+  template <typename Signature, typename Traits_, bool isLegacy>
+  struct MultiTransformerFilter;
+
+  template <typename... Out, typename... In, typename Traits_>
+  struct MultiTransformerFilter<std::tuple<Out...>(const In&...), Traits_, true>
+      : Gaudi::Functional::details::DataHandleMixin<std::tuple<Out...>,
+                                                    Gaudi::Functional::details::filter_evtcontext<In...>, Traits_> {
+    using Gaudi::Functional::details::DataHandleMixin<
+        std::tuple<Out...>, Gaudi::Functional::details::filter_evtcontext<In...>, Traits_>::DataHandleMixin;
+
+    // derived classes can NOT implement execute
+    StatusCode execute() override final {
+      try {
+        return std::apply(
+                   [&](auto&... ohandle) {
+                     GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN
+                     return std::apply(
+                         [&ohandle...](bool passed, auto&&... data) {
+                           (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
+                           return passed;
+                         },
+                         Gaudi::Functional::details::filter_evtcontext_t<In...>::apply(*this, this->m_inputs));
+                     GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
+                   },
+                   this->m_outputs)
+            ? Gaudi::Functional::FilterDecision::PASSED
+            : Gaudi::Functional::FilterDecision::FAILED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // instead they MUST implement this operator
+    virtual std::tuple<bool, Out...> operator()(const In&...) const = 0;
+  };
+
+  template <typename... Out, typename... In, typename Traits_>
+  struct MultiTransformerFilter<std::tuple<Out...>(const In&...), Traits_, false>
+      : Gaudi::Functional::details::DataHandleMixin<std::tuple<Out...>,
+                                                    Gaudi::Functional::details::filter_evtcontext<In...>, Traits_> {
+    using Gaudi::Functional::details::DataHandleMixin<
+        std::tuple<Out...>, Gaudi::Functional::details::filter_evtcontext<In...>, Traits_>::DataHandleMixin;
+
+    // derived classes can NOT implement execute
+    StatusCode execute(const EventContext& ctx) const override final {
+      try {
+        return std::apply(
+                   GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN[&](auto&... ohandle) {
+                     return std::apply(
+                         [&ohandle...](bool passed, auto&&... data) {
+                           (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
+                           return passed;
+                         },
+                         Gaudi::Functional::details::filter_evtcontext_t<In...>::apply(*this, ctx, this->m_inputs));
+                   },
+                   GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
+
+                   this->m_outputs)
+            ? Gaudi::Functional::FilterDecision::PASSED
+            : Gaudi::Functional::FilterDecision::FAILED;
+      } catch (GaudiException& e) {
+        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+        return e.code();
+      }
+    }
+
+    // instead they MUST implement this operator
+    virtual std::tuple<bool, Out...> operator()(const In&...) const = 0;
+  };
+} // namespace details
+
+template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+using Transformer = details::Transformer<Signature, Traits_>;
+
+template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+using MultiTransformer = details::MultiTransformer<Signature, Traits_>;
+
+template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+using MultiTransformerFilter = details::MultiTransformerFilter<Signature, Traits_, false>;
+
+} // namespace k4FWCore
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 8bb5209d..2ee10bdc 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -105,11 +105,10 @@ add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIE
 add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit")
 
 
-add_test_with_env(ExampleFunctionalProducer options/runExampleFunctionalProducer.py)
-add_test_with_env(ExampleFunctionalConsumer options/runExampleFunctionalConsumer.py PROPERTIES DEPENDS ExampleFunctionalProducer)
-add_test_with_env(ExampleFunctionalTransformer options/runExampleFunctionalTransformer.py PROPERTIES DEPENDS ExampleFunctionalProducer)
-add_test_with_env(ExampleFunctionalProducerMultiple options/runExampleFunctionalProducerMultiple.py)
-add_test_with_env(ExampleFunctionalConsumerMultiple options/runExampleFunctionalConsumerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
-add_test_with_env(ExampleFunctionalTransformerMultiple options/runExampleFunctionalTransformerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
-add_test_with_env(FunctionalChain options/runFunctionalChain.py)
+add_test_with_env(FunctionalsMemory options/ExampleFunctionalsMemory.py)
+add_test_with_env(FunctionalsMultipleMemory options/ExampleFunctionalsMultipleMemory.py)
+add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
+add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
+add_test_with_env(FunctionalsFile options/ExampleFunctionalsFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
+add_test_with_env(FunctionalsMultipleFile options/ExampleFunctionalsFileMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
new file mode 100644
index 00000000..d8406702
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with a single input
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer
+from Configurables import EventDataSvc
+from Configurables import ApplicationMgr
+
+producer = ExampleFunctionalProducer("ExampleFunctionalProducer",
+                                     OutputCollection="MCParticles",
+                                     )
+
+consumer = ExampleFunctionalConsumer("ExampleFunctionalConsumer",
+                                     InputCollection="MCParticles",
+                                     )
+
+ApplicationMgr(TopAlg=[producer, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
new file mode 100644
index 00000000..ae18cc1d
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with a single input
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerMultiple
+from Configurables import EventDataSvc
+from Configurables import ApplicationMgr
+
+producer = ExampleFunctionalProducerMultiple("Producer",
+                                             OutputCollectionFloat="VectorFloat",
+                                             OutputCollectionParticles1="MCParticles1",
+                                             OutputCollectionParticles2="MCParticles2",
+                                             OutputCollectionSimTrackerHits="SimTrackerHits",
+                                             OutputCollectionTrackerHits="TrackerHits",
+                                             OutputCollectionTracks="Tracks",
+                                             ExampleInt=5
+                                             )
+
+consumer = ExampleFunctionalConsumerMultiple("Consumer",
+                                             InputCollectionFloat="VectorFloat",
+                                             InputCollectionParticles1="MCParticles1",
+                                             InputCollectionParticles2="MCParticles2",
+                                             InputCollectionSimTrackerHits="SimTrackerHits",
+                                             InputCollectionTrackerHits="TrackerHits",
+                                             InputCollectionTracks="Tracks",
+                                             ExampleInt=5
+                                             )
+
+ApplicationMgr(TopAlg=[producer, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
similarity index 100%
rename from test/k4FWCoreTest/options/runExampleFunctionalProducer.py
rename to test/k4FWCoreTest/options/ExampleFunctionalProducer.py
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalProducerMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
similarity index 100%
rename from test/k4FWCoreTest/options/runExampleFunctionalProducerMultiple.py
rename to test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFile.py b/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
new file mode 100644
index 00000000..c439eb98
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformer
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc, IOSvc
+from Configurables import Reader, SaveToFile
+import podio
+
+svc = IOSvc("IOSvc")
+svc.input = ['output_k4test_exampledata_producer.root']
+svc.output = 'output_k4test_exampledata_transformer.root'
+# svc.CollectionNames = ['MCParticles']
+
+reader = Reader("Reader")
+
+writer = SaveToFile("Writer")
+
+# out = PodioOutput("out")
+# out.filename = "output_k4test_exampledata_transformer.root"
+# # The collections that we don't drop will also be present in the output file
+# out.outputCommands = ["drop MCParticles"]
+
+transformer = ExampleFunctionalTransformer("Transformer",
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
+
+mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
+# podio_reader = podio.root_io.Reader('output_k4test_exampledata_transformer.root')
+# for frame in podio_reader.get('events'):
+#     frame.get('NewMCParticles')
+    
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
new file mode 100644
index 00000000..053ba6e7
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformerMultiple
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc, IOSvc
+from Configurables import Reader, SaveToFile
+import podio
+
+svc = IOSvc("IOSvc")
+svc.input = ['output_k4test_exampledata_producer_multiple.root']
+svc.output = 'output_k4test_exampledata_transformer_multiple.root'
+# svc.CollectionNames = ['MCParticles']
+
+reader = Reader("Reader")
+
+writer = SaveToFile("Writer")
+
+# out = PodioOutput("out")
+# out.filename = "output_k4test_exampledata_transformer.root"
+# # The collections that we don't drop will also be present in the output file
+# out.outputCommands = ["drop MCParticles"]
+
+transformer = ExampleFunctionalTransformerMultiple("Transformer",
+                                           # InputCollection="MCParticles",
+                                           # OutputCollection="NewMCParticles")
+                                                   )
+
+mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
+# podio_reader = podio.root_io.Reader('output_k4test_exampledata_transformer.root')
+# for frame in podio_reader.get('events'):
+#     frame.get('NewMCParticles')
+    
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalTransformer.py b/test/k4FWCoreTest/options/ExampleFunctionalsMemory.py
similarity index 100%
rename from test/k4FWCoreTest/options/runExampleFunctionalTransformer.py
rename to test/k4FWCoreTest/options/ExampleFunctionalsMemory.py
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py
new file mode 100644
index 00000000..26f42afb
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalTransformerMultiple, ExampleFunctionalConsumerMultiple
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+transformer = ExampleFunctionalTransformerMultiple("Transformer",
+                                                   InputCollectionFloat="VectorFloat",
+                                                   InputCollectionParticles="MCParticles1",
+                                                   InputCollectionSimTrackerHits="SimTrackerHits",
+                                                   InputCollectionTrackerHits="TrackerHits",
+                                                   InputCollectionTracks="Tracks",
+                                                   OutputCollectionCounter="Counter",
+                                                   OutputCollectionParticles="NewMCParticles",
+                                                   Offset=10,
+                                                   )
+
+producer = ExampleFunctionalProducerMultiple("Producer")
+
+consumer = ExampleFunctionalConsumerMultiple("Consumer",
+                                             InputCollectionFloat="VectorFloat",
+                                             InputCollectionParticles="NewMCParticles",
+                                             InputCollectionSimTrackerHits="SimTrackerHits",
+                                             InputCollectionTrackerHits="TrackerHits",
+                                             InputCollectionTracks="Tracks",
+                                             Offset=10,
+                                             )
+
+
+ApplicationMgr(TopAlg=[producer, transformer, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py
index b7d0af72..69396ee6 100644
--- a/test/k4FWCoreTest/options/runEventHeaderCheck.py
+++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py
@@ -19,23 +19,22 @@
 #
 
 from Gaudi.Configuration import INFO
-from Configurables import k4DataSvc
-from Configurables import PodioInput
 from Configurables import ExampleEventHeaderConsumer
 from Configurables import ApplicationMgr
+from Configurables import EventDataSvc, IOSvc, Reader
 
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "eventHeader.root"
+svc = IOSvc("IOSvc")
+svc.FileNames = ["eventHeader.root"]
+svc.CollectionNames = ['MCParticles']
 
-inp = PodioInput()
-inp.collections = []
+reader = Reader("Reader")
 
 consumer = ExampleEventHeaderConsumer("EventHeaderCheck", runNumber=42, eventNumberOffset=42)
 
 ApplicationMgr(
-    TopAlg=[inp, consumer],
+    TopAlg=[reader, consumer],
     EvtSel="NONE",
     EvtMax=-1,
-    ExtSvc=[podioevent],
+    ExtSvc=[EventDataSvc("EventDataSvc")],
     OutputLevel=INFO,
 )
diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
index 7f1f6d70..4d33775a 100644
--- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
@@ -20,7 +20,7 @@
 #include "edm4hep/Constants.h"
 #include "edm4hep/EventHeaderCollection.h"
 
-#include "k4FWCore/BaseClass.h"
+#include "k4FWCore/Consumer.h"
 
 #include <Gaudi/Functional/Consumer.h>
 #include <Gaudi/Property.h>
@@ -34,7 +34,7 @@
 #include <string>
 
 struct ExampleEventHeaderConsumer final
-    : Gaudi::Functional::Consumer<void(const edm4hep::EventHeaderCollection&), BaseClass_t> {
+    : k4FWCore::Consumer<void(const edm4hep::EventHeaderCollection&)> {
   ExampleEventHeaderConsumer(const std::string& name, ISvcLocator* svcLoc)
       : Consumer(name, svcLoc, {KeyValue("EventHeaderName", edm4hep::EventHeaderName)}) {}
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index b2bcf0f7..f5043dc8 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -18,95 +18,49 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Consumer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 
-// Define BaseClass_t
-#include "k4FWCore/BaseClass.h"
+#include "k4FWCore/Consumer.h"
 
 #include <stdexcept>
 #include <string>
 #include <thread>
 
-#include "k4FWCore/Consumer.h"
-
-
-struct ExampleFunctionalConsumer final : Gaudi::Functional::CConsumer<void(const edm4hep::MCParticleCollection& input)> {
-// struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const edm4hep::MCParticleCollection& input), BaseClass_t> {
+struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::MCParticleCollection& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the input collection
   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
-      : CConsumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+      : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as the collections
   // we get from the input
   void operator()(const edm4hep::MCParticleCollection& input) const override {
-  // void operator()(const edm4hep::MCParticleCollection& input) const override {
-  // void operator()(const edm4hep::MCParticleCollection& input) const override {
-    info() << "ExampleFunctionalConsumer::operator()" << endmsg;
-    info() << "Size of collection " << input.size() << endmsg;
-    // std::this_thread::sleep_for(std::chrono::milliseconds(2000));
+    if (input.size() != 2) {
+      fatal() << "Wrong size of MCParticle collection, expected 2 got " << input.size() << endmsg;
+    }
     int i = 0;
     for (const auto& particle : input) {
-      info() << particle.getPDG() << endmsg;
-      if ((particle.getPDG() != 1 + i + m_possibleOffset) ||
-          (particle.getGeneratorStatus() != 2 + i + m_possibleOffset) ||
-          (particle.getSimulatorStatus() != 3 + i + m_possibleOffset) ||
-          (particle.getCharge() != 4 + i + m_possibleOffset) || (particle.getTime() != 5 + i + m_possibleOffset) ||
-          (particle.getMass() != 6 + i + m_possibleOffset)) {
+      if ((particle.getPDG() != 1 + i + m_offset) ||
+          (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+          (particle.getSimulatorStatus() != 3 + i + m_offset) ||
+          (particle.getCharge() != 4 + i + m_offset) || (particle.getTime() != 5 + i + m_offset) ||
+          (particle.getMass() != 6 + i + m_offset)) {
         std::stringstream error;
-        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_possibleOffset << ", "
-              << 2 + i + m_possibleOffset << ", " << 3 + i + m_possibleOffset << ", " << 4 + i + m_possibleOffset
-              << ", " << 5 + i + m_possibleOffset << ", " << 6 + i + m_possibleOffset << " got " << particle.getPDG()
+        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", "
+              << 2 + i + m_offset << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset
+              << ", " << 5 + i + m_offset << ", " << 6 + i + m_offset << " got " << particle.getPDG()
               << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
               << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
-        // throw std::runtime_error(error.str());
+        throw std::runtime_error(error.str());
       }
       i++;
     }
   }
 
-  Gaudi::Property<int> m_possibleOffset{this, "PossibleOffset", 0, "Possible offset in the values data"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10,
+      "Integer to add to the dummy values written to the edm"};
 };
 
-
-// struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const std::shared_ptr<podio::CollectionBase>& input)> {
-// // struct ExampleFunctionalConsumer final : Gaudi::Functional::Consumer<void(const edm4hep::MCParticleCollection& input), BaseClass_t> {
-//   // The pair in KeyValue can be changed from python and it corresponds
-//   // to the name of the input collection
-//   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
-//       : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
-
-//   // This is the function that will be called to transform the data
-//   // Note that the function has to be const, as well as the collections
-//   // we get from the input
-//   void operator()(const std::shared_ptr<podio::CollectionBase>& input) const override {
-//   // void operator()(const edm4hep::MCParticleCollection& input) const override {
-//     info() << "ExampleFunctionalConsumer::operator()" << endmsg;
-//     info() << "Size of collection " << input->size() << endmsg;
-//     // std::this_thread::sleep_for(std::chrono::milliseconds(2000));
-//     int i = 0;
-//     for (const auto& particle : static_cast<const edm4hep::MCParticleCollection&>(*input)) {
-//       if ((particle.getPDG() != 1 + i + m_possibleOffset) ||
-//           (particle.getGeneratorStatus() != 2 + i + m_possibleOffset) ||
-//           (particle.getSimulatorStatus() != 3 + i + m_possibleOffset) ||
-//           (particle.getCharge() != 4 + i + m_possibleOffset) || (particle.getTime() != 5 + i + m_possibleOffset) ||
-//           (particle.getMass() != 6 + i + m_possibleOffset)) {
-//         std::stringstream error;
-//         error << "Wrong data in MCParticle collection, expected " << 1 + i + m_possibleOffset << ", "
-//               << 2 + i + m_possibleOffset << ", " << 3 + i + m_possibleOffset << ", " << 4 + i + m_possibleOffset
-//               << ", " << 5 + i + m_possibleOffset << ", " << 6 + i + m_possibleOffset << " got " << particle.getPDG()
-//               << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
-//               << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
-//         // throw std::runtime_error(error.str());
-//       }
-//       i++;
-//     }
-//   }
-
-//   Gaudi::Property<int> m_possibleOffset{this, "PossibleOffset", 0, "Possible offset in the values data"};
-// };
-
 DECLARE_COMPONENT(ExampleFunctionalConsumer)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
index 02109493..95a82350 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
@@ -18,7 +18,6 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Consumer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
@@ -33,8 +32,6 @@ namespace edm4hep {
 #endif
 #include "podio/UserDataCollection.h"
 
-// Define BaseClass_t
-#include "k4FWCore/BaseClass.h"
 #include "k4FWCore/Consumer.h"
 
 #include <sstream>
@@ -49,12 +46,12 @@ using TrackerHitColl    = edm4hep::TrackerHit3DCollection;
 using TrackColl         = edm4hep::TrackCollection;
 
 struct ExampleFunctionalConsumerMultiple final
-    : Gaudi::Functional::CConsumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
+    : k4FWCore::Consumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
                                        const TrackerHitColl&, const TrackColl&)> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the input collection
   ExampleFunctionalConsumerMultiple(const std::string& name, ISvcLocator* svcLoc)
-      : CConsumer(name, svcLoc,
+      : Consumer(name, svcLoc,
                  {
                      KeyValue("InputCollectionFloat", "VectorFloat"),
                      KeyValue("InputCollectionParticles", "MCParticles1"),
@@ -68,7 +65,6 @@ struct ExampleFunctionalConsumerMultiple final
   // we get from the input
   void operator()(const FloatColl& floatVector, const ParticleColl& particles, const SimTrackerHitColl& simTrackerHits,
                   const TrackerHitColl& trackerHits, const TrackColl& tracks) const override {
-    info() << "ExampleFunctionalConsumerMultiple called" << endmsg;
     if (floatVector.size() != 3) {
       throw std::runtime_error("Wrong size of floatVector collection, expected 3, got " +
                                std::to_string(floatVector.size()) + "");
@@ -80,15 +76,22 @@ struct ExampleFunctionalConsumerMultiple final
       throw std::runtime_error(error.str());
     }
 
-    info() << "VectorFloat collection is ok" << endmsg;
-    auto p4 = particles.momentum()[0];
-    if ((p4.x != m_magicNumberOffset + 5) || (p4.y != m_magicNumberOffset + 6) || (p4.z != m_magicNumberOffset + 7) ||
-        (particles[0].getMass() != m_magicNumberOffset + 8)) {
-      std::stringstream error;
-      error << "Wrong data in particles collection, expected " << m_magicNumberOffset + 5 << ", "
-            << m_magicNumberOffset + 6 << ", " << m_magicNumberOffset + 7 << ", " << m_magicNumberOffset + 8 << " got "
-            << p4.x << ", " << p4.y << ", " << p4.z << ", " << particles[0].getMass() << "";
-      throw std::runtime_error(error.str());
+    int i = 0;
+    for (const auto& particle : particles) {
+      if ((particle.getPDG() != 1 + i + m_offset) ||
+          (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+          (particle.getSimulatorStatus() != 3 + i + m_offset) ||
+          (particle.getCharge() != 4 + i + m_offset) || (particle.getTime() != 5 + i + m_offset) ||
+          (particle.getMass() != 6 + i + m_offset)) {
+        std::stringstream error;
+        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", "
+              << 2 + i + m_offset << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset
+              << ", " << 5 + i + m_offset << ", " << 6 + i + m_offset << " got " << particle.getPDG()
+              << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
+              << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
+        throw std::runtime_error(error.str());
+      }
+      i++;
     }
 
     if ((simTrackerHits[0].getPosition()[0] != 3) || (simTrackerHits[0].getPosition()[1] != 4) ||
@@ -119,9 +122,8 @@ struct ExampleFunctionalConsumerMultiple final
   }
 
 private:
-  // integer to add to the dummy values written to the edm
-  Gaudi::Property<int> m_magicNumberOffset{this, "magicNumberOffset", 0,
-                                           "Integer to add to the dummy values written to the edm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10,
+      "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalConsumerMultiple)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
index fbd319b3..4f4a41c9 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
@@ -19,13 +19,14 @@
 
 #include "Gaudi/Property.h"
 #include "GaudiAlg/Producer.h"
-#include "k4FWCore/BaseClass.h"
 
 #include "edm4hep/MCParticleCollection.h"
 
+#include "k4FWCore/Producer.h"
+
 #include <string>
 
-struct ExampleFunctionalProducer final : Gaudi::Functional::Producer<edm4hep::MCParticleCollection(), BaseClass_t> {
+struct ExampleFunctionalProducer final : k4FWCore::Producer<edm4hep::MCParticleCollection()> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducer(const std::string& name, ISvcLocator* svcLoc)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index ae14422f..e9504908 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -19,19 +19,13 @@
 
 #include "Gaudi/Property.h"
 #include "GaudiAlg/Producer.h"
-#include "k4FWCore/BaseClass.h"
+
+#include "k4FWCore/Producer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
-#if __has_include("edm4hep/TrackerHit3DCollection.h")
-#include "edm4hep/TrackerHit3DCollection.h"
-#else
 #include "edm4hep/TrackerHitCollection.h"
-namespace edm4hep {
-  using TrackerHit3DCollection = edm4hep::TrackerHitCollection;
-}  // namespace edm4hep
-#endif
 #include "podio/UserDataCollection.h"
 
 #include <string>
@@ -40,13 +34,12 @@ namespace edm4hep {
 using Float         = podio::UserDataCollection<float>;
 using Particle      = edm4hep::MCParticleCollection;
 using SimTrackerHit = edm4hep::SimTrackerHitCollection;
-using TrackerHit    = edm4hep::TrackerHit3DCollection;
+using TrackerHit    = edm4hep::TrackerHitCollection;
 using Track         = edm4hep::TrackCollection;
 using Ptr = std::shared_ptr<podio::CollectionBase>;
 
 struct ExampleFunctionalProducerMultiple final
-    // : Gaudi::Functional::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>(),
-  : Gaudi::Functional::Producer<std::tuple<Ptr, Ptr, Ptr, Ptr, Ptr, Ptr>()> {
+  : k4FWCore::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>()> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the output collections
   ExampleFunctionalProducerMultiple(const std::string& name, ISvcLocator* svcLoc)
@@ -58,14 +51,14 @@ struct ExampleFunctionalProducerMultiple final
              KeyValue("OutputCollectionTrackerHits", "TrackerHits"), KeyValue("OutputCollectionTracks", "Tracks")}) {}
 
   // This is the function that will be called to produce the data
-  std::tuple<Ptr, Ptr, Ptr, Ptr, Ptr, Ptr> operator()() const override {
+  std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track> operator()() const override {
     // The following was copied and adapted from the
     // k4FWCoreTest_CreateExampleEventData test
 
-    auto floatVector = new podio::UserDataCollection<float>();
-    floatVector->push_back(125.);
-    floatVector->push_back(25.);
-    floatVector->push_back(m_event);
+    auto floatVector = podio::UserDataCollection<float>();
+    floatVector.push_back(125.);
+    floatVector.push_back(25.);
+    floatVector.push_back(m_event);
 
     auto particles = edm4hep::MCParticleCollection();
     auto particle  = particles.create();
@@ -75,17 +68,17 @@ struct ExampleFunctionalProducerMultiple final
     p4.z            = m_magicNumberOffset + 7;
     particle.setMass(m_magicNumberOffset + m_event + 8);
 
-    auto simTrackerHits = new edm4hep::SimTrackerHitCollection();
-    auto hit            = simTrackerHits->create();
+    auto simTrackerHits = edm4hep::SimTrackerHitCollection();
+    auto hit            = simTrackerHits.create();
     hit.setPosition({3, 4, 5});
 
-    auto trackerHits = new edm4hep::TrackerHit3DCollection();
-    auto trackerHit  = trackerHits->create();
+    auto trackerHits = edm4hep::TrackerHitCollection();
+    auto trackerHit  = trackerHits.create();
     trackerHit.setPosition({3, 4, 5});
 
-    auto tracks = new edm4hep::TrackCollection();
-    auto track  = tracks->create();
-    auto track2 = tracks->create();
+    auto tracks = edm4hep::TrackCollection();
+    auto track  = tracks.create();
+    auto track2 = tracks.create();
     // set members
     track.setType(1);
     track.setChi2(2.1);
@@ -100,14 +93,8 @@ struct ExampleFunctionalProducerMultiple final
     track.addToTrackerHits(trackerHit);
     track.addToTracks(track2);
 
-    // return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits),
-    //                        std::move(trackerHits), std::move(tracks));
-    return std::make_tuple(std::shared_ptr<podio::CollectionBase>(floatVector),
-                            std::shared_ptr<podio::CollectionBase>(particles),
-                            std::shared_ptr<podio::CollectionBase>(new Particle()),
-                            std::shared_ptr<podio::CollectionBase>(simTrackerHits),
-                            std::shared_ptr<podio::CollectionBase>(trackerHits),
-                            std::shared_ptr<podio::CollectionBase>(tracks));
+    return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits),
+                           std::move(trackerHits), std::move(tracks));
   }
 
 private:
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
index 0abe3c31..05280b76 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
@@ -23,8 +23,7 @@
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/MutableMCParticle.h"
 
-// Define BaseClass_t
-#include "k4FWCore/BaseClass.h"
+#include "k4FWCore/Transformer.h"
 
 #include <string>
 
@@ -33,7 +32,7 @@ using colltype_in  = edm4hep::MCParticleCollection;
 using colltype_out = edm4hep::MCParticleCollection;
 
 struct ExampleFunctionalTransformer final
-    : Gaudi::Functional::Transformer<colltype_out(const colltype_in&), BaseClass_t> {
+    : k4FWCore::Transformer<colltype_out(const colltype_in&)> {
   ExampleFunctionalTransformer(const std::string& name, ISvcLocator* svcLoc)
       : Transformer(name, svcLoc, KeyValue("InputCollection", "MCParticles"),
                     KeyValue("OutputCollection", "NewMCParticles")) {}
@@ -45,16 +44,21 @@ struct ExampleFunctionalTransformer final
     auto coll_out = edm4hep::MCParticleCollection();
     for (const auto& particle : input) {
       auto new_particle = edm4hep::MutableMCParticle();
-      new_particle.setPDG(particle.getPDG() + 10);
-      new_particle.setGeneratorStatus(particle.getGeneratorStatus() + 10);
-      new_particle.setSimulatorStatus(particle.getSimulatorStatus() + 10);
-      new_particle.setCharge(particle.getCharge() + 10);
-      new_particle.setTime(particle.getTime() + 10);
-      new_particle.setMass(particle.getMass() + 10);
+      new_particle.setPDG(particle.getPDG() + m_offset);
+      new_particle.setGeneratorStatus(particle.getGeneratorStatus() + m_offset);
+      new_particle.setSimulatorStatus(particle.getSimulatorStatus() + m_offset);
+      new_particle.setCharge(particle.getCharge() + m_offset);
+      new_particle.setTime(particle.getTime() + m_offset);
+      new_particle.setMass(particle.getMass() + m_offset);
       coll_out->push_back(new_particle);
     }
     return coll_out;
   }
+
+private:
+  // integer to add to the dummy values written to the edm
+  Gaudi::Property<int> m_offset{this, "Offset", 10,
+      "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalTransformer)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
index e84587fa..1d354812 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
@@ -18,7 +18,6 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Transformer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
@@ -34,8 +33,7 @@ namespace edm4hep {
 #include "edm4hep/TrackerHit3DCollection.h"
 #include "podio/UserDataCollection.h"
 
-// Define BaseClass_t
-#include "k4FWCore/BaseClass.h"
+#include "k4FWCore/Transformer.h"
 
 #include <string>
 
@@ -51,10 +49,9 @@ using Counter  = podio::UserDataCollection<int>;
 using Particle = edm4hep::MCParticleCollection;
 
 struct ExampleFunctionalTransformerMultiple final
-    : Gaudi::Functional::MultiTransformer<std::tuple<Counter, Particle>(const FloatColl&, const ParticleColl&,
+    : k4FWCore::MultiTransformer<std::tuple<Counter, Particle>(const FloatColl&, const ParticleColl&,
                                                                         const SimTrackerHitColl&, const TrackerHitColl&,
-                                                                        const TrackColl&),
-                                          BaseClass_t> {
+                                                                        const TrackColl&)> {
   ExampleFunctionalTransformerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : MultiTransformer(
             name, svcLoc,
@@ -80,23 +77,23 @@ struct ExampleFunctionalTransformerMultiple final
 
       // We could create a new one
       auto newParticle = newParticlesColl->create();
-      newParticle.setPDG(p.getPDG());
-      newParticle.setGeneratorStatus(p.getGeneratorStatus() + 1);
-      newParticle.setSimulatorStatus(p.getSimulatorStatus() + 1);
-      newParticle.setCharge(p.getCharge() + 2);
-      newParticle.setTime(p.getTime() + 3);
-      newParticle.setMass(p.getMass() + 4);
+      newParticle.setPDG(p.getPDG() + m_offset);
+      newParticle.setGeneratorStatus(p.getGeneratorStatus() + m_offset);
+      newParticle.setSimulatorStatus(p.getSimulatorStatus() + m_offset);
+      newParticle.setCharge(p.getCharge() + m_offset);
+      newParticle.setTime(p.getTime() + m_offset);
+      newParticle.setMass(p.getMass() + m_offset);
     }
     counter.push_back(particles.size());
-
     counter.push_back(simTrackerHits.size());
-
     counter.push_back(trackerHits.size());
-
     counter.push_back(tracks.size());
 
     return std::make_tuple(std::move(counter), std::move(newParticlesColl));
   }
+
+  Gaudi::Property<int> m_offset{this, "Offset", 10,
+      "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalTransformerMultiple)

From 2a91804fb334da3ef2e096a6551b19dba3b45408 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sat, 30 Dec 2023 12:52:56 +0100
Subject: [PATCH 004/127] Add fixes to read multiple collections

---
 k4FWCore/components/IIOSvc.h   |  2 +-
 k4FWCore/components/IOSvc.cpp  |  4 +-
 k4FWCore/components/IOSvc.h    |  3 +-
 k4FWCore/components/Reader.cpp | 80 ++++++++++++----------------------
 4 files changed, 33 insertions(+), 56 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 18a043a3..7ce1c3f2 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -41,7 +41,7 @@ class IIOSvc : virtual public IInterface {
    * by the first one
    * @throws IIOSvc::EndOfInput
    */
-  virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> next( ) = 0;
+  virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next( ) = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
   virtual std::shared_ptr<podio::ROOTFrameWriter> getWriter() = 0;
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index ed0911ed..a2555376 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -49,7 +49,7 @@ StatusCode IOSvc::initialize() {
 
 StatusCode IOSvc::finalize() { return Service::finalize(); }
 
-std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> IOSvc::next() {
+std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> IOSvc::next() {
   info() << "IOSvc::next()" << endmsg;
   podio::Frame frame;
   {
@@ -68,7 +68,7 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> IO
     collections.push_back(std::shared_ptr<podio::CollectionBase>(ptr));
   }
 
-  return std::make_tuple(collections, std::move(frame));
+  return std::make_tuple(collections, m_collectionNames, std::move(frame));
 }
 
 // After every event if there is still a frame in the TES
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index e61feba0..57c97d27 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -50,7 +50,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   StatusCode initialize() override;
   StatusCode finalize() override;
 
-  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, podio::Frame> next() override;
+  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next() override;
 
   std::shared_ptr<std::vector<std::string>> getCollectionNames() const override {
     return std::make_shared<std::vector<std::string>>(m_collectionNames);
@@ -64,7 +64,6 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   Gaudi::Property<unsigned int> m_bufferNbEvents{
       this, "BufferNbEvents", 20000,
       "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
-  Gaudi::Property<std::vector<std::string>> m_input{this, "Input", {}, "List of inputs"};
   Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
 
   Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index f511b4f4..575181b3 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -20,67 +20,56 @@
 #include "Gaudi/Functional/utilities.h"
 #include "GaudiKernel/FunctionalFilterDecision.h"
 #include "GaudiKernel/StatusCode.h"
+#include "GaudiKernel/IDataHandleHolder.h"
+#include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/DataHandle.h"
+#include "GaudiKernel/IDataManagerSvc.h"
 
 #include "IIOSvc.h"
 
 #include "podio/CollectionBase.h"
 #include "podio/Frame.h"
 
-#include <GaudiKernel/AnyDataWrapper.h>
-#include <GaudiKernel/IDataManagerSvc.h>
 #include <memory>
 
-using podio::CollectionBase;
-
-namespace Gaudi::Functional {
 template <typename Container>
 using vector_of_ = std::vector<Container>;
 template <typename Container>
 using vector_of_optional_ = std::vector<std::optional<Container>>;
 
-namespace details {
 
-  class MyTransformer : public BaseClass_t<Gaudi::Functional::Traits::useDefaults> {
+class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::Functional::Traits::useDefaults> {
     using Traits_ = Gaudi::Functional::Traits::useDefaults;
     using Out = std::shared_ptr<podio::CollectionBase>;
-    using base_class = BaseClass_t<Traits_>;
+    using base_class = Gaudi::Functional::details::BaseClass_t<Traits_>;
     static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
 
   public:
-    using KeyValues = std::pair<std::string, std::vector<std::string>>;
-
-    MyTransformer(std::string name, ISvcLocator* locator, const KeyValues& outputs) :
-        base_class(std::move(name), locator),
-        m_outputLocations(
-            this, outputs.first, details::to_DataObjID(outputs.second),
-            [this](Gaudi::Details::PropertyBase&) {
-              this->m_outputs = details::make_vector_of_handles<decltype(this->m_outputs)>(this, m_outputLocations);
-              // if constexpr (details::is_optional_v<Out>) { // handle constructor does not (yet) allow to
-              //                                              // set optional flag... so
-              //                                              // do it explicitly here...
-              //   std::for_each(this->m_outputs.begin(), this->m_outputs.end(), [](auto& h) { h.setOptional(true); });
-              // }
-            },
-            Gaudi::Details::Property::ImmediatelyInvokeHandler{true})
+    CollectionPusher(std::string name, ISvcLocator* locator) :
+      base_class(std::move(name), locator)
     {}
 
     // derived classes can NOT implement execute
     StatusCode execute(const EventContext&) const override final {
-      // Gaudi::Algorithm::info() << "MyTransformer::execute()" << endmsg;
       try {
-        // TODO:FIXME: how does operator() know the number and order of expected outputs?
         auto out = (*this)();
+
+        auto outColls = std::get<0>(out);
+        auto outputLocations = std::get<1>(out);
+
         // if (out.size() != m_outputs.size()) {
         //   throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
         //                            " containers, got " + std::to_string(out.size()) + " instead",
         //                        this->name(), StatusCode::FAILURE);
         // }
-        for (unsigned i = 0; i != out.size(); ++i) {
-          // Gaudi::Algorithm::info() << "MyTransformer::execute() : putting " << m_outputs[i].fullKey() << endmsg;
-          details::put(m_outputs[0], std::move(out[i]));
+        for (unsigned i = 0; i != outColls.size(); ++i) {
+          auto objectp = std::make_unique<AnyDataWrapper<Out>>( std::move( outColls[i] ) );
+          if ( auto sc = datasvc->registerObject( outputLocations[i], objectp.get() ); sc.isFailure() ) {
+          }
+          // The store has the ownership so we shouldn't delete the object
+          auto ptr = objectp.release();
         }
-        // details::put(m_frame[0], std::move(frame));
-        return FilterDecision::PASSED;
+        return Gaudi::Functional::FilterDecision::PASSED;
       } catch (GaudiException& e) {
         (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
         return e.code();
@@ -90,7 +79,7 @@ namespace details {
     // TODO/FIXME: how does the callee know in which order to produce the outputs?
     //             (note: 'missing' items can be specified by making Out an std::optional<Out>,
     //              and only those entries which contain an Out are stored)
-    virtual vector_of_<Out> operator()() const = 0;
+    virtual std::tuple<vector_of_<Out>, std::vector<std::string>> operator()() const = 0;
 
   private:
     // if In is a pointer, it signals optional (as opposed to mandatory) input
@@ -100,31 +89,21 @@ namespace details {
     // TODO/FIXME: replace vector of DataObjID property + call-back with a
     //             vector<handle> property ... as soon as declareProperty can deal with that.
     template <typename T>
-    using OutputHandle = details::OutputHandle_t<Traits_, details::remove_optional_t<T>>;
-    std::vector<OutputHandle<Out>> m_outputs;
-    Gaudi::Property<std::vector<DataObjID>> m_outputLocations;
-    // Gaudi::Property<std::vector<DataObjID>> m_outputFrameLocations;
-    // std::vector<OutputHandle<std::shared_ptr<CollectionBase>>> m_frame{};
+    using OutputHandle = Gaudi::Functional::details::OutputHandle_t<Traits_, Gaudi::Functional::details::remove_optional_t<T>>;
+    mutable std::vector<OutputHandle<Out>> m_outputs;
+    ServiceHandle<IDataProviderSvc> datasvc{this, "EventDataSvc", "EventDataSvc"};
 
   };
 
-} // namespace details
-
-} // namespace Gaudi::Functional
-
-using Gaudi::Functional::details::MyTransformer;
-
-class Reader final : public MyTransformer {
-// class Reader final : public MyTransformer<std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::shared_ptr<CollectionBase>>()> {
+class Reader final : public CollectionPusher {
 public:
   Reader(const std::string& name, ISvcLocator* svcLoc) :
-    MyTransformer(name, svcLoc, {"OutputLocations", {"MCParticless"}}
-                  
-                  ) {
+    CollectionPusher(name, svcLoc) {
   }
 
   // Gaudi doesn't run the destructor of the Services so we have to
   // manually ask for the reader to be deleted so it will call finish()
+  // See https://gitlab.cern.ch/gaudi/Gaudi/-/issues/169
   ~Reader() {
     iosvc->deleteReader();
   }
@@ -147,19 +126,18 @@ class Reader final : public MyTransformer {
     return StatusCode::SUCCESS;
   }
 
-  std::vector<std::shared_ptr<podio::CollectionBase>> operator()() const override {
+  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>> operator()() const override {
     info() << "Reader::operator()" << endmsg;
     auto val = iosvc->next();
 
     auto eds = eventSvc().as<IDataProviderSvc>();
-    auto frame = std::move(std::get<1>(val));
+    auto frame = std::move(std::get<2>(val));
 
     // We'll hand the ownership of 
     auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
     auto code = eds->registerObject("/Event/_Frame", tmp);
 
-
-    return std::get<0>(val);
+    return std::make_tuple(std::get<0>(val), std::get<1>(val));
   }
 
 };

From e434386b7d819750e1c9b62fe31cededd7328848 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sun, 31 Dec 2023 13:35:38 +0100
Subject: [PATCH 005/127] Fix tests

---
 k4FWCore/components/IIOSvc.h                  |  8 ------
 k4FWCore/components/IOSvc.cpp                 | 12 +++++++--
 k4FWCore/components/IOSvc.h                   |  4 +++
 k4FWCore/components/Reader.cpp                | 15 +++--------
 k4FWCore/components/SaveFile.cpp              | 26 ++++++++++++-------
 k4FWCore/include/k4FWCore/DataHandle.h        |  6 +++++
 .../options/ExampleFunctionalsFile.py         |  2 +-
 .../options/ExampleFunctionalsFileMultiple.py |  2 +-
 .../options/runEventHeaderCheck.py            |  4 +--
 .../ExampleFunctionalProducerMultiple.cpp     | 10 +++----
 10 files changed, 49 insertions(+), 40 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 7ce1c3f2..2f7ff924 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -19,8 +19,6 @@
 #include <memory>
 #include <vector>
 
-#pragma once
-
 /**
  * The interface implemented by any class making IO and reading RawEvent Data
  */
@@ -35,12 +33,6 @@ class IIOSvc : virtual public IInterface {
   /// InterfaceID
   DeclareInterfaceID( IIOSvc, 1, 0 );
 
-  /**
-   * get next event from input
-   * @return a pair RawEvent, shared_ptr<Buffer> where the second one holds the data pointed to
-   * by the first one
-   * @throws IIOSvc::EndOfInput
-   */
   virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next( ) = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index a2555376..f0c0934c 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -26,6 +26,7 @@
 #include "k4FWCore/KeepDropSwitch.h"
 
 #include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/IEventProcessor.h"
 
 #include <mutex>
 #include <thread>
@@ -35,6 +36,7 @@ StatusCode IOSvc::initialize() {
   if (!m_readingFileNames.empty()) {
     m_reader = std::make_unique<podio::ROOTFrameReader>();
     m_reader->openFiles(m_readingFileNames);
+    m_entries = m_reader->getEntries(podio::Category::Event);
   }
 
   m_switch = KeepDropSwitch(m_outputCommands);
@@ -54,16 +56,22 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
   podio::Frame frame;
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
-    frame = podio::Frame(std::move(m_reader->readNextEntry(podio::Category::Event)));
+    frame = podio::Frame(std::move(m_reader->readEntry(podio::Category::Event, m_nextEntry)));
+    m_nextEntry++;
     if (m_collectionNames.empty()) {
         m_collectionNames = frame.getAvailableCollections();
     }
   }
 
+  if (m_nextEntry >= m_entries) {
+    IEventProcessor* eventProcessor;
+    StatusCode sc = service("ApplicationMgr", eventProcessor);
+    sc = eventProcessor->stopRun();
+  }
+
   std::vector<std::shared_ptr<podio::CollectionBase>> collections;
 
   for (const auto& name : m_collectionNames) {
-    info() << "Collection name: " << name << endmsg;
     auto ptr = const_cast<podio::CollectionBase*>(frame.get(name));
     collections.push_back(std::shared_ptr<podio::CollectionBase>(ptr));
   }
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 57c97d27..310572a3 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -71,6 +71,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   Gaudi::Property<std::string> m_writingFileName{this, "output", {}, "List of files to read"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
+  Gaudi::Property<std::string> m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"};
 
   /// lock for handling the change of buffer
   std::mutex m_changeBufferLock;
@@ -101,6 +102,9 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   SmartIF<IDataProviderSvc> m_dataSvc;
   SmartIF<IIncidentSvc> m_incidentSvc;
   void handle(const Incident& incident) override;
+
+  int m_entries{0};
+  int m_nextEntry{0};
 };
 
 #endif
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 575181b3..c212d1a0 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -20,10 +20,8 @@
 #include "Gaudi/Functional/utilities.h"
 #include "GaudiKernel/FunctionalFilterDecision.h"
 #include "GaudiKernel/StatusCode.h"
-#include "GaudiKernel/IDataHandleHolder.h"
 #include "GaudiKernel/AnyDataWrapper.h"
-#include "GaudiKernel/DataHandle.h"
-#include "GaudiKernel/IDataManagerSvc.h"
+#include "GaudiKernel/IDataProviderSvc.h"
 
 #include "IIOSvc.h"
 
@@ -76,9 +74,6 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
       }
     }
 
-    // TODO/FIXME: how does the callee know in which order to produce the outputs?
-    //             (note: 'missing' items can be specified by making Out an std::optional<Out>,
-    //              and only those entries which contain an Out are stored)
     virtual std::tuple<vector_of_<Out>, std::vector<std::string>> operator()() const = 0;
 
   private:
@@ -88,9 +83,6 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
     // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
     // TODO/FIXME: replace vector of DataObjID property + call-back with a
     //             vector<handle> property ... as soon as declareProperty can deal with that.
-    template <typename T>
-    using OutputHandle = Gaudi::Functional::details::OutputHandle_t<Traits_, Gaudi::Functional::details::remove_optional_t<T>>;
-    mutable std::vector<OutputHandle<Out>> m_outputs;
     ServiceHandle<IDataProviderSvc> datasvc{this, "EventDataSvc", "EventDataSvc"};
 
   };
@@ -126,14 +118,15 @@ class Reader final : public CollectionPusher {
     return StatusCode::SUCCESS;
   }
 
+  // The IOSvc takes care of reading and passing the data
+  // By convention the Frame is pushed to the store
+  // so that it's deleted at the right time
   std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>> operator()() const override {
-    info() << "Reader::operator()" << endmsg;
     auto val = iosvc->next();
 
     auto eds = eventSvc().as<IDataProviderSvc>();
     auto frame = std::move(std::get<2>(val));
 
-    // We'll hand the ownership of 
     auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
     auto code = eds->registerObject("/Event/_Frame", tmp);
 
diff --git a/k4FWCore/components/SaveFile.cpp b/k4FWCore/components/SaveFile.cpp
index 135b1ece..e0398e02 100644
--- a/k4FWCore/components/SaveFile.cpp
+++ b/k4FWCore/components/SaveFile.cpp
@@ -20,16 +20,16 @@
 #include "GaudiKernel/AnyDataWrapper.h"
 #include "GaudiKernel/IDataManagerSvc.h"
 #include "GaudiKernel/IDataProviderSvc.h"
-#include "GaudiKernel/IHiveWhiteBoard.h"
 #include "GaudiKernel/SmartDataPtr.h"
 #include "GaudiKernel/StatusCode.h"
 
 #include "podio/Frame.h"
-
 #include "edm4hep/MCParticleCollection.h"
 
 #include "IIOSvc.h"
 
+#include "k4FWCore/DataWrapper.h"
+
 #include <memory>
 #include <thread>
 
@@ -69,7 +69,6 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
   void getOutputCollections(IRegistry* pObj) const {
     SmartIF<IDataManagerSvc> m_mgr;
     m_mgr = eventSvc();
-    info() << "pObj = " << pObj << endmsg;
     if (!pObj) {
       error() << "Failed to retrieve root object" << endmsg;
       return;
@@ -135,17 +134,26 @@ class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
         error() << "Failed to unregister collection " << coll << endmsg;
         return;
       }
+      info() << "Retrieved collection " << coll << endmsg;
       const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
       if (!collection) {
-        error() << "Failed to cast collection " << coll << endmsg;
-        return;
+
+        // Check the case when the data has been produced using the old DataHandle
+        const auto old_collection = dynamic_cast<DataWrapper<podio::CollectionBase>*>(p);
+        if (!old_collection) {
+          error() << "Failed to cast collection " << coll << endmsg;
+          return;
+        }
+        else {
+          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->getData()));
+          ptr->getData().put(std::move(uptr), coll);
+        }
+
       }
       else {
-        // info() << "collection = " << collection << endmsg;
-        // info() << "Collection " << coll << " has " << collection->getData()->size() << " elements" << endmsg;
+        std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
+        ptr->getData().put(std::move(uptr), coll);
       }
-      std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
-      ptr->getData().put(std::move(uptr), coll);
     }
 
     // ptr->addRef();
diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h
index bea0ff5a..53fc7c67 100644
--- a/k4FWCore/include/k4FWCore/DataHandle.h
+++ b/k4FWCore/include/k4FWCore/DataHandle.h
@@ -29,6 +29,7 @@
 
 #include "TTree.h"
 
+#include <GaudiKernel/AnyDataWrapper.h>
 #include <type_traits>
 
 /**
@@ -131,6 +132,11 @@ template <typename T> const T* DataHandle<T>::get() {
       DataWrapper<podio::CollectionBase>* tmp = static_cast<DataWrapper<podio::CollectionBase>*>(dataObjectp);
       return reinterpret_cast<const T*>(tmp->collectionBase());
     } else {
+      // When a functional has pushed an std::shared_ptr<podio::CollectionBase> into the store
+      auto ptr = static_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(dataObjectp)->getData();
+      if (ptr) {
+        return reinterpret_cast<const T*>(ptr.get());
+      }
       std::string errorMsg("The type provided for " + DataObjectHandle<DataWrapper<T>>::pythonRepr() +
                            " is different from the one of the object in the store.");
       throw GaudiException(errorMsg, "wrong product type", StatusCode::FAILURE);
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFile.py b/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
index c439eb98..2f929c3a 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
@@ -47,7 +47,7 @@
 
 mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
                EvtSel="NONE",
-               EvtMax=10,
+               EvtMax=-1,
                ExtSvc=[EventDataSvc("EventDataSvc")],
                OutputLevel=INFO,
                )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
index 053ba6e7..09bd15f1 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
@@ -48,7 +48,7 @@
 
 mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
                EvtSel="NONE",
-               EvtMax=10,
+               EvtMax=-1,
                ExtSvc=[EventDataSvc("EventDataSvc")],
                OutputLevel=INFO,
                )
diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py
index 69396ee6..b0b40d88 100644
--- a/test/k4FWCoreTest/options/runEventHeaderCheck.py
+++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py
@@ -24,8 +24,8 @@
 from Configurables import EventDataSvc, IOSvc, Reader
 
 svc = IOSvc("IOSvc")
-svc.FileNames = ["eventHeader.root"]
-svc.CollectionNames = ['MCParticles']
+svc.input = ["eventHeader.root"]
+# svc.CollectionNames = ['MCParticles']
 
 reader = Reader("Reader")
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index e9504908..5d13a3a2 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -61,12 +61,10 @@ struct ExampleFunctionalProducerMultiple final
     floatVector.push_back(m_event);
 
     auto particles = edm4hep::MCParticleCollection();
-    auto particle  = particles.create();
-    auto& p4        = particle.getMomentum();
-    p4.x            = m_magicNumberOffset + m_event + 5;
-    p4.y            = m_magicNumberOffset + 6;
-    p4.z            = m_magicNumberOffset + 7;
-    particle.setMass(m_magicNumberOffset + m_event + 8);
+    edm4hep::Vector3d v{0, 0, 0};
+    edm4hep::Vector3f vv{3, 0, 0};
+    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, vv);
+    particles.create(2, 3, 4, 5.f, 6.f, 7.f);
 
     auto simTrackerHits = edm4hep::SimTrackerHitCollection();
     auto hit            = simTrackerHits.create();

From 9428e4c3c64c02b3d9d8e2e4a398399cc7b68da9 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sun, 31 Dec 2023 13:36:04 +0100
Subject: [PATCH 006/127] Add another test

---
 test/k4FWCoreTest/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 2ee10bdc..ae055d15 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -110,5 +110,7 @@ add_test_with_env(FunctionalsMultipleMemory options/ExampleFunctionalsMultipleMe
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
 add_test_with_env(FunctionalsFile options/ExampleFunctionalsFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
+add_test_with_env(FunctionalsFile_toolong -n 999 options/ExampleFunctionalsFile.py PROPERTIES DEPENDS ExampleFunctionalProducer PASS_REGULAR_EXPRESSION
+                  "Application Manager Terminated successfully with a user requested ScheduledStop")
 add_test_with_env(FunctionalsMultipleFile options/ExampleFunctionalsFileMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)

From 5e7368c1b68ff7b3e187deb65671efb2263fe588 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 18 Jan 2024 13:47:52 +0100
Subject: [PATCH 007/127] Add changes to make multithreading reading and
 writing work

---
 k4FWCore/components/IIOSvc.h                  |   1 +
 k4FWCore/components/IOSvc.cpp                 |  53 +++-
 k4FWCore/components/IOSvc.h                   |   5 +-
 k4FWCore/components/Reader.cpp                |   8 +-
 k4FWCore/components/SaveFile.cpp              | 165 ------------
 k4FWCore/components/Writer.cpp                | 248 ++++++++++++++++++
 k4FWCore/include/k4FWCore/Consumer.h          |  33 ++-
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |   2 +
 k4FWCore/include/k4FWCore/Producer.h          |  33 ++-
 test/k4FWCoreTest/CMakeLists.txt              |  25 +-
 .../options/AvalancheSchedulerSimpleTest.py   |  21 +-
 test/k4FWCoreTest/options/CheckOutputFiles.py |  14 +
 ...ionalsFile.py => ExampleFunctionalFile.py} |  12 +-
 ...le.py => ExampleFunctionalFileMultiple.py} |  17 +-
 .../options/ExampleFunctionalMTFile.py        |  73 ++++++
 .../options/ExampleFunctionalMTMemory.py      |  60 +++++
 ...lsMemory.py => ExampleFunctionalMemory.py} |   0
 ....py => ExampleFunctionalMultipleMemory.py} |   0
 .../ExampleFunctionalOutputCommands.py        |  50 ++++
 .../options/ReadManyCollections.py            |  49 ++++
 test/k4FWCoreTest/options/SimpleTest.py       |  79 ++++++
 .../ExampleFunctionalTransformer.cpp          |   9 +-
 22 files changed, 713 insertions(+), 244 deletions(-)
 delete mode 100644 k4FWCore/components/SaveFile.cpp
 create mode 100644 k4FWCore/components/Writer.cpp
 create mode 100644 test/k4FWCoreTest/options/CheckOutputFiles.py
 rename test/k4FWCoreTest/options/{ExampleFunctionalsFile.py => ExampleFunctionalFile.py} (85%)
 rename test/k4FWCoreTest/options/{ExampleFunctionalsFileMultiple.py => ExampleFunctionalFileMultiple.py} (74%)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
 rename test/k4FWCoreTest/options/{ExampleFunctionalsMemory.py => ExampleFunctionalMemory.py} (100%)
 rename test/k4FWCoreTest/options/{ExampleFunctionalsMultipleMemory.py => ExampleFunctionalMultipleMemory.py} (100%)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
 create mode 100644 test/k4FWCoreTest/options/ReadManyCollections.py
 create mode 100644 test/k4FWCoreTest/options/SimpleTest.py

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 2f7ff924..367fd044 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -39,4 +39,5 @@ class IIOSvc : virtual public IInterface {
   virtual std::shared_ptr<podio::ROOTFrameWriter> getWriter() = 0;
   virtual void deleteWriter() = 0;
   virtual void deleteReader() = 0;
+  virtual bool writeCollection( const std::string& collName) = 0;
 };
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index f0c0934c..8ec1b7d3 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -35,7 +35,12 @@
 StatusCode IOSvc::initialize() {
   if (!m_readingFileNames.empty()) {
     m_reader = std::make_unique<podio::ROOTFrameReader>();
-    m_reader->openFiles(m_readingFileNames);
+    try {
+      m_reader->openFiles(m_readingFileNames);
+    } catch (std::runtime_error& e) {
+      error() << "Error when opening files: " << e.what() << endmsg;
+      throw e;
+    }
     m_entries = m_reader->getEntries(podio::Category::Event);
   }
 
@@ -46,17 +51,25 @@ StatusCode IOSvc::initialize() {
 
   m_dataSvc = service("EventDataSvc");
 
+  m_hiveWhiteBoard = service("EventDataSvc");
+
   return Service::initialize();
 }
 
 StatusCode IOSvc::finalize() { return Service::finalize(); }
 
 std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> IOSvc::next() {
-  info() << "IOSvc::next()" << endmsg;
+
   podio::Frame frame;
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
-    frame = podio::Frame(std::move(m_reader->readEntry(podio::Category::Event, m_nextEntry)));
+    info() << "m_nextEntry = " << m_nextEntry << " m_entries = " << m_entries << endmsg;
+    if (m_nextEntry < m_entries) {
+      frame = podio::Frame(std::move(m_reader->readEntry(podio::Category::Event, m_nextEntry)));
+    }
+    else {
+      return std::make_tuple(std::vector<std::shared_ptr<podio::CollectionBase>>(), std::vector<std::string>(), std::move(frame));
+    }
     m_nextEntry++;
     if (m_collectionNames.empty()) {
         m_collectionNames = frame.getAvailableCollections();
@@ -64,9 +77,14 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
   }
 
   if (m_nextEntry >= m_entries) {
-    IEventProcessor* eventProcessor;
-    StatusCode sc = service("ApplicationMgr", eventProcessor);
-    sc = eventProcessor->stopRun();
+  // if (true) {
+    auto ep = serviceLocator()->as<IEventProcessor>();
+    StatusCode sc = ep->stopRun();
+    if (sc.isFailure()) {
+      error() << "Error when stopping run" << endmsg;
+      throw GaudiException("Error when stopping run", name(), StatusCode::FAILURE);
+    }
+    info() << "m_nextEntry = " << m_nextEntry << " m_entries = " << m_entries << endmsg;
   }
 
   std::vector<std::shared_ptr<podio::CollectionBase>> collections;
@@ -83,9 +101,25 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
 // that means it hasn't been written so the collections inside the Frame
 // should be removed so that they are deleted when the Frame is deleted
 void IOSvc::handle( const Incident& incident ) {
+
+  StatusCode code;
+  if (m_hiveWhiteBoard) {
+    if (!incident.context().valid()) {
+      info() << "No context found in IOSvc" << endmsg;
+      return;
+    }
+    info() << "Setting store to " << incident.context().slot() << endmsg;
+    code = m_hiveWhiteBoard->selectStore(incident.context().slot());
+    if (code.isFailure()) {
+      error() << "Error when setting store" << endmsg;
+      throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
+    }
+  }
+  info() << "IOSvc::handle()" << endmsg;
   DataObject *p;
-  auto code = m_dataSvc->retrieveObject("/Event/_Frame", p);
+  code = m_dataSvc->retrieveObject("/Event/_Frame", p);
   if (code.isFailure()) {
+    info() << "No frame found" << endmsg;
     return;
   }
 
@@ -94,6 +128,7 @@ void IOSvc::handle( const Incident& incident ) {
     DataObject *collPtr;
     code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);
     if (code.isSuccess()) {
+      info() << "Removing collection: " << coll << endmsg;
       code = m_dataSvc->unregisterObject(collPtr);
     }
     // else {
@@ -110,4 +145,8 @@ void IOSvc::setReadingFileNames(const std::vector<std::string>& names) {
   m_readingFileNames = names;
 }
 
+bool IOSvc::writeCollection(const std::string& collName) {
+  return m_switch.isOn(collName);
+}
+
 DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 310572a3..5734176d 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -24,6 +24,7 @@
 #include "GaudiKernel/IIncidentListener.h"
 #include "GaudiKernel/IIncidentSvc.h"
 #include "GaudiKernel/IDataProviderSvc.h"
+#include "GaudiKernel/IHiveWhiteBoard.h"
 
 #include "podio/ROOTFrameReader.h"
 #include "podio/ROOTFrameWriter.h"
@@ -73,7 +74,6 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
   Gaudi::Property<std::string> m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"};
 
-  /// lock for handling the change of buffer
   std::mutex m_changeBufferLock;
 
   KeepDropSwitch m_switch;
@@ -101,10 +101,13 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   SmartIF<IDataProviderSvc> m_dataSvc;
   SmartIF<IIncidentSvc> m_incidentSvc;
+  SmartIF<IHiveWhiteBoard> m_hiveWhiteBoard;
   void handle(const Incident& incident) override;
 
   int m_entries{0};
   int m_nextEntry{0};
+  
+  bool writeCollection(const std::string& collName) override;
 };
 
 #endif
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index c212d1a0..f8aaa640 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -28,6 +28,8 @@
 #include "podio/CollectionBase.h"
 #include "podio/Frame.h"
 
+#include "k4FWCore/FunctionalUtils.h"
+
 #include <memory>
 
 template <typename Container>
@@ -62,7 +64,7 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
         // }
         for (unsigned i = 0; i != outColls.size(); ++i) {
           auto objectp = std::make_unique<AnyDataWrapper<Out>>( std::move( outColls[i] ) );
-          if ( auto sc = datasvc->registerObject( outputLocations[i], objectp.get() ); sc.isFailure() ) {
+          if ( auto sc = m_dataSvc->registerObject( outputLocations[i], objectp.get() ); sc.isFailure() ) {
           }
           // The store has the ownership so we shouldn't delete the object
           auto ptr = objectp.release();
@@ -83,7 +85,7 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
     // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
     // TODO/FIXME: replace vector of DataObjID property + call-back with a
     //             vector<handle> property ... as soon as declareProperty can deal with that.
-    ServiceHandle<IDataProviderSvc> datasvc{this, "EventDataSvc", "EventDataSvc"};
+    ServiceHandle<IDataProviderSvc> m_dataSvc{this, "EventDataSvc", "EventDataSvc"};
 
   };
 
@@ -128,7 +130,7 @@ class Reader final : public CollectionPusher {
     auto frame = std::move(std::get<2>(val));
 
     auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
-    auto code = eds->registerObject("/Event/_Frame", tmp);
+    auto code = eds->registerObject("/Event" + k4FWCore::frameLocation, tmp);
 
     return std::make_tuple(std::get<0>(val), std::get<1>(val));
   }
diff --git a/k4FWCore/components/SaveFile.cpp b/k4FWCore/components/SaveFile.cpp
deleted file mode 100644
index e0398e02..00000000
--- a/k4FWCore/components/SaveFile.cpp
+++ /dev/null
@@ -1,165 +0,0 @@
-/* * Copyright (c) 2014-2023 Key4hep-Project.
- *
- * This file is part of Key4hep.
- * See https://key4hep.github.io/key4hep-doc/ for further info.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Gaudi/Functional/Consumer.h"
-#include "GaudiKernel/AnyDataWrapper.h"
-#include "GaudiKernel/IDataManagerSvc.h"
-#include "GaudiKernel/IDataProviderSvc.h"
-#include "GaudiKernel/SmartDataPtr.h"
-#include "GaudiKernel/StatusCode.h"
-
-#include "podio/Frame.h"
-#include "edm4hep/MCParticleCollection.h"
-
-#include "IIOSvc.h"
-
-#include "k4FWCore/DataWrapper.h"
-
-#include <memory>
-#include <thread>
-
-class SaveToFile final : public Gaudi::Functional::Consumer<void()> {
-public:
-  SaveToFile(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) {
-  }
-
-  mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
-  mutable std::set<std::string> m_availableCollections;
-
-  SmartIF<IIOSvc> iosvc;
-  SmartIF<IDataProviderSvc> datasvc;
-  mutable bool m_first {true};
-
-  StatusCode initialize() override {
-    iosvc = service("IOSvc", true);
-    if (!iosvc) {
-      error() << "Unable to locate IIOSvc interface" << endmsg;
-      return StatusCode::FAILURE;
-    }
-
-    datasvc = service("EventDataSvc", true);
-    if (!datasvc) {
-      error() << "Unable to locate IDataSvc interface" << endmsg;
-      return StatusCode::FAILURE;
-    }
-
-    return StatusCode::SUCCESS;
-  }
-
-  StatusCode finalize() override {
-    iosvc->deleteWriter();
-    return StatusCode::SUCCESS;
-  }
-
-  void getOutputCollections(IRegistry* pObj) const {
-    SmartIF<IDataManagerSvc> m_mgr;
-    m_mgr = eventSvc();
-    if (!pObj) {
-      error() << "Failed to retrieve root object" << endmsg;
-      return;
-    }
-    auto mgr = eventSvc().as<IDataManagerSvc>();
-    if (!mgr) {
-      error() << "Failed to retrieve IDataManagerSvc" << endmsg;
-      return;
-    }
-    std::vector<IRegistry*> leaves;
-    StatusCode sc = m_mgr->objectLeaves(pObj, leaves);
-    if (!sc.isSuccess()) {
-      error() << "Failed to retrieve object leaves" << endmsg;
-      return;
-    }
-    for (auto& pReg : leaves) {
-      if (pReg->name() == "/_Frame") {
-        continue;
-      }
-      // info() << "Found leaf: " << pReg->name() << endmsg;
-      /// We are only interested in leaves with an object
-      // if (!pReg->address() || !pReg->object()) {
-      //   info() << "Leaf " << pReg->name() << " has no object" << endmsg;
-      //   continue;
-      // }
-      m_availableCollections.insert(pReg->name().substr(1, pReg->name().size() - 1));
-      const std::string& id = pReg->identifier();
-    }
-  }
-
-  void operator()() const override {
-    if (m_first) {
-      SmartDataPtr<DataObject> root(eventSvc(), "/Event");
-      if (!root) {
-        info() << "Failed to retrieve root object /Event" << endmsg;
-        return;
-      }
-      getOutputCollections(root->registry());
-
-      m_first = false;
-    }
-
-    DataObject *p;
-    auto code = datasvc->retrieveObject("/Event/Frame", p);
-    AnyDataWrapper<podio::Frame>* ptr;
-    if (code.isSuccess()) {
-      code = datasvc->unregisterObject(p);
-      ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
-    }
-    else {
-      ptr = new AnyDataWrapper<podio::Frame>(podio::Frame());
-    }
-    for (auto& coll : m_availableCollections) {
-      DataObject* p;
-      auto code = datasvc->retrieveObject("/Event/" + coll, p);
-      if (code.isFailure()) {
-        error() << "Failed to retrieve collection " << coll << endmsg;
-        return;
-      }
-      // We take ownership back from the store
-      code = datasvc->unregisterObject(p);
-      if (code.isFailure()) {
-        error() << "Failed to unregister collection " << coll << endmsg;
-        return;
-      }
-      info() << "Retrieved collection " << coll << endmsg;
-      const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-      if (!collection) {
-
-        // Check the case when the data has been produced using the old DataHandle
-        const auto old_collection = dynamic_cast<DataWrapper<podio::CollectionBase>*>(p);
-        if (!old_collection) {
-          error() << "Failed to cast collection " << coll << endmsg;
-          return;
-        }
-        else {
-          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->getData()));
-          ptr->getData().put(std::move(uptr), coll);
-        }
-
-      }
-      else {
-        std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
-        ptr->getData().put(std::move(uptr), coll);
-      }
-    }
-
-    // ptr->addRef();
-    iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event);
-
-  }
-};
-
-DECLARE_COMPONENT(SaveToFile)
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
new file mode 100644
index 00000000..c04a2fa4
--- /dev/null
+++ b/k4FWCore/components/Writer.cpp
@@ -0,0 +1,248 @@
+/* * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Functional/Consumer.h"
+#include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/IDataManagerSvc.h"
+#include "GaudiKernel/IDataProviderSvc.h"
+#include "GaudiKernel/SmartDataPtr.h"
+#include "GaudiKernel/StatusCode.h"
+
+#include "podio/Frame.h"
+#include "edm4hep/MCParticleCollection.h"
+
+#include "IIOSvc.h"
+
+#include "k4FWCore/DataWrapper.h"
+#include "k4FWCore/FunctionalUtils.h"
+
+#include <GaudiKernel/IHiveWhiteBoard.h>
+#include <memory>
+#include <thread>
+
+class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&)> {
+public:
+  Writer(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) {
+
+    // Non-reeentrant algorithms have a cardinality of 1
+    setProperty("Cardinality", 1).ignore();
+
+  }
+
+  mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
+  mutable std::set<std::string> m_availableCollections;
+  mutable std::vector<std::string> m_collectionsToAdd;
+  mutable std::vector<std::string> m_collectionsRemaining;
+  mutable std::vector<std::string> m_collectionsToSave;
+
+  mutable std::mutex m_mutex;
+
+  ServiceHandle<IIOSvc> iosvc{this, "IOSvc", "IOSvc"};
+  ServiceHandle<IHiveWhiteBoard> m_hiveWhiteBoard{this, "EventDataSvc", "EventDataSvc"};
+  SmartIF<IDataProviderSvc> m_dataSvc;
+  mutable bool m_first {true};
+
+  StatusCode initialize() override {
+    if (!iosvc.isValid()) {
+      error() << "Unable to locate IIOSvc interface" << endmsg;
+      return StatusCode::FAILURE;
+    }
+
+    m_dataSvc = service("EventDataSvc", true);
+    if (!m_dataSvc) {
+      error() << "Unable to locate IDataSvc interface" << endmsg;
+      return StatusCode::FAILURE;
+    }
+
+    return StatusCode::SUCCESS;
+  }
+
+  StatusCode finalize() override {
+    iosvc->deleteWriter();
+    return StatusCode::SUCCESS;
+  }
+
+  void getOutputCollections() const {
+    SmartIF<IDataManagerSvc> m_mgr;
+    m_mgr = eventSvc();
+
+    SmartDataPtr<DataObject> root(eventSvc(), "/Event");
+    if (!root) {
+      info() << "Failed to retrieve root object /Event" << endmsg;
+      return;
+    }
+
+    auto pObj = root->registry();
+    if (!pObj) {
+      error() << "Failed to retrieve root object" << endmsg;
+      return;
+    }
+    auto mgr = eventSvc().as<IDataManagerSvc>();
+    if (!mgr) {
+      error() << "Failed to retrieve IDataManagerSvc" << endmsg;
+      return;
+    }
+    std::vector<IRegistry*> leaves;
+    StatusCode sc = m_mgr->objectLeaves(pObj, leaves);
+    if (!sc.isSuccess()) {
+      error() << "Failed to retrieve object leaves" << endmsg;
+      return;
+    }
+    for (auto& pReg : leaves) {
+      if (pReg->name() == k4FWCore::frameLocation) {
+        continue;
+      }
+      // info() << "Found leaf: " << pReg->name() << endmsg;
+      /// We are only interested in leaves with an object
+      // if (!pReg->address() || !pReg->object()) {
+      //   info() << "Leaf " << pReg->name() << " has no object" << endmsg;
+      //   continue;
+      // }
+      m_availableCollections.insert(pReg->name().substr(1, pReg->name().size() - 1));
+      const std::string& id = pReg->identifier();
+    }
+  }
+
+  // void operator()() const override {
+  void operator()(const EventContext& ctx) const override {
+
+    // It seems that even when setting Cardinality to 1,
+    // more than one instance is created
+    std::scoped_lock<std::mutex> lock(m_mutex);
+
+    StatusCode code;
+
+    if (m_hiveWhiteBoard) {
+      // It's never set to valid but it has the slot information
+      // if (ctx.valid()) {
+      //   info() << "No context found in Writer" << endmsg;
+      //   return;
+      // }
+      info() << "Setting store to " << ctx.slot() << endmsg;
+      code = m_hiveWhiteBoard->selectStore(ctx.slot());
+      if (code.isFailure()) {
+        error() << "Error when setting store" << endmsg;
+        throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
+      }
+    }
+
+    DataObject *p;
+    code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
+    AnyDataWrapper<podio::Frame>* ptr;
+    // This is the case when we are reading from a file
+    if (code.isSuccess()) {
+      code = m_dataSvc->unregisterObject(p);
+      ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+    }
+    // This is the case when no reading is being done
+    // needs to be fixed? (new without delete)
+    else {
+      ptr = new AnyDataWrapper<podio::Frame>(podio::Frame());
+    }
+
+    if (m_first) {
+
+      // Assume all the output collections are the same for all events
+      // and cache them
+      getOutputCollections();
+      for (const auto& coll : m_availableCollections) {
+        if (iosvc->writeCollection(coll)) {
+          m_collectionsToSave.push_back(coll);
+          const auto& frameCollections = ptr->getData().getAvailableCollections();
+          if (std::find(frameCollections.begin(), frameCollections.end(), coll) == frameCollections.end()) {
+            m_collectionsToAdd.push_back(coll);
+          }
+          else {
+            m_collectionsRemaining.push_back(coll);
+          }
+        }
+      }
+
+      m_first = false;
+    }
+
+    for (auto& coll : ptr->getData().getAvailableCollections()) {
+      info() << "Available collection " << coll << endmsg;
+    }
+
+    for (auto& coll : m_collectionsToAdd) {
+      info() << "Saving collection " << coll << endmsg;
+    }
+   
+    for (auto& coll : m_collectionsRemaining) {
+      info() << "Collection " << coll << " already saved" << endmsg;
+    }
+
+
+    for (auto& coll : ptr->getData().getAvailableCollections()) {
+      DataObject* storeCollection;
+      code = m_dataSvc->retrieveObject("/Event/" + coll, storeCollection);
+      if (code.isFailure()) {
+        error() << "Failed to retrieve collection " << coll << endmsg;
+        return;
+      }
+      // We take ownership back from the store
+      code = m_dataSvc->unregisterObject(storeCollection);
+      if (code.isFailure()) {
+        error() << "Failed to unregister collection " << coll << endmsg;
+        return;
+      }
+    }
+
+    for (auto& coll : m_collectionsToAdd) {
+      DataObject* storeCollection;
+      code = m_dataSvc->retrieveObject("/Event/" + coll, storeCollection);
+      if (code.isFailure()) {
+        error() << "Failed to retrieve collection " << coll << endmsg;
+        return;
+      }
+      // We take ownership back from the store
+      code = m_dataSvc->unregisterObject(storeCollection);
+      if (code.isFailure()) {
+        error() << "Failed to unregister collection " << coll << endmsg;
+        return;
+      }
+      info() << "Retrieved collection " << coll << endmsg;
+      const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(storeCollection);
+      if (!collection) {
+
+        // Check the case when the data has been produced using the old DataHandle
+        const auto old_collection = dynamic_cast<DataWrapper<podio::CollectionBase>*>(storeCollection);
+        if (!old_collection) {
+          error() << "Failed to cast collection " << coll << endmsg;
+          return;
+        }
+        else {
+          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->getData()));
+          ptr->getData().put(std::move(uptr), coll);
+        }
+
+      }
+      else {
+        std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
+        ptr->getData().put(std::move(uptr), coll);
+      }
+    }
+
+    info() << "Writing frame, with time: " << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count() << endmsg;
+    iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
+
+  }
+};
+
+DECLARE_COMPONENT(Writer)
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index ff78ef0c..fb659448 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -1,14 +1,23 @@
-/***********************************************************************************\
-* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
-*                                                                                   *
-* This software is distributed under the terms of the Apache version 2 licence,     *
-* copied verbatim in the file "LICENSE".                                            *
-*                                                                                   *
-* In applying this licence, CERN does not waive the privileges and immunities       *
-* granted to it by virtue of its status as an Intergovernmental Organization        *
-* or submit itself to any jurisdiction.                                             *
-\***********************************************************************************/
-#pragma once
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_CONSUMER_H
+#define FWCORE_CONSUMER_H
 
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
@@ -58,3 +67,5 @@ template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useD
 using Consumer = details::Consumer<Signature, Traits_>;
 
 } // namespace k4FWCore
+
+#endif // FWCORE_CONSUMER_H
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 1e61b1c2..511e3bb7 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -10,6 +10,8 @@
 
 namespace k4FWCore {
 
+static const std::string frameLocation = "/_Frame";
+
 namespace details {
 
   template <typename T, typename P, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, P>, int> = 0>
diff --git a/k4FWCore/include/k4FWCore/Producer.h b/k4FWCore/include/k4FWCore/Producer.h
index efa50c95..1b0da23e 100644
--- a/k4FWCore/include/k4FWCore/Producer.h
+++ b/k4FWCore/include/k4FWCore/Producer.h
@@ -1,14 +1,23 @@
-/***********************************************************************************\
-* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
-*                                                                                   *
-* This software is distributed under the terms of the Apache version 2 licence,     *
-* copied verbatim in the file "LICENSE".                                            *
-*                                                                                   *
-* In applying this licence, CERN does not waive the privileges and immunities       *
-* granted to it by virtue of its status as an Intergovernmental Organization        *
-* or submit itself to any jurisdiction.                                             *
-\***********************************************************************************/
-#pragma once
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_PRODUCER_H
+#define FWCORE_PRODUCER_H
 
 #include "FunctionalUtils.h"
 #include "Gaudi/Functional/details.h"
@@ -49,3 +58,5 @@ namespace k4FWCore {
   using Producer = details::Producer<Signature, Traits_>;
 
 } // namespace k4FWCore
+
+#endif // FWCORE_PRODUCER_H
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index ae055d15..faee0ce9 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -49,8 +49,8 @@ function(add_test_with_env testname)
     endif()
   endforeach()
   add_test(NAME ${testname}
-           WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-           COMMAND ${K4RUN} ${TEST_ARGUMENTS})
+           # WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+           COMMAND ${K4RUN} ${CMAKE_CURRENT_LIST_DIR}/${TEST_ARGUMENTS})
   if(TEST_PROPERTIES_FOUND)
     set_tests_properties(${testname} PROPERTIES ${TEST_PROPERTIES})
   endif()
@@ -62,7 +62,7 @@ add_test_with_env(CreateExampleEventDataInDirectory options/createExampleEventDa
 
 add_test_with_env(CheckExampleEventData options/checkExampleEventData.py PROPERTIES DEPENDS CreateExampleEventData)
 add_test_with_env(CheckExampleEventData_noCollections options/checkExampleEventData.py --collections= PROPERTIES DEPENDS CreateExampleEventData)
-add_test_with_env(CheckExampleEventData_toolong -n 999 options/checkExampleEventData.py PROPERTIES PASS_REGULAR_EXPRESSION
+add_test_with_env(CheckExampleEventData_toolong options/checkExampleEventData.py -n 999 PROPERTIES PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop" DEPENDS CreateExampleEventData)
 add_test_with_env(CheckExampleEventData_unbounded options/checkExampleEventData.py -n -1 PROPERTIES PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop" DEPENDS CreateExampleEventData)
@@ -99,18 +99,25 @@ add_test_with_env(EventHeaderCheck options/runEventHeaderCheck.py PROPERTIES DEP
 add_test(NAME TestExec WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND python options/TestExec.py)
 set_test_env(TestExec)
 
-add_test_with_env(Testk4runNoArguments PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run <options_file.py>, use --help to get a complete list of arguments")
+add_test(NAME Testk4runNoArgumentsHelp COMMAND ${K4RUN})
+set_tests_properties(Testk4runNoArgumentsHelp PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run <options_file.py>, use --help to get a complete list of arguments")
 add_test_with_env(Testk4runCustomArguments options/TestArgs.py --foo=42 PROPERTIES PASS_REGULAR_EXPRESSION "The answer is 42")
 add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIES PASS_REGULAR_EXPRESSION " VERBOSE ")
 add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit")
 
 
-add_test_with_env(FunctionalsMemory options/ExampleFunctionalsMemory.py)
-add_test_with_env(FunctionalsMultipleMemory options/ExampleFunctionalsMultipleMemory.py)
+add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py)
+add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py)
+add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py)
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
-add_test_with_env(FunctionalsFile options/ExampleFunctionalsFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
-add_test_with_env(FunctionalsFile_toolong -n 999 options/ExampleFunctionalsFile.py PROPERTIES DEPENDS ExampleFunctionalProducer PASS_REGULAR_EXPRESSION
+add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
+add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
+add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS ExampleFunctionalProducer PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
-add_test_with_env(FunctionalsMultipleFile options/ExampleFunctionalsFileMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
+add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
+add_test_with_env(FunctionalOutputCommands outputCommands/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
+
+add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
+set_property(TEST FunctionalCheckFiles APPEND PROPERTY DEPENDS FunctionalFile)
diff --git a/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py b/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
index d7b84d5a..b7d42d8b 100644
--- a/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
+++ b/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
@@ -4,11 +4,11 @@
 from Configurables import AlgResourcePool, AvalancheSchedulerSvc
 from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard
 from Configurables import ApplicationMgr, Gaudi__Sequencer
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, SaveToFile, StoreSnifferAlg
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, Writer
 from Gaudi.Configuration import INFO, DEBUG, WARNING
 
-evtslots = 2
-threads = 2
+evtslots = 1
+threads = 1
 # -------------------------------------------------------------------------------
 
 # The configuration of the whiteboard ------------------------------------------
@@ -46,21 +46,22 @@
 
 
 svc = IOSvc("IOSvc")
-svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/sim1.root']
-svc.CollectionNames = ['MCParticle', 'EventHeader']
+# svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/output_k4test_exampledata_producer.root']
+svc.FileNames = ['/home/juanmi/Key4hep/Workarea/build/output_k4test_exampledata_producer.root']
+svc.CollectionNames = ['MCParticles']
 
 io = Reader("Reader",
-           OutputLocations=['MCParticles', 'EventHeader'])
+           OutputLocations=['MCParticles'])
 io.Cardinality = 1
 
-save = SaveToFile("SaveToFile")
-save.CollectionNames = ['MCParticles', 'EventHeader']
+save = Writer("Writer")
+save.CollectionNames = ['MCParticles']
 save.Cardinality = 1
 
-node = Gaudi__Sequencer("Node", Members=[io, save], Sequential=True, OutputLevel=INFO)
+node = Gaudi__Sequencer("Node", Members=[io, a1], Sequential=True, OutputLevel=INFO)
 
 app = ApplicationMgr(
-    EvtMax=2,
+    EvtMax=1e7,
     EvtSel="NONE",
     ExtSvc=[whiteboard],
     EventLoop=slimeventloopmgr,
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
new file mode 100644
index 00000000..dd893aa2
--- /dev/null
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -0,0 +1,14 @@
+import podio
+
+def check_collections(filename, names):
+    podio_reader = podio.root_io.Reader(filename)
+    for frame in podio_reader.get('events'):
+        available = set(frame.collections)
+        if available != set(names):
+            print(f'These collections should be in the frame but are not: {set(names) - available}')
+            print(f'These collections are in the frame but should not be: {available - set(names)}')
+            raise RuntimeError('Collections in frame do not match expected collections')
+
+check_collections('functional_transformer.root', ['MCParticles', 'NewMCParticles'])
+check_collections('functional_transformer_multiple.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits', 'Tracks', 'Counter', 'NewMCParticles'])
+check_collections('functional_transformer_multiple_output_commands.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits'])
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
similarity index 85%
rename from test/k4FWCoreTest/options/ExampleFunctionalsFile.py
rename to test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 2f929c3a..872e1e3f 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalsFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -24,17 +24,16 @@
 from Configurables import ExampleFunctionalTransformer
 from Configurables import ApplicationMgr
 from Configurables import EventDataSvc, IOSvc
-from Configurables import Reader, SaveToFile
-import podio
+from Configurables import Reader, Writer
 
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer.root']
-svc.output = 'output_k4test_exampledata_transformer.root'
+svc.output = 'functional_transformer.root'
 # svc.CollectionNames = ['MCParticles']
 
 reader = Reader("Reader")
 
-writer = SaveToFile("Writer")
+writer = Writer("Writer")
 
 # out = PodioOutput("out")
 # out.filename = "output_k4test_exampledata_transformer.root"
@@ -51,7 +50,6 @@
                ExtSvc=[EventDataSvc("EventDataSvc")],
                OutputLevel=INFO,
                )
-# podio_reader = podio.root_io.Reader('output_k4test_exampledata_transformer.root')
-# for frame in podio_reader.get('events'):
-#     frame.get('NewMCParticles')
+
+
     
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
similarity index 74%
rename from test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
rename to test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
index 09bd15f1..a6c2ad24 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalsFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
@@ -24,22 +24,15 @@
 from Configurables import ExampleFunctionalTransformerMultiple
 from Configurables import ApplicationMgr
 from Configurables import EventDataSvc, IOSvc
-from Configurables import Reader, SaveToFile
-import podio
+from Configurables import Reader, Writer
 
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer_multiple.root']
-svc.output = 'output_k4test_exampledata_transformer_multiple.root'
-# svc.CollectionNames = ['MCParticles']
+svc.output = 'functional_transformer_multiple.root'
 
 reader = Reader("Reader")
 
-writer = SaveToFile("Writer")
-
-# out = PodioOutput("out")
-# out.filename = "output_k4test_exampledata_transformer.root"
-# # The collections that we don't drop will also be present in the output file
-# out.outputCommands = ["drop MCParticles"]
+writer = Writer("Writer")
 
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
                                            # InputCollection="MCParticles",
@@ -52,7 +45,3 @@
                ExtSvc=[EventDataSvc("EventDataSvc")],
                OutputLevel=INFO,
                )
-# podio_reader = podio.root_io.Reader('output_k4test_exampledata_transformer.root')
-# for frame in podio_reader.get('events'):
-#     frame.get('NewMCParticles')
-    
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
new file mode 100644
index 00000000..cfdee9b1
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO, WARNING
+from Gaudi.Configuration import Gaudi__Sequencer
+from Configurables import ExampleFunctionalTransformer
+from Configurables import ApplicationMgr
+from Configurables import IOSvc
+from Configurables import Reader, Writer
+from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc
+
+evtslots = 2
+threads = 2
+
+whiteboard = HiveWhiteBoard("EventDataSvc",
+                            EventSlots=evtslots,
+                            ForceLeaves=True,
+                            )
+
+slimeventloopmgr = HiveSlimEventLoopMgr("HiveSlimEventLoopMgr",
+    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+)
+
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+
+svc = IOSvc("IOSvc")
+svc.input = ['output_k4test_exampledata_producer.root']
+svc.output = 'functional_transformerMT.root'
+# svc.CollectionNames = ['MCParticles']
+
+reader = Reader("Reader")
+
+writer = Writer("Writer")
+
+# out = PodioOutput("out")
+# out.filename = "output_k4test_exampledata_transformer.root"
+# # The collections that we don't drop will also be present in the output file
+# out.outputCommands = ["drop MCParticles"]
+
+transformer = ExampleFunctionalTransformer("Transformer",
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
+
+seq = Gaudi__Sequencer("Node", Members=[reader, transformer, writer], Sequential=True, OutputLevel=INFO)
+
+mgr = ApplicationMgr(TopAlg=[seq],
+               EvtSel="NONE",
+               EvtMax=-1,
+               ExtSvc=[whiteboard],
+               EventLoop=slimeventloopmgr,      
+               MessageSvcType="InertMessageSvc",
+               OutputLevel=INFO,
+               )
+    
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
new file mode 100644
index 00000000..4d2939b1
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO, WARNING
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalTransformer, ExampleFunctionalConsumer
+from Configurables import ApplicationMgr, HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc
+
+evtslots = 5
+threads = 3
+
+whiteboard = HiveWhiteBoard("EventDataSvc",
+                            EventSlots=evtslots,
+                            ForceLeaves=True,
+                            )
+
+slimeventloopmgr = HiveSlimEventLoopMgr(
+    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+)
+
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+
+transformer = ExampleFunctionalTransformer("Transformer",
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
+
+producer = ExampleFunctionalProducer("Producer",
+                                     OutputCollection="MCParticles")
+
+consumer = ExampleFunctionalConsumer("Consumer",
+                                     InputCollection="NewMCParticles",
+                                     Offset=10,
+                                     )
+
+
+ApplicationMgr(TopAlg=[producer, transformer, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               EventLoop=slimeventloopmgr,
+               ExtSvc=[whiteboard],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
similarity index 100%
rename from test/k4FWCoreTest/options/ExampleFunctionalsMemory.py
rename to test/k4FWCoreTest/options/ExampleFunctionalMemory.py
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
similarity index 100%
rename from test/k4FWCoreTest/options/ExampleFunctionalsMultipleMemory.py
rename to test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
new file mode 100644
index 00000000..882a8051
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformerMultiple
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc, IOSvc
+from Configurables import Reader, Writer
+
+svc = IOSvc("IOSvc")
+svc.input = ['output_k4test_exampledata_producer_multiple.root']
+svc.output = 'functional_transformer_multiple_output_commands.root'
+svc.outputCommands = ["drop Tracks",
+                      "drop Counter",
+                      "drop NewMCParticles",]
+
+reader = Reader("Reader")
+
+writer = Writer("Writer")
+
+transformer = ExampleFunctionalTransformerMultiple("Transformer",
+                                           # InputCollection="MCParticles",
+                                           # OutputCollection="NewMCParticles")
+                                                   )
+
+mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+               EvtSel="NONE",
+               EvtMax=-1,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ReadManyCollections.py b/test/k4FWCoreTest/options/ReadManyCollections.py
new file mode 100644
index 00000000..a2be9bed
--- /dev/null
+++ b/test/k4FWCoreTest/options/ReadManyCollections.py
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2014-2023 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example using a producer with a multiple outputs and saving that to a file
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducerMultiple
+from Configurables import ApplicationMgr
+from Configurables import k4DataSvc
+from Configurables import PodioOutput
+
+podioevent = k4DataSvc("EventDataSvc")
+
+out = PodioOutput("out")
+out.filename = "output_k4test_exampledata_producer_multiple.root"
+# Collections can be dropped
+# out.outputCommands = ["drop *"]
+
+producer = ExampleFunctionalProducerMultiple("ExampleFunctionalProducerMultiple",
+                                             OutputCollectionFloat="VectorFloat",
+                                             OutputCollectionParticles1="MCParticles1",
+                                             OutputCollectionParticles2="MCParticles2",
+                                             OutputCollectionSimTrackerHits="SimTrackerHits",
+                                             OutputCollectionTrackerHits="TrackerHits",
+                                             OutputCollectionTracks="Tracks",
+                                             ExampleInt=5)
+
+ApplicationMgr(TopAlg=[producer, out],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[k4DataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/SimpleTest.py b/test/k4FWCoreTest/options/SimpleTest.py
new file mode 100644
index 00000000..761958f6
--- /dev/null
+++ b/test/k4FWCoreTest/options/SimpleTest.py
@@ -0,0 +1,79 @@
+# Test that we can get reproducible random numbers in a multithreaded environment
+# given that the seed is the same (obtained, for example, with the UniqueIDGenSvc)
+
+from Configurables import AlgResourcePool, AvalancheSchedulerSvc
+from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard
+from Configurables import ApplicationMgr, Gaudi__Sequencer
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, Writer, EventDataSvc
+from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerMultiple
+from Gaudi.Configuration import INFO, DEBUG, WARNING
+
+evtslots = 1
+threads = 1
+# -------------------------------------------------------------------------------
+
+# The configuration of the whiteboard ------------------------------------------
+# It is useful to call it EventDataSvc to replace the usual data service with
+# the whiteboard transparently.
+
+# whiteboard = HiveWhiteBoard("EventDataSvc",
+#                             EventSlots=evtslots,
+#                             ForceLeaves=True,
+#                             )
+
+# Event Loop Manager -----------------------------------------------------------
+# It's called slim since it has less functionalities overall than the good-old
+# event loop manager. Here we just set its outputlevel to DEBUG.
+
+# slimeventloopmgr = HiveSlimEventLoopMgr(
+#     SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+# )
+
+# AvalancheScheduler -----------------------------------------------------------
+# We just decide how many algorithms in flight we want to have and how many
+# threads in the pool. The default value is -1, which is for TBB equivalent
+# to take over the whole machine.
+
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+
+# Algo Resource Pool -----------------------------------------------------------
+# Nothing special here, we just set the debug level.
+AlgResourcePool(OutputLevel=DEBUG)
+
+a1 = ExampleFunctionalProducer("FirstSingle")
+a2 = ExampleFunctionalConsumer("ConsumerSingle")
+b1 = ExampleFunctionalProducerMultiple("First")
+b2 = ExampleFunctionalConsumerMultiple("Consumer")
+# a2 = ExampleFunctionalProducer("Second", OutputCollection="MySecondCollection")
+# a3 = ExampleFunctionalProducer("Third", OutputCollection="MyThirdCollection")
+
+datasvc = EventDataSvc("EventDataSvc")
+
+
+svc = IOSvc("IOSvc")
+# svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/output_k4test_exampledata_producer.root']
+svc.FileNames = ['/home/juanmi/Key4hep/Workarea/build/output_k4test_exampledata_producer.root']
+svc.CollectionNames = ['MCParticles']
+
+io = Reader("Reader",
+           OutputLocations=['MCParticles'])
+io.Cardinality = 1
+
+save = Writer("Writer")
+save.CollectionNames = ['MCParticles']
+save.Cardinality = 1
+
+# node = Gaudi__Sequencer("Node", Members=[io, a1], Sequential=True, OutputLevel=INFO)
+
+app = ApplicationMgr(
+    EvtMax=10,
+    EvtSel="NONE",
+    ExtSvc=[datasvc],
+    TopAlg=[b1, b2],
+    # TopAlg=[io, a2],
+    MessageSvcType="InertMessageSvc",
+    OutputLevel=INFO,
+)
+
+
+print(app)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
index 05280b76..e3df13d0 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
@@ -27,12 +27,8 @@
 
 #include <string>
 
-// Which type of collection we are reading and writing
-using colltype_in  = edm4hep::MCParticleCollection;
-using colltype_out = edm4hep::MCParticleCollection;
-
 struct ExampleFunctionalTransformer final
-    : k4FWCore::Transformer<colltype_out(const colltype_in&)> {
+    : k4FWCore::Transformer<  edm4hep::MCParticleCollection(const edm4hep::MCParticleCollection&)> {
   ExampleFunctionalTransformer(const std::string& name, ISvcLocator* svcLoc)
       : Transformer(name, svcLoc, KeyValue("InputCollection", "MCParticles"),
                     KeyValue("OutputCollection", "NewMCParticles")) {}
@@ -40,7 +36,8 @@ struct ExampleFunctionalTransformer final
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as all pointers to collections
   // we get from the input
-  colltype_out operator()(const colltype_in& input) const override {
+  edm4hep::MCParticleCollection operator()(const edm4hep::MCParticleCollection& input) const override {
+    info() << "Transforming " << input.size() << " particles" << endmsg;
     auto coll_out = edm4hep::MCParticleCollection();
     for (const auto& particle : input) {
       auto new_particle = edm4hep::MutableMCParticle();

From 2cb92bf919ddb95c2da1e5e4439bc83bc3d71cb4 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sun, 4 Feb 2024 21:06:22 +0100
Subject: [PATCH 008/127] Add changes to read multiple collections

---
 CMakeLists.txt                                |  2 +
 k4FWCore/components/IOSvc.cpp                 |  1 -
 k4FWCore/components/IOSvc.h                   |  8 +-
 k4FWCore/components/Writer.cpp                | 93 +++++++++++--------
 k4FWCore/include/k4FWCore/Consumer.h          | 86 ++++++++++-------
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 68 +++++++++++---
 k4FWCore/include/k4FWCore/Transformer.h       | 56 ++++++-----
 test/k4FWCoreTest/CMakeLists.txt              | 11 ++-
 test/k4FWCoreTest/options/CheckOutputFiles.py |  1 +
 .../ExampleFunctionalConsumerMemory.py        |  2 +-
 ...ExampleFunctionalConsumerMultipleMemory.py |  2 +-
 ...pleFunctionalConsumerRuntimeCollections.py | 47 ++++++++++
 .../options/ExampleFunctionalFile.py          | 17 +---
 .../options/ExampleFunctionalFileMultiple.py  | 11 +--
 .../options/ExampleFunctionalMTFile.py        | 18 +---
 .../options/ExampleFunctionalMTMemory.py      |  3 +-
 .../ExampleFunctionalMultipleMemory.py        |  2 +-
 .../ExampleFunctionalOutputCommands.py        | 11 +--
 .../ExampleFunctionalProducerAbsolutePath.py  | 37 ++++++++
 ...pleFunctionalProducerRuntimeCollections.py | 49 ++++++++++
 .../components/ExampleFunctionalConsumer.cpp  |  1 +
 ...leFunctionalConsumerRuntimeCollections.cpp | 69 ++++++++++++++
 .../ExampleFunctionalProducerMultiple.cpp     |  3 +-
 ...leFunctionalProducerRuntimeCollections.cpp | 56 +++++++++++
 24 files changed, 481 insertions(+), 173 deletions(-)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c52ccdef..df46b4d3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -77,6 +77,8 @@ if(BUILD_TESTING)
   add_subdirectory(test/k4FWCoreTest)
 endif()
 
+add_subdirectory(python)
+
 option(ENABLE_CPACK "Whether or not to use cpack config" OFF)
 if(ENABLE_CPACK)
   include(cmake/${PROJECT_NAME}CPack.cmake)
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 8ec1b7d3..fb426c35 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -115,7 +115,6 @@ void IOSvc::handle( const Incident& incident ) {
       throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
     }
   }
-  info() << "IOSvc::handle()" << endmsg;
   DataObject *p;
   code = m_dataSvc->retrieveObject("/Event/_Frame", p);
   if (code.isFailure()) {
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 5734176d..cb040655 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -28,10 +28,8 @@
 
 #include "podio/ROOTFrameReader.h"
 #include "podio/ROOTFrameWriter.h"
-#include "podio/ROOTRNTupleReader.h"
-#include "podio/ROOTRNTupleWriter.h"
-#include "podio/IROOTFrameReader.h"
-#include "podio/IROOTFrameWriter.h"
+#include "podio/RNTupleReader.h"
+#include "podio/RNTupleWriter.h"
 
 #include "k4FWCore/KeepDropSwitch.h"
 
@@ -78,7 +76,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   KeepDropSwitch m_switch;
 
-  std::unique_ptr<podio::IROOTFrameReader> m_reader{nullptr};
+  std::unique_ptr<podio::ROOTFrameReader> m_reader{nullptr};
   std::shared_ptr<podio::ROOTFrameWriter> m_writer{nullptr};
 
   std::shared_ptr<podio::ROOTFrameWriter> getWriter() override {
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index c04a2fa4..b5e1d441 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -23,8 +23,8 @@
 #include "GaudiKernel/SmartDataPtr.h"
 #include "GaudiKernel/StatusCode.h"
 
-#include "podio/Frame.h"
 #include "edm4hep/MCParticleCollection.h"
+#include "podio/Frame.h"
 
 #include "IIOSvc.h"
 
@@ -38,24 +38,22 @@
 class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&)> {
 public:
   Writer(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) {
-
     // Non-reeentrant algorithms have a cardinality of 1
     setProperty("Cardinality", 1).ignore();
-
   }
 
   mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
-  mutable std::set<std::string> m_availableCollections;
-  mutable std::vector<std::string> m_collectionsToAdd;
-  mutable std::vector<std::string> m_collectionsRemaining;
-  mutable std::vector<std::string> m_collectionsToSave;
+  mutable std::set<std::string>                     m_availableCollections;
+  mutable std::vector<std::string>                  m_collectionsToAdd;
+  mutable std::vector<std::string>                  m_collectionsRemaining;
+  mutable std::vector<std::string>                  m_collectionsToSave;
 
   mutable std::mutex m_mutex;
 
-  ServiceHandle<IIOSvc> iosvc{this, "IOSvc", "IOSvc"};
+  ServiceHandle<IIOSvc>          iosvc{this, "IOSvc", "IOSvc"};
   ServiceHandle<IHiveWhiteBoard> m_hiveWhiteBoard{this, "EventDataSvc", "EventDataSvc"};
-  SmartIF<IDataProviderSvc> m_dataSvc;
-  mutable bool m_first {true};
+  SmartIF<IDataProviderSvc>      m_dataSvc;
+  mutable bool                   m_first{true};
 
   StatusCode initialize() override {
     if (!iosvc.isValid()) {
@@ -73,6 +71,42 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
   }
 
   StatusCode finalize() override {
+
+    podio::Frame config_metadata_frame;
+
+    //// prepare job options metadata ///////////////////////
+    // retrieve the configuration of the job
+    // and write it to file as vector of strings
+    std::vector<std::string> config_data;
+    for (const auto& per_property : Gaudi::svcLocator()->getOptsSvc().items()) {
+      std::stringstream config_stream;
+      // sample output:
+      // HepMCToEDMConverter.genparticles = "GenParticles";
+      // Note that quotes are added to all property values,
+      // which leads to problems with ints, lists, dicts and bools.
+      // For theses types, the quotes must be removed in postprocessing.
+      config_stream << std::get<0>(per_property) << " = \"" << std::get<1>(per_property) << "\";" << std::endl;
+      config_data.push_back(config_stream.str());
+    }
+    // Some default components are not captured by the job option service
+    // and have to be traversed like this. Note that Gaudi!577 will improve this.
+    for (const auto* name : {"NTupleSvc"}) {
+      std::stringstream config_stream;
+      auto svc = service<IProperty>(name);
+      if (!svc.isValid())
+        continue;
+      for (const auto* property : svc->getProperties()) {
+        config_stream << name << "." << property->name() << " = \"" << property->toString() << "\";" << std::endl;
+      }
+      config_data.push_back(config_stream.str());
+    }
+
+    config_metadata_frame.putParameter("gaudiConfigOptions", config_data);
+    if (auto env_key4hep_stack = std::getenv("KEY4HEP_STACK")) {
+      config_metadata_frame.putParameter("key4hepstack", env_key4hep_stack);
+    }
+    iosvc->getWriter()->writeFrame(config_metadata_frame, "configuration_metadata");
+
     iosvc->deleteWriter();
     return StatusCode::SUCCESS;
   }
@@ -98,7 +132,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       return;
     }
     std::vector<IRegistry*> leaves;
-    StatusCode sc = m_mgr->objectLeaves(pObj, leaves);
+    StatusCode              sc = m_mgr->objectLeaves(pObj, leaves);
     if (!sc.isSuccess()) {
       error() << "Failed to retrieve object leaves" << endmsg;
       return;
@@ -118,9 +152,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     }
   }
 
-  // void operator()() const override {
   void operator()(const EventContext& ctx) const override {
-
     // It seems that even when setting Cardinality to 1,
     // more than one instance is created
     std::scoped_lock<std::mutex> lock(m_mutex);
@@ -141,13 +173,13 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       }
     }
 
-    DataObject *p;
+    DataObject* p;
     code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
     AnyDataWrapper<podio::Frame>* ptr;
     // This is the case when we are reading from a file
     if (code.isSuccess()) {
       code = m_dataSvc->unregisterObject(p);
-      ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+      ptr  = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
     }
     // This is the case when no reading is being done
     // needs to be fixed? (new without delete)
@@ -156,7 +188,6 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     }
 
     if (m_first) {
-
       // Assume all the output collections are the same for all events
       // and cache them
       getOutputCollections();
@@ -166,8 +197,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
           const auto& frameCollections = ptr->getData().getAvailableCollections();
           if (std::find(frameCollections.begin(), frameCollections.end(), coll) == frameCollections.end()) {
             m_collectionsToAdd.push_back(coll);
-          }
-          else {
+          } else {
             m_collectionsRemaining.push_back(coll);
           }
         }
@@ -176,19 +206,6 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       m_first = false;
     }
 
-    for (auto& coll : ptr->getData().getAvailableCollections()) {
-      info() << "Available collection " << coll << endmsg;
-    }
-
-    for (auto& coll : m_collectionsToAdd) {
-      info() << "Saving collection " << coll << endmsg;
-    }
-   
-    for (auto& coll : m_collectionsRemaining) {
-      info() << "Collection " << coll << " already saved" << endmsg;
-    }
-
-
     for (auto& coll : ptr->getData().getAvailableCollections()) {
       DataObject* storeCollection;
       code = m_dataSvc->retrieveObject("/Event/" + coll, storeCollection);
@@ -217,31 +234,29 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
         error() << "Failed to unregister collection " << coll << endmsg;
         return;
       }
-      info() << "Retrieved collection " << coll << endmsg;
       const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(storeCollection);
       if (!collection) {
-
         // Check the case when the data has been produced using the old DataHandle
         const auto old_collection = dynamic_cast<DataWrapper<podio::CollectionBase>*>(storeCollection);
         if (!old_collection) {
           error() << "Failed to cast collection " << coll << endmsg;
           return;
-        }
-        else {
+        } else {
           std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->getData()));
           ptr->getData().put(std::move(uptr), coll);
         }
 
-      }
-      else {
+      } else {
         std::unique_ptr<podio::CollectionBase> uptr(collection->getData().get());
         ptr->getData().put(std::move(uptr), coll);
       }
     }
 
-    info() << "Writing frame, with time: " << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count() << endmsg;
+    info() << "Writing frame, with time: "
+           << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
+                  .count()
+           << endmsg;
     iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
-
   }
 };
 
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index fb659448..138008bd 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -19,53 +19,75 @@
 #ifndef FWCORE_CONSUMER_H
 #define FWCORE_CONSUMER_H
 
+#include <GaudiKernel/FunctionalFilterDecision.h>
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
-#include <GaudiKernel/FunctionalFilterDecision.h>
 
 #include "podio/CollectionBase.h"
 
 #include "k4FWCore/FunctionalUtils.h"
 
+#include "GaudiKernel/CommonMessaging.h"
+
 #include <type_traits>
 #include <utility>
 
 namespace k4FWCore {
 
-namespace details {
-
-  template <typename Signature, typename Traits_>
-  struct Consumer;
-
-  template <typename... In, typename Traits_>
-  struct Consumer<void(const In&...), Traits_>
-      : Gaudi::Functional::details::DataHandleMixin<
-            std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
-            Traits_> {
-    using Gaudi::Functional::details::DataHandleMixin<
-        std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
-        Traits_>::DataHandleMixin;
-
-    // derived classes are NOT allowed to implement execute ...
-    StatusCode execute(const EventContext& ctx) const override final {
-      try {
-        filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-        return Gaudi::Functional::FilterDecision::PASSED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+  namespace details {
+
+    template <typename Signature, typename Traits_> struct Consumer;
+
+    template <typename... In, typename Traits_>
+    struct Consumer<void(const In&...), Traits_>
+        : Gaudi::Functional::details::DataHandleMixin<
+              std::tuple<>,
+              Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<
+          std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
+          Traits_>::DataHandleMixin;
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        try {
+          if constexpr (std::is_same_v<
+                            std::decay_t<decltype(std::get<0>(this->m_inputs))>,
+                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+            Gaudi::Algorithm::info() << "CONSUMER::::" << std::get<0>(this->m_inputs).objKey() << endmsg;
+            // DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>> handle = std::get<0>(this->m_inputs);
+            // handle.
+
+            auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
+            std::istringstream ss(std::get<0>(this->m_inputs).objKey());
+            std::string        token;
+            while (ss >> token) {
+              DataObject* p;
+              this->evtSvc()->retrieveObject(token, p);
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[token]            = collection->getData();
+            }
+            std::get<0>(this->m_inputs).put(std::move(map));
+
+            filter_evtcontext_ttt<In...>::apply(*this, ctx, this->m_inputs);
+          } else {
+            filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+          }
+          return Gaudi::Functional::FilterDecision::PASSED;
+        } catch (GaudiException& e) {
+          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+          return e.code();
+        }
       }
-    }
 
-    // ... instead, they must implement the following operator
-    virtual void operator()(const In&...) const = 0;
-  };
+      // ... instead, they must implement the following operator
+      virtual void operator()(const In&...) const = 0;
+    };
 
-} // namespace details
+  }  // namespace details
 
-template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-using Consumer = details::Consumer<Signature, Traits_>;
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using Consumer = details::Consumer<Signature, Traits_>;
 
-} // namespace k4FWCore
+}  // namespace k4FWCore
 
-#endif // FWCORE_CONSUMER_H
+#endif  // FWCORE_CONSUMER_H
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 511e3bb7..7dcbe2e1 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -2,9 +2,11 @@
 
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
+#include "GaudiKernel/CommonMessaging.h"
 
 #include "podio/CollectionBase.h"
 
+#include <stdexcept>
 #include <tuple>
 #include <type_traits>
 
@@ -26,18 +28,24 @@ namespace details {
 
   template <typename T>
   std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(const T& arg) {
-      // Transformation logic for types derived from podio::CollectionBase
-      // For example:
-      // return /* Transformation logic for derived types */;
-      // Replace /* Transformation logic for derived types */ with your specific transformation logic
     return std::shared_ptr<podio::CollectionBase>(arg);
   }
-  // Transformation function for types not derived from podio::CollectionBase
   template <typename T>
   std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
       // Default: no transformation
-      // return arg;
-      return std::shared_ptr<podio::CollectionBase>(arg);
+      return arg;
+  }
+
+  template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+  auto ptrOrCast(T&& arg) {
+    // return arg;
+    return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
+  }
+  template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+  auto ptrOrCast(T&& arg) {
+    // return arg;
+    std::cout << "Calling static_cast<const T&>(*arg) (ptrOrCast)" << std::endl;
+    return static_cast<const T&>(*arg);
   }
 
 
@@ -55,14 +63,52 @@ namespace details {
 
     template <typename Algorithm, typename Handles>
     static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
-      // return std::apply( [&]( const auto&... handle ) {
-      // printType(*get( handle, algo, ctx )...);
-      //   return algo( static_cast<const
-      // edm4hep::MCParticleCollection&>(*get( handle, algo, ctx ))... ); }, handles );
       return std::apply(
           [&](const auto&... handle) { return algo(transformIfDerivedFromBase<In>(get(handle, algo, ctx))...); },
           handles);
     }
   };
+
+  template <typename... In>
+  struct filter_evtcontext_ttt {
+
+    static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
+                  "EventContext can only appear as first argument");
+
+    template <typename Algorithm, typename Handles>
+    static auto apply(const Algorithm& algo, Handles& handles) {
+      return std::apply(
+          [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
+    }
+
+    template <typename Algorithm, typename Handles>
+    static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+      if constexpr (sizeof...(In) == 1) {
+        return algo(*reinterpret_cast<std::map<std::string, std::shared_ptr<podio::CollectionBase>>*>(std::get<0>(handles).get()));
+      }
+    // return std::apply(
+    //     [&](const auto& firstHandle, const auto&... otherHandles) {
+    //         // Do nothing for the first handle (SpecificType)
+    //       if constexpr (!std::is_same_v<std::decay_t<decltype(firstHandle)>, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
+    //             // Perform the transformation for other handles
+    //         return algo(firstHandle,
+    //                         transformIfDerivedFromBase<In>(get(otherHandles, algo, ctx))...);
+    //         }
+    //     },
+    //     handles);
+}
+
+  //   template <typename Algorithm, typename Handles>
+  //   static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+  //     if constexpr (std::is_same_v<std::decay_t<decltype(std::get<0>(handles))>, DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+  //       return; 
+  //     }
+  //     return std::apply(
+  //         [&](const auto&... handle) { return algo(transformIfDerivedFromBase<In>(get(handle, algo, ctx))...); },
+  //         handles);
+  //   }
+  };
+
+
 } // namespace details
 } // namespace k4FWCore
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 03bfa3ec..6f225bf5 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -1,14 +1,23 @@
-/***********************************************************************************\
-* (c) Copyright 1998-2023 CERN for the benefit of the LHCb and ATLAS collaborations *
-*                                                                                   *
-* This software is distributed under the terms of the Apache version 2 licence,     *
-* copied verbatim in the file "LICENSE".                                            *
-*                                                                                   *
-* In applying this licence, CERN does not waive the privileges and immunities       *
-* granted to it by virtue of its status as an Intergovernmental Organization        *
-* or submit itself to any jurisdiction.                                             *
-\***********************************************************************************/
-#pragma once
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_TRANSFORMER_H
+#define FWCORE_TRANSFORMER_H
 
 #include "FunctionalUtils.h"
 #include "Gaudi/Functional/details.h"
@@ -27,19 +36,6 @@ namespace k4FWCore {
 
 namespace details {
 
-  template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-  auto transformm(T&& arg) {
-    // return arg;
-    return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
-  }
-  template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-  auto transformm(T&& arg) {
-    // return arg;
-    std::cout << "Calling static_cast<const T&>(*arg) (transformm)" << std::endl;
-    return static_cast<const T&>(*arg);
-  }
-
-
   template <typename Signature, typename Traits_>
   struct Transformer;
 
@@ -54,12 +50,12 @@ namespace details {
     StatusCode execute(const EventContext& ctx) const override final {
       try {
         if constexpr (sizeof...(In) == 0) {
-          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), transformm((*this)()));
+          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), ptrOrCast((*this)()));
         } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
           Gaudi::Functional::details::put(std::get<0>(this->m_outputs), (*this)(ctx));
         } else {
           Gaudi::Functional::details::put(std::get<0>(this->m_outputs),
-              transformm(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
+              ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
                                               );
         }
         return Gaudi::Functional::FilterDecision::PASSED;
@@ -95,20 +91,20 @@ namespace details {
               if constexpr (sizeof...(In) == 0) {
                 std::apply(
                     [&ohandle...](auto&&... data) {
-                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
                     },
                     (*this)());
               } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
                 std::apply(
                     [&ohandle...](auto&&... data) {
-                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
                     },
                     (*this)(ctx));
               } else {
                 std::apply(
                     [&ohandle...](auto&&... data) {
                       // (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
-                      (Gaudi::Functional::details::put(ohandle, transformm(std::forward<decltype(data)>(data))), ...);
+                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
                     },
                     filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs));
               }
@@ -211,3 +207,5 @@ template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useD
 using MultiTransformerFilter = details::MultiTransformerFilter<Signature, Traits_, false>;
 
 } // namespace k4FWCore
+
+#endif // FWCORE_TRANSFORMER_H
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index faee0ce9..67fff89b 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -111,13 +111,16 @@ add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py)
 add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py)
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
+add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py)
 add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS ExampleFunctionalProducer PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
-add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
-add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
-add_test_with_env(FunctionalOutputCommands outputCommands/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple)
+add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
+add_test_with_env(FunctionalRuntimeCollections options/ExampleFunctionalRuntimeCollections.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_property(TEST FunctionalCheckFiles APPEND PROPERTY DEPENDS FunctionalFile)
+set_property(TEST FunctionalCheckFiles APPEND PROPERTY DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index dd893aa2..d26b9233 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -12,3 +12,4 @@ def check_collections(filename, names):
 check_collections('functional_transformer.root', ['MCParticles', 'NewMCParticles'])
 check_collections('functional_transformer_multiple.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits', 'Tracks', 'Counter', 'NewMCParticles'])
 check_collections('functional_transformer_multiple_output_commands.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits'])
+check_collections('/tmp/a/b/c/output_k4test_exampledata_producer.root', ['MCParticles'])
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
index d8406702..d5630a84 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
@@ -23,7 +23,7 @@
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer
 from Configurables import EventDataSvc
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer",
                                      OutputCollection="MCParticles",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
index ae18cc1d..ea2eb698 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
@@ -23,7 +23,7 @@
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerMultiple
 from Configurables import EventDataSvc
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 
 producer = ExampleFunctionalProducerMultiple("Producer",
                                              OutputCollectionFloat="VectorFloat",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
new file mode 100644
index 00000000..1cb953b5
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a producer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumerRuntimeCollections
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+producer0 = ExampleFunctionalProducer("Producer0",
+                                     OutputCollection="MCParticles0",
+                                     )
+producer1 = ExampleFunctionalProducer("Producer1",
+                                     OutputCollection="MCParticles1",
+                                     )
+producer2 = ExampleFunctionalProducer("Producer2",
+                                     OutputCollection="MCParticles2",
+                                     )
+consumer = ExampleFunctionalConsumerRuntimeCollections("Consumer",
+                                     InputCollection="MCParticles0 MCParticles1 MCParticles2",
+                                     Offset=0,
+                                     )
+
+
+ApplicationMgr(TopAlg=[producer0, producer1, producer2, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 872e1e3f..5ee39426 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -22,29 +22,18 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformer
-from Configurables import ApplicationMgr
-from Configurables import EventDataSvc, IOSvc
-from Configurables import Reader, Writer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer.root']
 svc.output = 'functional_transformer.root'
-# svc.CollectionNames = ['MCParticles']
-
-reader = Reader("Reader")
-
-writer = Writer("Writer")
-
-# out = PodioOutput("out")
-# out.filename = "output_k4test_exampledata_transformer.root"
-# # The collections that we don't drop will also be present in the output file
-# out.outputCommands = ["drop MCParticles"]
 
 transformer = ExampleFunctionalTransformer("Transformer",
                                            InputCollection="MCParticles",
                                            OutputCollection="NewMCParticles")
 
-mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
                EvtMax=-1,
                ExtSvc=[EventDataSvc("EventDataSvc")],
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
index a6c2ad24..ea68c898 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
@@ -22,24 +22,19 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformerMultiple
-from Configurables import ApplicationMgr
-from Configurables import EventDataSvc, IOSvc
-from Configurables import Reader, Writer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer_multiple.root']
 svc.output = 'functional_transformer_multiple.root'
 
-reader = Reader("Reader")
-
-writer = Writer("Writer")
-
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
                                            # InputCollection="MCParticles",
                                            # OutputCollection="NewMCParticles")
                                                    )
 
-mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
                EvtMax=-1,
                ExtSvc=[EventDataSvc("EventDataSvc")],
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index cfdee9b1..4d354857 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -21,12 +21,9 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO, WARNING
-from Gaudi.Configuration import Gaudi__Sequencer
 from Configurables import ExampleFunctionalTransformer
-from Configurables import ApplicationMgr
-from Configurables import IOSvc
-from Configurables import Reader, Writer
 from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc
+from k4FWCore import ApplicationMgr, IOSvc
 
 evtslots = 2
 threads = 2
@@ -45,24 +42,13 @@
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer.root']
 svc.output = 'functional_transformerMT.root'
-# svc.CollectionNames = ['MCParticles']
 
-reader = Reader("Reader")
-
-writer = Writer("Writer")
-
-# out = PodioOutput("out")
-# out.filename = "output_k4test_exampledata_transformer.root"
-# # The collections that we don't drop will also be present in the output file
-# out.outputCommands = ["drop MCParticles"]
 
 transformer = ExampleFunctionalTransformer("Transformer",
                                            InputCollection="MCParticles",
                                            OutputCollection="NewMCParticles")
 
-seq = Gaudi__Sequencer("Node", Members=[reader, transformer, writer], Sequential=True, OutputLevel=INFO)
-
-mgr = ApplicationMgr(TopAlg=[seq],
+mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
                EvtMax=-1,
                ExtSvc=[whiteboard],
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index 4d2939b1..a4a2d969 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -22,7 +22,8 @@
 
 from Gaudi.Configuration import INFO, WARNING
 from Configurables import ExampleFunctionalProducer, ExampleFunctionalTransformer, ExampleFunctionalConsumer
-from Configurables import ApplicationMgr, HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc
+from Configurables import HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc
+from k4FWCore import ApplicationMgr
 
 evtslots = 5
 threads = 3
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index 26f42afb..24cafc37 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -22,8 +22,8 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalTransformerMultiple, ExampleFunctionalConsumerMultiple
-from Configurables import ApplicationMgr
 from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr
 
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
                                                    InputCollectionFloat="VectorFloat",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
index 882a8051..b8e4c7de 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
@@ -22,9 +22,8 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformerMultiple
-from Configurables import ApplicationMgr
-from Configurables import EventDataSvc, IOSvc
-from Configurables import Reader, Writer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
 svc.input = ['output_k4test_exampledata_producer_multiple.root']
@@ -33,16 +32,12 @@
                       "drop Counter",
                       "drop NewMCParticles",]
 
-reader = Reader("Reader")
-
-writer = Writer("Writer")
-
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
                                            # InputCollection="MCParticles",
                                            # OutputCollection="NewMCParticles")
                                                    )
 
-mgr = ApplicationMgr(TopAlg=[reader, transformer, writer],
+mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
                EvtMax=-1,
                ExtSvc=[EventDataSvc("EventDataSvc")],
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
new file mode 100644
index 00000000..32b02c3e
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example using a producer with a single output and saving that to a file
+
+from Gaudi.Configuration import INFO
+from Configurables import EventDataSvc
+from Configurables import ExampleFunctionalProducer
+from k4FWCore import ApplicationMgr, IOSvc
+
+io = IOSvc("IOSvc")
+io.output = "/tmp/a/b/c/output_k4test_exampledata_producer.root"
+
+producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
+
+ApplicationMgr(TopAlg=[producer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
new file mode 100644
index 00000000..9e6ccf41
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducerRuntimeCollections, ExampleFunctionalConsumer
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+producer = ExampleFunctionalProducerRuntimeCollections("Producer",
+                                                       NumberOfCollections=3)
+
+consumer0 = ExampleFunctionalConsumer("Consumer0",
+                                     InputCollection="MCParticles0",
+                                     Offset=0,
+                                     )
+consumer1 = ExampleFunctionalConsumer("Consumer1",
+                                     InputCollection="MCParticles1",
+                                     Offset=0,
+                                     )
+consumer2 = ExampleFunctionalConsumer("Consumer2",
+                                     InputCollection="MCParticles2",
+                                     Offset=0,
+                                     )
+
+
+ApplicationMgr(TopAlg=[producer, consumer0, consumer1, consumer2],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index f5043dc8..5c7b2816 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -39,6 +39,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
   void operator()(const edm4hep::MCParticleCollection& input) const override {
     if (input.size() != 2) {
       fatal() << "Wrong size of MCParticle collection, expected 2 got " << input.size() << endmsg;
+      throw std::runtime_error("Wrong size of MCParticle collection");
     }
     int i = 0;
     for (const auto& particle : input) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
new file mode 100644
index 00000000..647f9816
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+
+#include "edm4hep/MCParticleCollection.h"
+
+#include "k4FWCore/Consumer.h"
+
+#include <memory>
+#include <string>
+
+struct ExampleFunctionalConsumerRuntimeCollections final
+    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& input)> {
+  // The pair in KeyValue can be changed from python and it corresponds
+  // to the name of the output collection
+  ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
+      : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+
+  // This is the function that will be called to produce the data
+  void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& input) const override {
+    if (input.size() != 3) {
+      fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
+    }
+    for (auto& [key, val] : input) {
+      auto coll = std::dynamic_pointer_cast<edm4hep::MCParticleCollection>(val);
+      int  i    = 0;
+      for (const auto& particle : *coll) {
+        if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+            (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
+            (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
+          std::stringstream error;
+          error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset
+                << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", "
+                << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", "
+                << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
+                << particle.getMass() << "";
+          throw std::runtime_error(error.str());
+        }
+        i++;
+      }
+    }
+  }
+
+private:
+  // We can define any property we want that can be set from python
+  // and use it inside operator()
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 1,
+                                             "Example int that can be used in the algorithm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
+};
+
+DECLARE_COMPONENT(ExampleFunctionalConsumerRuntimeCollections)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index 5d13a3a2..85094937 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -62,8 +62,7 @@ struct ExampleFunctionalProducerMultiple final
 
     auto particles = edm4hep::MCParticleCollection();
     edm4hep::Vector3d v{0, 0, 0};
-    edm4hep::Vector3f vv{3, 0, 0};
-    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, vv);
+    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, v);
     particles.create(2, 3, 4, 5.f, 6.f, 7.f);
 
     auto simTrackerHits = edm4hep::SimTrackerHitCollection();
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
new file mode 100644
index 00000000..01b4f0fc
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+#include "GaudiAlg/Producer.h"
+
+#include "edm4hep/MCParticleCollection.h"
+
+// #include "k4FWCore/ProducerMany.h"
+#include "k4FWCore/TransformerMany.h"
+
+#include <string>
+
+struct ExampleFunctionalProducerRuntimeCollections final : k4FWCore::ProducerMany<std::map<std::string, std::shared_ptr<podio::CollectionBase>>()> {
+  // The pair in KeyValue can be changed from python and it corresponds
+  // to the name of the output collection
+  ExampleFunctionalProducerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
+      : ProducerMany(name, svcLoc, KeyValue("OutputCollection", "MCParticles")) {}
+
+  // This is the function that will be called to produce the data
+  std::map<std::string, std::shared_ptr<podio::CollectionBase>> operator()() const override {
+    std::map<std::string, std::shared_ptr<podio::CollectionBase>> m_outputCollections;
+    for (int i = 0; i < m_numberOfCollections; ++i) {
+      std::string name = "MCParticles" + std::to_string(i);
+      auto coll = std::make_shared<edm4hep::MCParticleCollection>();
+      coll->create(1, 2, 3, 4.f, 5.f, 6.f);
+      coll->create(2, 3, 4, 5.f, 6.f, 7.f);
+      m_outputCollections[name] = std::dynamic_pointer_cast<podio::CollectionBase>(coll);
+    }
+    return m_outputCollections;
+
+  }
+
+private:
+  // We can define any property we want that can be set from python
+  // and use it inside operator()
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 1, "Example int that can be used in the algorithm"};
+};
+
+DECLARE_COMPONENT(ExampleFunctionalProducerRuntimeCollections)

From 6b727d3d7d1da6aa643d39ab01f80afb7d165d0d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 5 Feb 2024 15:52:16 +0100
Subject: [PATCH 009/127] Add functionality to read multiple collections

---
 k4FWCore/include/k4FWCore/Consumer.h          |  42 ++++--
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 130 ++++++------------
 test/k4FWCoreTest/CMakeLists.txt              |  10 +-
 .../options/ExampleFunctionalFile.py          |   2 +-
 ...leFunctionalConsumerRuntimeCollections.cpp |   2 -
 .../ExampleFunctionalProducerMultiple.cpp     |   1 -
 6 files changed, 77 insertions(+), 110 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 138008bd..5000a5d0 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -47,31 +47,43 @@ namespace k4FWCore {
           std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
           Traits_>::DataHandleMixin;
 
-      // derived classes are NOT allowed to implement execute ...
-      StatusCode execute(const EventContext& ctx) const override final {
-        try {
+      // When reading multiple collections we assume that the variable used is a
+      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+      // and read the collections in a space separated string
+      template <size_t Index, typename... Handles>
+      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
+
           if constexpr (std::is_same_v<
-                            std::decay_t<decltype(std::get<0>(this->m_inputs))>,
+                            HandleType,
                             DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-            Gaudi::Algorithm::info() << "CONSUMER::::" << std::get<0>(this->m_inputs).objKey() << endmsg;
-            // DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>> handle = std::get<0>(this->m_inputs);
-            // handle.
-
+            // Transformation logic for the specific type
             auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
-            std::istringstream ss(std::get<0>(this->m_inputs).objKey());
+            std::istringstream ss(std::get<Index>(handles).objKey());
             std::string        token;
             while (ss >> token) {
               DataObject* p;
-              this->evtSvc()->retrieveObject(token, p);
+              auto sc = this->evtSvc()->retrieveObject(token, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
+              }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
               map[token]            = collection->getData();
             }
-            std::get<0>(this->m_inputs).put(std::move(map));
-
-            filter_evtcontext_ttt<In...>::apply(*this, ctx, this->m_inputs);
-          } else {
-            filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+            std::get<Index>(handles).put(std::move(map));
           }
+
+          // Recursive call for the next index
+          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+        }
+      }
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        try {
+          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 7dcbe2e1..d6172703 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -12,103 +12,61 @@
 
 namespace k4FWCore {
 
-static const std::string frameLocation = "/_Frame";
+  static const std::string frameLocation = "/_Frame";
 
-namespace details {
+  namespace details {
 
-  template <typename T, typename P, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, P>, int> = 0>
-  const auto& transformIfDerivedFromBase(const P& arg) {
-    return arg;
-  }
-  template <typename T, typename P, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, P>, int> = 0>
-  const auto& transformIfDerivedFromBase(const P& arg) {
-    return static_cast<const T&>(*arg);
-  }
+    // This function will be used to modify std::shared_ptr<podio::CollectionBase> to the actual collection type
+    template <typename T, typename P>
+    const auto& maybeTransformToEDM4hep(const P& arg) {
+      return arg;
+    }
 
+    template <typename T, typename P>
+    requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
+    const auto& maybeTransformToEDM4hep(const P& arg) {
+      return static_cast<const T&>(*arg);
+    }
 
-  template <typename T>
-  std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(const T& arg) {
-    return std::shared_ptr<podio::CollectionBase>(arg);
-  }
-  template <typename T>
-  std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
+    template <typename T>
+    std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(
+        const T& arg) {
+      return std::shared_ptr<podio::CollectionBase>(arg);
+    }
+    template <typename T>
+    std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
       // Default: no transformation
       return arg;
-  }
-
-  template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-  auto ptrOrCast(T&& arg) {
-    // return arg;
-    return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
-  }
-  template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-  auto ptrOrCast(T&& arg) {
-    // return arg;
-    std::cout << "Calling static_cast<const T&>(*arg) (ptrOrCast)" << std::endl;
-    return static_cast<const T&>(*arg);
-  }
-
-
-  template <typename... In>
-  struct filter_evtcontext_tt {
-
-    static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
-                  "EventContext can only appear as first argument");
-
-    template <typename Algorithm, typename Handles>
-    static auto apply(const Algorithm& algo, Handles& handles) {
-      return std::apply(
-          [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
     }
 
-    template <typename Algorithm, typename Handles>
-    static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
-      return std::apply(
-          [&](const auto&... handle) { return algo(transformIfDerivedFromBase<In>(get(handle, algo, ctx))...); },
-          handles);
+    template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+    auto ptrOrCast(T&& arg) {
+      // return arg;
+      return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
     }
-  };
-
-  template <typename... In>
-  struct filter_evtcontext_ttt {
-
-    static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
-                  "EventContext can only appear as first argument");
-
-    template <typename Algorithm, typename Handles>
-    static auto apply(const Algorithm& algo, Handles& handles) {
-      return std::apply(
-          [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
+    template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
+    auto ptrOrCast(T&& arg) {
+      // return arg;
+      std::cout << "Calling static_cast<const T&>(*arg) (ptrOrCast)" << std::endl;
+      return static_cast<const T&>(*arg);
     }
 
-    template <typename Algorithm, typename Handles>
-    static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
-      if constexpr (sizeof...(In) == 1) {
-        return algo(*reinterpret_cast<std::map<std::string, std::shared_ptr<podio::CollectionBase>>*>(std::get<0>(handles).get()));
-      }
-    // return std::apply(
-    //     [&](const auto& firstHandle, const auto&... otherHandles) {
-    //         // Do nothing for the first handle (SpecificType)
-    //       if constexpr (!std::is_same_v<std::decay_t<decltype(firstHandle)>, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
-    //             // Perform the transformation for other handles
-    //         return algo(firstHandle,
-    //                         transformIfDerivedFromBase<In>(get(otherHandles, algo, ctx))...);
-    //         }
-    //     },
-    //     handles);
-}
+    template <typename... In> struct filter_evtcontext_tt {
+      static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
+                    "EventContext can only appear as first argument");
 
-  //   template <typename Algorithm, typename Handles>
-  //   static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
-  //     if constexpr (std::is_same_v<std::decay_t<decltype(std::get<0>(handles))>, DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-  //       return; 
-  //     }
-  //     return std::apply(
-  //         [&](const auto&... handle) { return algo(transformIfDerivedFromBase<In>(get(handle, algo, ctx))...); },
-  //         handles);
-  //   }
-  };
+      template <typename Algorithm, typename Handles> static auto apply(const Algorithm& algo, Handles& handles) {
+        return std::apply(
+            [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
+      }
 
+      template <typename Algorithm, typename Handles>
+      static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+        return std::apply(
+            [&](const auto&... handle) { return algo(maybeTransformToEDM4hep<In>(get(handle, algo, ctx))...); },
+            handles);
+      }
+    };
 
-} // namespace details
-} // namespace k4FWCore
+  }  // namespace details
+}  // namespace k4FWCore
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 67fff89b..c6a89158 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -112,15 +112,15 @@ add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemo
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
 add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py)
-add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
-add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS ExampleFunctionalProducer)
-add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS ExampleFunctionalProducer PASS_REGULAR_EXPRESSION
+add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS FunctionalProducer)
+add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
+add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalProducer PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
+add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
 add_test_with_env(FunctionalRuntimeCollections options/ExampleFunctionalRuntimeCollections.py)
 
-add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_property(TEST FunctionalCheckFiles APPEND PROPERTY DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
+add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 5ee39426..dc72581a 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -26,7 +26,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ['output_k4test_exampledata_producer.root']
+svc.input = 'output_k4test_exampledata_producer.root'
 svc.output = 'functional_transformer.root'
 
 transformer = ExampleFunctionalTransformer("Transformer",
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index 647f9816..82dbaaac 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -61,8 +61,6 @@ struct ExampleFunctionalConsumerRuntimeCollections final
 private:
   // We can define any property we want that can be set from python
   // and use it inside operator()
-  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 1,
-                                             "Example int that can be used in the algorithm"};
   Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
 };
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index 85094937..df7dacb8 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -36,7 +36,6 @@ using Particle      = edm4hep::MCParticleCollection;
 using SimTrackerHit = edm4hep::SimTrackerHitCollection;
 using TrackerHit    = edm4hep::TrackerHitCollection;
 using Track         = edm4hep::TrackCollection;
-using Ptr = std::shared_ptr<podio::CollectionBase>;
 
 struct ExampleFunctionalProducerMultiple final
   : k4FWCore::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>()> {

From 8394e2af4982d90b121954f47259370a1005009e Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 8 Feb 2024 11:01:52 +0100
Subject: [PATCH 010/127] Change to ROOTReader and ROOTWriter

---
 k4FWCore/components/IOSvc.cpp                        |  2 +-
 k4FWCore/components/IOSvc.h                          | 12 ++++++------
 .../components/ExampleFunctionalProducerMultiple.cpp |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index fb426c35..50484575 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -34,7 +34,7 @@
 
 StatusCode IOSvc::initialize() {
   if (!m_readingFileNames.empty()) {
-    m_reader = std::make_unique<podio::ROOTFrameReader>();
+    m_reader = std::make_unique<podio::ROOTReader>();
     try {
       m_reader->openFiles(m_readingFileNames);
     } catch (std::runtime_error& e) {
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index cb040655..54cf66a8 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -26,8 +26,8 @@
 #include "GaudiKernel/IDataProviderSvc.h"
 #include "GaudiKernel/IHiveWhiteBoard.h"
 
-#include "podio/ROOTFrameReader.h"
-#include "podio/ROOTFrameWriter.h"
+#include "podio/ROOTReader.h"
+#include "podio/ROOTWriter.h"
 #include "podio/RNTupleReader.h"
 #include "podio/RNTupleWriter.h"
 
@@ -76,12 +76,12 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   KeepDropSwitch m_switch;
 
-  std::unique_ptr<podio::ROOTFrameReader> m_reader{nullptr};
-  std::shared_ptr<podio::ROOTFrameWriter> m_writer{nullptr};
+  std::unique_ptr<podio::ROOTReader> m_reader{nullptr};
+  std::shared_ptr<podio::ROOTWriter> m_writer{nullptr};
 
-  std::shared_ptr<podio::ROOTFrameWriter> getWriter() override {
+  std::shared_ptr<podio::ROOTWriter> getWriter() override {
     if (!m_writer) {
-      m_writer = std::shared_ptr<podio::ROOTFrameWriter>(new podio::ROOTFrameWriter(m_writingFileName.value()));
+      m_writer = std::shared_ptr<podio::ROOTWriter>(new podio::ROOTWriter(m_writingFileName.value()));
     }
     return m_writer;
   }
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index df7dacb8..05c9de28 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -61,7 +61,8 @@ struct ExampleFunctionalProducerMultiple final
 
     auto particles = edm4hep::MCParticleCollection();
     edm4hep::Vector3d v{0, 0, 0};
-    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, v);
+    edm4hep::Vector3f vv{0, 0, 0};
+    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, vv);
     particles.create(2, 3, 4, 5.f, 6.f, 7.f);
 
     auto simTrackerHits = edm4hep::SimTrackerHitCollection();

From b609670df18ac1a8db5f357cbd52ec4697a01653 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:15:28 +0100
Subject: [PATCH 011/127] Add reading multiple sets of multiple collections

---
 ...ionalConsumerRuntimeCollectionsMultiple.py | 68 +++++++++++++
 ...onalConsumerRuntimeCollectionsMultiple.cpp | 96 +++++++++++++++++++
 2 files changed, 164 insertions(+)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
new file mode 100644
index 00000000..8e8c4c9c
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -0,0 +1,68 @@
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a producer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerRuntimeCollectionsMultiple
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+producer0 = ExampleFunctionalProducerMultiple("Producer0",
+                                              OutputCollectionFloat="VectorFloat0",
+                                              OutputCollectionParticles1="MCParticles0",
+                                              OutputCollectionParticles2="MCParticles1",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
+                                              OutputCollectionTrackerHits="TrackerHits0",
+                                              OutputCollectionTracks="Tracks0",
+                                              ExampleInt=5
+                                              )
+producer1 = ExampleFunctionalProducerMultiple("Producer1",
+                                              OutputCollectionFloat="VectorFloat1",
+                                              OutputCollectionParticles1="MCParticles2",
+                                              OutputCollectionParticles2="MCParticles3",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
+                                              OutputCollectionTrackerHits="TrackerHits1",
+                                              OutputCollectionTracks="Tracks1",
+                                              ExampleInt=5
+                                              )
+producer2 = ExampleFunctionalProducerMultiple("Producer2",
+                                              OutputCollectionFloat="VectorFloat2",
+                                              OutputCollectionParticles1="MCParticles4",
+                                              OutputCollectionParticles2="MCParticles5",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
+                                              OutputCollectionTrackerHits="TrackerHits2",
+                                              OutputCollectionTracks="Tracks2",
+                                              ExampleInt=5
+                                              )
+
+consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple("Consumer",
+                                     Particles="MCParticles0 MCParticles1 MCParticles2 MCParticles3 MCParticles4",
+                                     Tracks="Tracks0 Tracks1 Tracks2",
+                                     SimTrackerHits="SimTrackerHits0",
+                                     Offset=0,
+                                     )
+
+
+ApplicationMgr(TopAlg=[producer0, producer1, producer2, consumer],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
new file mode 100644
index 00000000..12e0e619
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+
+#include "edm4hep/MCParticleCollection.h"
+#include "edm4hep/SimTrackerHitCollection.h"
+#include "edm4hep/TrackCollection.h"
+
+#include "k4FWCore/Consumer.h"
+
+#include <memory>
+#include <string>
+
+struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
+    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& particleMap,
+                              const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
+                              const edm4hep::SimTrackerHitCollection&                              simTrackerHits)> {
+  // The pair in KeyValue can be changed from python and it corresponds
+  // to the name of the output collection
+  ExampleFunctionalConsumerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc)
+      : Consumer(name, svcLoc,
+                 {
+                     KeyValue("Particles", "MCParticles"),
+                     KeyValue("Tracks", "MCParticles"),
+                     KeyValue("SimTrackerHits", "MCParticles"),
+                 }) {}
+
+  // This is the function that will be called to produce the data
+  void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& particleMap,
+                  const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
+                  const edm4hep::SimTrackerHitCollection&                              simTrackerHits) const override {
+    info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size() << " track collections"
+           << endmsg;
+    if (particleMap.size() != 5) {
+      fatal() << "Wrong size of the particleMap map, expected 5, got " << particleMap.size() << endmsg;
+    }
+    for (auto& [key, val] : particleMap) {
+      const auto& particles = *std::static_pointer_cast<edm4hep::MCParticleCollection>(val);
+      int  i    = 0;
+      for (const auto& particle : particles) {
+        if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+            (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
+            (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
+          std::stringstream error;
+          error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset
+                << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", "
+                << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", "
+                << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
+                << particle.getMass() << "";
+          throw std::runtime_error(error.str());
+        }
+        i++;
+      }
+    }
+    if (trackMap.size() != 3) {
+      fatal() << "Wrong size of the tracks map, expected 3, got " << trackMap.size() << endmsg;
+    }
+    for (auto& [key, val] : trackMap) {
+      const auto& tracks = *std::static_pointer_cast<edm4hep::TrackCollection>(val);
+      if ((tracks[0].getType() != 1) || (std::abs(tracks[0].getChi2() - 2.1) > 1e-6) || (tracks[0].getNdf() != 3) ||
+          (std::abs(tracks[0].getDEdx() - 4.1) > 1e-6) || (std::abs(tracks[0].getDEdxError() - 5.1) > 1e-6) ||
+          (std::abs(tracks[0].getRadiusOfInnermostHit() - 6.1) > 1e-6)) {
+        std::stringstream error;
+        error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks[0].getType()
+              << ", " << tracks[0].getChi2() << ", " << tracks[0].getNdf() << ", " << tracks[0].getDEdx() << ", "
+              << tracks[0].getDEdxError() << ", " << tracks[0].getRadiusOfInnermostHit() << "";
+        throw std::runtime_error(error.str());
+      }
+    }
+  }
+
+private:
+// We can define any property we want that can be set from python
+// and use it inside operator()
+Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
+}
+;
+
+DECLARE_COMPONENT(ExampleFunctionalConsumerRuntimeCollectionsMultiple)

From a237f64faf2689ee76f18cb57f4e8046452c6a39 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sun, 11 Feb 2024 17:20:17 +0100
Subject: [PATCH 012/127] Add a new version of the consumer

---
 k4FWCore/include/k4FWCore/NewConsumer.h       | 346 ++++++++++++++++++
 ...pleFunctionalConsumerRuntimeCollections.py |   3 +-
 ...ionalConsumerRuntimeCollectionsMultiple.py |  10 +-
 ...leFunctionalConsumerRuntimeCollections.cpp |   5 +-
 ...onalConsumerRuntimeCollectionsMultiple.cpp |  11 +-
 5 files changed, 360 insertions(+), 15 deletions(-)
 create mode 100644 k4FWCore/include/k4FWCore/NewConsumer.h

diff --git a/k4FWCore/include/k4FWCore/NewConsumer.h b/k4FWCore/include/k4FWCore/NewConsumer.h
new file mode 100644
index 00000000..44c164d7
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/NewConsumer.h
@@ -0,0 +1,346 @@
+// /*
+//  * Copyright (c) 2014-2024 Key4hep-Project.
+//  *
+//  * This file is part of Key4hep.
+//  * See https://key4hep.github.io/key4hep-doc/ for further info.
+//  *
+//  * Licensed under the Apache License, Version 2.0 (the "License");
+//  * you may not use this file except in compliance with the License.
+//  * You may obtain a copy of the License at
+//  *
+//  *     http://www.apache.org/licenses/LICENSE-2.0
+//  *
+//  * Unless required by applicable law or agreed to in writing, software
+//  * distributed under the License is distributed on an "AS IS" BASIS,
+//  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  * See the License for the specific language governing permissions and
+//  * limitations under the License.
+//  */
+// #ifndef FWCORE_CONSUMER_H
+// #define FWCORE_CONSUMER_H
+
+// #include <GaudiKernel/FunctionalFilterDecision.h>
+// #include "Gaudi/Functional/details.h"
+// #include "Gaudi/Functional/utilities.h"
+
+// #include "podio/CollectionBase.h"
+
+// #include "k4FWCore/FunctionalUtils.h"
+
+// #include "GaudiKernel/CommonMessaging.h"
+
+// #include <type_traits>
+// #include <utility>
+// #include <concepts>
+
+// namespace k4FWCore {
+
+//   namespace details {
+
+//     template <typename F, size_t... Is>
+//     auto for_impl( F&& f, std::index_sequence<Is...> ) {
+//       // if constexpr ( std::disjunction_v<std::is_void<std::invoke_result_t<F, std::integral_constant<int, Is>>>...> ) {
+//       //   ( std::invoke( f, std::integral_constant<int, Is>{} ), ... );
+//       // } else {
+//         return std::array{ std::invoke( f, std::integral_constant<int, Is>{} )... };
+//       // }
+//     }
+
+//     template <auto N, typename F>
+//     decltype( auto ) for_( F&& f ) {
+//       return for_impl( std::forward<F>( f ), std::make_index_sequence<N>{} );
+//     }
+
+//     template <typename Signature, typename Traits_> struct Consumer;
+
+//     template <typename... In, typename Traits_>
+//     struct Consumer<void(const In&...), Traits_>
+//         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
+//     private:
+//       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
+//       template <typename T>
+//       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+
+//       template <typename T>
+//       struct maybeVector {
+//         using type = std::vector<InputHandle_t<T>>;
+//       };
+
+//       template <>
+//       struct maybeVector<std::map<std::string, std::shared_ptr<podio::CollectionBase>>> {
+//         using type = InputHandle_t<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>;
+//       };
+
+//       std::tuple<maybeVector<In>...> m_inputs; //   and make the handles properties instead...
+//       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...( In )> m_inputLocations; // TODO/FIXME: remove
+
+//     public:
+//       using KeyValue  = typename base_class::KeyValue;
+//       using KeyValues = typename base_class::KeyValues;
+//       using InKeys     = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...( In )>;
+
+//     private:
+//       auto construct_properties( InKeys inputs ) {
+//         return details::for_<sizeof...( In )>( [&]( auto I ) {
+//           constexpr auto i   = decltype( I )::value;
+//           auto&          ins = std::get<i>( inputs );
+//           return Gaudi::Property<std::vector<DataObjID>>{
+//             this, ins.first, Gaudi::Functional::details::to_DataObjID( ins.second ),
+//               [this]( auto&& ) {
+//                 auto& handles = std::get<i>( this->m_inputs );
+//                 auto& ins     = std::get<i>( this->m_inputLocations );
+//                 using Handles = typename std::decay_t<decltype( handles )>;
+//                 handles       = make_vector_of_handles<Handles>( this, ins );
+//                 // if ( std::is_pointer_v<typename Handles::value_type> ) { // handle constructor does not (yet) allow to
+//                 //                                                          // set
+//                 //                                                          // optional flag... so do it
+//                 //                                                          // explicitly here...
+//                 //   std::for_each( handles.begin(), handles.end(), []( auto& h ) { h.setOptional( true ); } );
+//                 // }
+//               },
+//               Gaudi::Details::Property::ImmediatelyInvokeHandler{ true } };
+//         } );
+//       }
+//     public:
+
+//       Consumer( std::string name, ISvcLocator* locator, InKeys inputs )
+//           : base_class( std::move( name ), locator ),
+//             m_inputLocations{ construct_properties( inputs ) } {
+//       }
+
+//       Consumer(std::string name, ISvcLocator* locator, const KeyValues& inputs)
+//         : Consumer{std::move(name), locator, InKeys{inputs}} {
+//       }
+
+//       // When reading multiple collections we assume that the variable used is a
+//       // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+//       // and read the collections in a space separated string
+//       template <size_t Index, typename... Handles>
+//       void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+//         if constexpr (Index < sizeof...(Handles)) {
+//           using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
+
+//           if constexpr (std::is_same_v<
+//                             HandleType,
+//                             DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+//             // Transformation logic for the specific type
+//             auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
+//             std::istringstream ss(std::get<Index>(handles).objKey());
+//             std::string        token;
+//             while (ss >> token) {
+//               DataObject* p;
+//               auto        sc = this->evtSvc()->retrieveObject(token, p);
+//               if (!sc.isSuccess()) {
+//                 throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
+//               }
+//               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+//               map[token]            = collection->getData();
+//             }
+//             std::get<Index>(handles).put(std::move(map));
+//           }
+
+//           // Recursive call for the next index
+//           transformAndApplyAlgoAtIndex<Index + 1>(handles);
+//         }
+//       }
+
+//       // derived classes are NOT allowed to implement execute ...
+//       StatusCode execute(const EventContext& ctx) const override final {
+//         try {
+//           transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+//           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+//           return Gaudi::Functional::FilterDecision::PASSED;
+//         } catch (GaudiException& e) {
+//           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+//           return e.code();
+//         }
+//       }
+
+//       // ... instead, they must implement the following operator
+//       virtual void operator()(const In&...) const = 0;
+//     };
+
+//   }  // namespace details
+
+//   template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+//   using Consumer = details::Consumer<Signature, Traits_>;
+
+// }  // namespace k4FWCore
+
+// #endif  // FWCORE_CONSUMER_H
+
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_CONSUMER_H
+#define FWCORE_CONSUMER_H
+
+#include <GaudiKernel/DataObjectHandle.h>
+#include <GaudiKernel/FunctionalFilterDecision.h>
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+
+#include "podio/CollectionBase.h"
+
+#include "k4FWCore/FunctionalUtils.h"
+
+#include "GaudiKernel/CommonMessaging.h"
+
+#include <type_traits>
+#include <utility>
+
+namespace k4FWCore {
+
+  namespace details {
+
+    template <typename Signature, typename Traits_> struct Consumer;
+
+    template <typename... In, typename Traits_>
+    struct Consumer<void(const In&...), Traits_>
+        : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
+
+      template <typename T>
+      using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...> m_inputs;  //   and make the handles properties instead...
+      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
+      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
+      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
+
+      using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
+
+      using KeyValue  = typename base_class::KeyValue;
+      using KeyValues = typename base_class::KeyValues;
+      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
+
+      // Consumer(std::string name, ISvcLocator* locator, const std::vector<KeyValues>& inputs)
+      template <typename IArgs, std::size_t... I>
+      Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
+          : base_class(std::move(name), locator),
+            m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<I>(inputs).first,
+                {DataObjID{std::get<I>(inputs).second[0]}},
+                [this](Gaudi::Details::PropertyBase&) {
+                  Gaudi::Algorithm::info() << "Inside the property handler" << endmsg;
+                  if constexpr (!std::is_same_v<std::decay_t<decltype(std::get<I>(m_inputs))>,
+                                                DataObjectReadHandle<
+                                                    std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+                    auto& handle  = std::get<I>(m_inputs);
+                    auto& ins     = m_inputLocations[I];
+                    handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                    Gaudi::Algorithm::info() << "Input location (inside lambda): " << ins << endmsg;
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
+            }
+        ...},
+            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
+            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
+
+      {
+        Gaudi::Algorithm::info() << "Creating Consumer with " << sizeof...(In) << " inputs" << endmsg;
+        for (auto& val : std::get<0>(inputs).second) {
+          Gaudi::Algorithm::info() << "Input location: " << val << endmsg;
+        }
+        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
+        //   // for (auto& value : std::get<I...>(inputs).second) {
+        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
+        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
+        //   // }
+        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
+        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
+        //   // auto& handles = std::get<0>(m_inputs);
+        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
+        // }
+      }
+
+      constexpr static std::size_t N_in = sizeof...(In);
+
+      Consumer(std::string name, ISvcLocator* locator,
+               Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const& inputs)
+          : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
+
+      // When reading multiple collections we assume that the variable used is a
+      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+      // and read the collections in a space separated string
+      template <size_t Index, typename... Handles>
+      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          if constexpr (std::is_same_v<
+                            std::decay_t<decltype(std::get<Index>(handles))>,
+                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+            auto map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
+
+            // To be locked
+            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+              auto vec = std::vector<std::string>();
+              vec.reserve(m_inputLocations[Index].value().size());
+              for (auto& val : m_inputLocations[Index].value()) {
+                vec.push_back(val.key());
+              }
+              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+            }
+
+            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
+              DataObject* p;
+              auto        sc = this->evtSvc()->retrieveObject(value, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
+              }
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[value]            = collection->getData();
+            }
+            std::get<Index>(handles).put(std::move(map));
+          }
+
+          // Recursive call for the next index
+          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+        }
+      }
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        for (auto& prop : m_inputLocations) {
+          Gaudi::Algorithm::info() << "Input location: " << prop << endmsg;
+        }
+        try {
+          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+          return Gaudi::Functional::FilterDecision::PASSED;
+        } catch (GaudiException& e) {
+          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+          return e.code();
+        }
+      }
+
+      // ... instead, they must implement the following operator
+      virtual void operator()(const In&...) const = 0;
+    };
+
+  }  // namespace details
+
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using Consumer = details::Consumer<Signature, Traits_>;
+
+}  // namespace k4FWCore
+
+#endif  // FWCORE_CONSUMER_H
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index 1cb953b5..ee7a7eaf 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -34,7 +34,8 @@
                                      OutputCollection="MCParticles2",
                                      )
 consumer = ExampleFunctionalConsumerRuntimeCollections("Consumer",
-                                     InputCollection="MCParticles0 MCParticles1 MCParticles2",
+                                     # InputCollection="MCParticles0 MCParticles1 MCParticles2",
+                                     InputCollection=["MCParticles0", "MCParticles1", "MCParticles2"],
                                      Offset=0,
                                      )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 8e8c4c9c..1fe8d6fd 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -53,11 +53,11 @@
                                               )
 
 consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple("Consumer",
-                                     Particles="MCParticles0 MCParticles1 MCParticles2 MCParticles3 MCParticles4",
-                                     Tracks="Tracks0 Tracks1 Tracks2",
-                                     SimTrackerHits="SimTrackerHits0",
-                                     Offset=0,
-                                     )
+                                                               Particles=["MCParticles0", "MCParticles1", "MCParticles2", "MCParticles3", "MCParticles4"],
+                                                               Tracks=["Tracks0", "Tracks1", "Tracks2"],
+                                                               SimTrackerHits=["SimTrackerHits0"],
+                                                               Offset=0,
+                                                               )
 
 
 ApplicationMgr(TopAlg=[producer0, producer1, producer2, consumer],
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index 82dbaaac..d4ebfc30 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -21,7 +21,8 @@
 
 #include "edm4hep/MCParticleCollection.h"
 
-#include "k4FWCore/Consumer.h"
+// #include "k4FWCore/Consumer.h"
+#include "k4FWCore/NewConsumer.h"
 
 #include <memory>
 #include <string>
@@ -31,7 +32,7 @@ struct ExampleFunctionalConsumerRuntimeCollections final
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+    : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {}
 
   // This is the function that will be called to produce the data
   void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& input) const override {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index 12e0e619..faaeec25 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -23,7 +23,8 @@
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
 
-#include "k4FWCore/Consumer.h"
+// #include "k4FWCore/Consumer.h"
+#include "k4FWCore/NewConsumer.h"
 
 #include <memory>
 #include <string>
@@ -35,12 +36,8 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc,
-                 {
-                     KeyValue("Particles", "MCParticles"),
-                     KeyValue("Tracks", "MCParticles"),
-                     KeyValue("SimTrackerHits", "MCParticles"),
-                 }) {}
+    : Consumer(name, svcLoc, {KeyValues("Particles", {"MCParticles"}), KeyValues("Tracks", {"MCParticles"}), KeyValues("SimTrackerHits", {"MCParticles"})}) {}
+    // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
 
   // This is the function that will be called to produce the data
   void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& particleMap,

From 3426bd5060ac3d34ce6885ea09a379b6494253b0 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 09:04:03 +0100
Subject: [PATCH 013/127] Add the old version of the consumer

---
 k4FWCore/include/k4FWCore/OldConsumer.h | 105 ++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 k4FWCore/include/k4FWCore/OldConsumer.h

diff --git a/k4FWCore/include/k4FWCore/OldConsumer.h b/k4FWCore/include/k4FWCore/OldConsumer.h
new file mode 100644
index 00000000..5000a5d0
--- /dev/null
+++ b/k4FWCore/include/k4FWCore/OldConsumer.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_CONSUMER_H
+#define FWCORE_CONSUMER_H
+
+#include <GaudiKernel/FunctionalFilterDecision.h>
+#include "Gaudi/Functional/details.h"
+#include "Gaudi/Functional/utilities.h"
+
+#include "podio/CollectionBase.h"
+
+#include "k4FWCore/FunctionalUtils.h"
+
+#include "GaudiKernel/CommonMessaging.h"
+
+#include <type_traits>
+#include <utility>
+
+namespace k4FWCore {
+
+  namespace details {
+
+    template <typename Signature, typename Traits_> struct Consumer;
+
+    template <typename... In, typename Traits_>
+    struct Consumer<void(const In&...), Traits_>
+        : Gaudi::Functional::details::DataHandleMixin<
+              std::tuple<>,
+              Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<
+          std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
+          Traits_>::DataHandleMixin;
+
+      // When reading multiple collections we assume that the variable used is a
+      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+      // and read the collections in a space separated string
+      template <size_t Index, typename... Handles>
+      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
+
+          if constexpr (std::is_same_v<
+                            HandleType,
+                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
+            // Transformation logic for the specific type
+            auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
+            std::istringstream ss(std::get<Index>(handles).objKey());
+            std::string        token;
+            while (ss >> token) {
+              DataObject* p;
+              auto sc = this->evtSvc()->retrieveObject(token, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
+              }
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[token]            = collection->getData();
+            }
+            std::get<Index>(handles).put(std::move(map));
+          }
+
+          // Recursive call for the next index
+          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+        }
+      }
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        try {
+          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+          return Gaudi::Functional::FilterDecision::PASSED;
+        } catch (GaudiException& e) {
+          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+          return e.code();
+        }
+      }
+
+      // ... instead, they must implement the following operator
+      virtual void operator()(const In&...) const = 0;
+    };
+
+  }  // namespace details
+
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using Consumer = details::Consumer<Signature, Traits_>;
+
+}  // namespace k4FWCore
+
+#endif  // FWCORE_CONSUMER_H

From b57f985fb6f5d4403a826bf18e933fd945ed0942 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 09:04:39 +0100
Subject: [PATCH 014/127] Add reading maps of edm4hep collections

---
 k4FWCore/include/k4FWCore/Consumer.h          | 110 +++++-
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |  26 +-
 k4FWCore/include/k4FWCore/NewConsumer.h       | 346 ------------------
 .../components/ExampleEventHeaderConsumer.cpp |   2 +-
 .../components/ExampleFunctionalConsumer.cpp  |   2 +-
 .../ExampleFunctionalConsumerMultiple.cpp     |  10 +-
 ...leFunctionalConsumerRuntimeCollections.cpp |  12 +-
 ...onalConsumerRuntimeCollectionsMultiple.cpp |   7 +-
 8 files changed, 127 insertions(+), 388 deletions(-)
 delete mode 100644 k4FWCore/include/k4FWCore/NewConsumer.h

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 5000a5d0..43dd6eef 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -19,7 +19,9 @@
 #ifndef FWCORE_CONSUMER_H
 #define FWCORE_CONSUMER_H
 
+#include <GaudiKernel/DataObjectHandle.h>
 #include <GaudiKernel/FunctionalFilterDecision.h>
+#include "FunctionalUtils.h"
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
 
@@ -29,6 +31,7 @@
 
 #include "GaudiKernel/CommonMessaging.h"
 
+#include <memory>
 #include <type_traits>
 #include <utility>
 
@@ -40,12 +43,70 @@ namespace k4FWCore {
 
     template <typename... In, typename Traits_>
     struct Consumer<void(const In&...), Traits_>
-        : Gaudi::Functional::details::DataHandleMixin<
-              std::tuple<>,
-              Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
-      using Gaudi::Functional::details::DataHandleMixin<
-          std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
-          Traits_>::DataHandleMixin;
+        : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
+
+      template <typename T>
+      using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...> m_inputs;  //   and make the handles properties instead...
+      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
+      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
+      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
+
+      using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
+
+      using KeyValue  = typename base_class::KeyValue;
+      using KeyValues = typename base_class::KeyValues;
+      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
+
+      // Consumer(std::string name, ISvcLocator* locator, const std::vector<KeyValues>& inputs)
+      template <typename IArgs, std::size_t... I>
+      Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
+          : base_class(std::move(name), locator),
+            m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<I>(inputs).first,
+                {DataObjID{std::get<I>(inputs).second[0]}},
+                [this, inputs](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<In>::value) {
+                    Gaudi::Algorithm::info() << "Inside the lambda for " << std::get<I>(inputs).first << endmsg;
+                    auto& handle  = std::get<I>(m_inputs);
+                    auto& ins     = m_inputLocations[I];
+                    handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                    Gaudi::Algorithm::info() << "Input location (inside lambda): " << ins << endmsg;
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
+            }
+        ...},
+            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
+            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
+
+      {
+        Gaudi::Algorithm::info() << "Creating Consumer with " << sizeof...(In) << " inputs" << endmsg;
+        for (auto& val : std::get<0>(inputs).second) {
+          Gaudi::Algorithm::info() << "Input location: " << val << endmsg;
+        }
+        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
+        //   // for (auto& value : std::get<I...>(inputs).second) {
+        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
+        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
+        //   // }
+        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
+        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
+        //   // auto& handles = std::get<0>(m_inputs);
+        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
+        // }
+      }
+
+      constexpr static std::size_t N_in = sizeof...(In);
+
+      Consumer(std::string name, ISvcLocator* locator,
+               Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const& inputs)
+          : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
 
       // When reading multiple collections we assume that the variable used is a
       // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
@@ -53,23 +114,31 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles>
       void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
-          using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
-
-          if constexpr (std::is_same_v<
-                            HandleType,
-                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-            // Transformation logic for the specific type
-            auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
-            std::istringstream ss(std::get<Index>(handles).objKey());
-            std::string        token;
-            while (ss >> token) {
+          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            
+            Gaudi::Algorithm::info() << "Treating as a map: " << std::get<Index>(handles).objKey() << endmsg;
+              
+            using EDM4hepType = typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+
+            // To be locked
+            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+              auto vec = std::vector<std::string>();
+              vec.reserve(m_inputLocations[Index].value().size());
+              for (auto& val : m_inputLocations[Index].value()) {
+                vec.push_back(val.key());
+              }
+              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+            }
+
+            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
               DataObject* p;
-              auto sc = this->evtSvc()->retrieveObject(token, p);
+              auto        sc = this->evtSvc()->retrieveObject(value, p);
               if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
+                throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[token]            = collection->getData();
+              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
             }
             std::get<Index>(handles).put(std::move(map));
           }
@@ -81,6 +150,9 @@ namespace k4FWCore {
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
+        for (auto& prop : m_inputLocations) {
+          Gaudi::Algorithm::info() << "Input location: " << prop << endmsg;
+        }
         try {
           transformAndApplyAlgoAtIndex<0>(this->m_inputs);
           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index d6172703..00de7d56 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -17,22 +17,38 @@ namespace k4FWCore {
   namespace details {
 
     // This function will be used to modify std::shared_ptr<podio::CollectionBase> to the actual collection type
-    template <typename T, typename P>
-    const auto& maybeTransformToEDM4hep(const P& arg) {
-      return arg;
-    }
+    template <typename T, typename P> const auto& maybeTransformToEDM4hep(const P& arg) { return arg; }
 
     template <typename T, typename P>
-    requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
+      requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
     const auto& maybeTransformToEDM4hep(const P& arg) {
       return static_cast<const T&>(*arg);
     }
 
+    template <typename T> struct is_map_like : std::false_type {};
+
+    template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
+
+    template <typename T> struct ExtractInnerType;
+
+    // Specialization for DataObjectReadHandle with map and shared_ptr
+    template <typename Value>
+    struct ExtractInnerType<DataObjectReadHandle<std::map<std::string, std::shared_ptr<Value>>>> {
+      using type = Value;
+    };
+
     template <typename T>
     std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(
         const T& arg) {
       return std::shared_ptr<podio::CollectionBase>(arg);
     }
+
+    // template <typename T>
+    // std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::map<std::string, std::shared_ptr<T>>>
+    // transformType(const std::map<std::string, std::shared_ptr<T>>& arg) {
+    //   return std::map<std::string, std::shared_ptr<T>>();
+    // }
+
     template <typename T>
     std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
       // Default: no transformation
diff --git a/k4FWCore/include/k4FWCore/NewConsumer.h b/k4FWCore/include/k4FWCore/NewConsumer.h
deleted file mode 100644
index 44c164d7..00000000
--- a/k4FWCore/include/k4FWCore/NewConsumer.h
+++ /dev/null
@@ -1,346 +0,0 @@
-// /*
-//  * Copyright (c) 2014-2024 Key4hep-Project.
-//  *
-//  * This file is part of Key4hep.
-//  * See https://key4hep.github.io/key4hep-doc/ for further info.
-//  *
-//  * Licensed under the Apache License, Version 2.0 (the "License");
-//  * you may not use this file except in compliance with the License.
-//  * You may obtain a copy of the License at
-//  *
-//  *     http://www.apache.org/licenses/LICENSE-2.0
-//  *
-//  * Unless required by applicable law or agreed to in writing, software
-//  * distributed under the License is distributed on an "AS IS" BASIS,
-//  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  * See the License for the specific language governing permissions and
-//  * limitations under the License.
-//  */
-// #ifndef FWCORE_CONSUMER_H
-// #define FWCORE_CONSUMER_H
-
-// #include <GaudiKernel/FunctionalFilterDecision.h>
-// #include "Gaudi/Functional/details.h"
-// #include "Gaudi/Functional/utilities.h"
-
-// #include "podio/CollectionBase.h"
-
-// #include "k4FWCore/FunctionalUtils.h"
-
-// #include "GaudiKernel/CommonMessaging.h"
-
-// #include <type_traits>
-// #include <utility>
-// #include <concepts>
-
-// namespace k4FWCore {
-
-//   namespace details {
-
-//     template <typename F, size_t... Is>
-//     auto for_impl( F&& f, std::index_sequence<Is...> ) {
-//       // if constexpr ( std::disjunction_v<std::is_void<std::invoke_result_t<F, std::integral_constant<int, Is>>>...> ) {
-//       //   ( std::invoke( f, std::integral_constant<int, Is>{} ), ... );
-//       // } else {
-//         return std::array{ std::invoke( f, std::integral_constant<int, Is>{} )... };
-//       // }
-//     }
-
-//     template <auto N, typename F>
-//     decltype( auto ) for_( F&& f ) {
-//       return for_impl( std::forward<F>( f ), std::make_index_sequence<N>{} );
-//     }
-
-//     template <typename Signature, typename Traits_> struct Consumer;
-
-//     template <typename... In, typename Traits_>
-//     struct Consumer<void(const In&...), Traits_>
-//         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
-//     private:
-//       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
-//       template <typename T>
-//       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
-
-//       template <typename T>
-//       struct maybeVector {
-//         using type = std::vector<InputHandle_t<T>>;
-//       };
-
-//       template <>
-//       struct maybeVector<std::map<std::string, std::shared_ptr<podio::CollectionBase>>> {
-//         using type = InputHandle_t<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>;
-//       };
-
-//       std::tuple<maybeVector<In>...> m_inputs; //   and make the handles properties instead...
-//       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...( In )> m_inputLocations; // TODO/FIXME: remove
-
-//     public:
-//       using KeyValue  = typename base_class::KeyValue;
-//       using KeyValues = typename base_class::KeyValues;
-//       using InKeys     = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...( In )>;
-
-//     private:
-//       auto construct_properties( InKeys inputs ) {
-//         return details::for_<sizeof...( In )>( [&]( auto I ) {
-//           constexpr auto i   = decltype( I )::value;
-//           auto&          ins = std::get<i>( inputs );
-//           return Gaudi::Property<std::vector<DataObjID>>{
-//             this, ins.first, Gaudi::Functional::details::to_DataObjID( ins.second ),
-//               [this]( auto&& ) {
-//                 auto& handles = std::get<i>( this->m_inputs );
-//                 auto& ins     = std::get<i>( this->m_inputLocations );
-//                 using Handles = typename std::decay_t<decltype( handles )>;
-//                 handles       = make_vector_of_handles<Handles>( this, ins );
-//                 // if ( std::is_pointer_v<typename Handles::value_type> ) { // handle constructor does not (yet) allow to
-//                 //                                                          // set
-//                 //                                                          // optional flag... so do it
-//                 //                                                          // explicitly here...
-//                 //   std::for_each( handles.begin(), handles.end(), []( auto& h ) { h.setOptional( true ); } );
-//                 // }
-//               },
-//               Gaudi::Details::Property::ImmediatelyInvokeHandler{ true } };
-//         } );
-//       }
-//     public:
-
-//       Consumer( std::string name, ISvcLocator* locator, InKeys inputs )
-//           : base_class( std::move( name ), locator ),
-//             m_inputLocations{ construct_properties( inputs ) } {
-//       }
-
-//       Consumer(std::string name, ISvcLocator* locator, const KeyValues& inputs)
-//         : Consumer{std::move(name), locator, InKeys{inputs}} {
-//       }
-
-//       // When reading multiple collections we assume that the variable used is a
-//       // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-//       // and read the collections in a space separated string
-//       template <size_t Index, typename... Handles>
-//       void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
-//         if constexpr (Index < sizeof...(Handles)) {
-//           using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
-
-//           if constexpr (std::is_same_v<
-//                             HandleType,
-//                             DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-//             // Transformation logic for the specific type
-//             auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
-//             std::istringstream ss(std::get<Index>(handles).objKey());
-//             std::string        token;
-//             while (ss >> token) {
-//               DataObject* p;
-//               auto        sc = this->evtSvc()->retrieveObject(token, p);
-//               if (!sc.isSuccess()) {
-//                 throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
-//               }
-//               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-//               map[token]            = collection->getData();
-//             }
-//             std::get<Index>(handles).put(std::move(map));
-//           }
-
-//           // Recursive call for the next index
-//           transformAndApplyAlgoAtIndex<Index + 1>(handles);
-//         }
-//       }
-
-//       // derived classes are NOT allowed to implement execute ...
-//       StatusCode execute(const EventContext& ctx) const override final {
-//         try {
-//           transformAndApplyAlgoAtIndex<0>(this->m_inputs);
-//           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-//           return Gaudi::Functional::FilterDecision::PASSED;
-//         } catch (GaudiException& e) {
-//           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-//           return e.code();
-//         }
-//       }
-
-//       // ... instead, they must implement the following operator
-//       virtual void operator()(const In&...) const = 0;
-//     };
-
-//   }  // namespace details
-
-//   template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-//   using Consumer = details::Consumer<Signature, Traits_>;
-
-// }  // namespace k4FWCore
-
-// #endif  // FWCORE_CONSUMER_H
-
-/*
- * Copyright (c) 2014-2024 Key4hep-Project.
- *
- * This file is part of Key4hep.
- * See https://key4hep.github.io/key4hep-doc/ for further info.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef FWCORE_CONSUMER_H
-#define FWCORE_CONSUMER_H
-
-#include <GaudiKernel/DataObjectHandle.h>
-#include <GaudiKernel/FunctionalFilterDecision.h>
-#include "Gaudi/Functional/details.h"
-#include "Gaudi/Functional/utilities.h"
-
-#include "podio/CollectionBase.h"
-
-#include "k4FWCore/FunctionalUtils.h"
-
-#include "GaudiKernel/CommonMessaging.h"
-
-#include <type_traits>
-#include <utility>
-
-namespace k4FWCore {
-
-  namespace details {
-
-    template <typename Signature, typename Traits_> struct Consumer;
-
-    template <typename... In, typename Traits_>
-    struct Consumer<void(const In&...), Traits_>
-        : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
-      using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
-
-      template <typename T>
-      using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
-
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...> m_inputs;  //   and make the handles properties instead...
-      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
-      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
-      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
-      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
-
-      using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
-
-      using KeyValue  = typename base_class::KeyValue;
-      using KeyValues = typename base_class::KeyValues;
-      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
-
-      // Consumer(std::string name, ISvcLocator* locator, const std::vector<KeyValues>& inputs)
-      template <typename IArgs, std::size_t... I>
-      Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
-          : base_class(std::move(name), locator),
-            m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                std::get<I>(inputs).first,
-                {DataObjID{std::get<I>(inputs).second[0]}},
-                [this](Gaudi::Details::PropertyBase&) {
-                  Gaudi::Algorithm::info() << "Inside the property handler" << endmsg;
-                  if constexpr (!std::is_same_v<std::decay_t<decltype(std::get<I>(m_inputs))>,
-                                                DataObjectReadHandle<
-                                                    std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-                    auto& handle  = std::get<I>(m_inputs);
-                    auto& ins     = m_inputLocations[I];
-                    handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
-                    Gaudi::Algorithm::info() << "Input location (inside lambda): " << ins << endmsg;
-                  }
-                },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
-            }
-        ...},
-            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
-            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
-
-      {
-        Gaudi::Algorithm::info() << "Creating Consumer with " << sizeof...(In) << " inputs" << endmsg;
-        for (auto& val : std::get<0>(inputs).second) {
-          Gaudi::Algorithm::info() << "Input location: " << val << endmsg;
-        }
-        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
-        //   // for (auto& value : std::get<I...>(inputs).second) {
-        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
-        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
-        //   // }
-        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
-        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
-        //   // auto& handles = std::get<0>(m_inputs);
-        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
-        // }
-      }
-
-      constexpr static std::size_t N_in = sizeof...(In);
-
-      Consumer(std::string name, ISvcLocator* locator,
-               Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const& inputs)
-          : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
-
-      // When reading multiple collections we assume that the variable used is a
-      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-      // and read the collections in a space separated string
-      template <size_t Index, typename... Handles>
-      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (std::is_same_v<
-                            std::decay_t<decltype(std::get<Index>(handles))>,
-                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-            auto map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
-
-            // To be locked
-            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-              auto vec = std::vector<std::string>();
-              vec.reserve(m_inputLocations[Index].value().size());
-              for (auto& val : m_inputLocations[Index].value()) {
-                vec.push_back(val.key());
-              }
-              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
-            }
-
-            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(value, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[value]            = collection->getData();
-            }
-            std::get<Index>(handles).put(std::move(map));
-          }
-
-          // Recursive call for the next index
-          transformAndApplyAlgoAtIndex<Index + 1>(handles);
-        }
-      }
-
-      // derived classes are NOT allowed to implement execute ...
-      StatusCode execute(const EventContext& ctx) const override final {
-        for (auto& prop : m_inputLocations) {
-          Gaudi::Algorithm::info() << "Input location: " << prop << endmsg;
-        }
-        try {
-          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
-          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-          return Gaudi::Functional::FilterDecision::PASSED;
-        } catch (GaudiException& e) {
-          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-          return e.code();
-        }
-      }
-
-      // ... instead, they must implement the following operator
-      virtual void operator()(const In&...) const = 0;
-    };
-
-  }  // namespace details
-
-  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-  using Consumer = details::Consumer<Signature, Traits_>;
-
-}  // namespace k4FWCore
-
-#endif  // FWCORE_CONSUMER_H
diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
index 4d33775a..09c139b5 100644
--- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
@@ -36,7 +36,7 @@
 struct ExampleEventHeaderConsumer final
     : k4FWCore::Consumer<void(const edm4hep::EventHeaderCollection&)> {
   ExampleEventHeaderConsumer(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc, {KeyValue("EventHeaderName", edm4hep::EventHeaderName)}) {}
+    : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::EventHeaderName})}) {}
 
   void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const {
     if (evtHeaderColl.empty()) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index 5c7b2816..3631b5ae 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -31,7 +31,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the input collection
   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
-      : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {}
+    : Consumer(name, svcLoc, KeyValues("InputCollection", {"MCParticles"})) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as the collections
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
index 95a82350..b1e9d846 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
@@ -53,11 +53,11 @@ struct ExampleFunctionalConsumerMultiple final
   ExampleFunctionalConsumerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : Consumer(name, svcLoc,
                  {
-                     KeyValue("InputCollectionFloat", "VectorFloat"),
-                     KeyValue("InputCollectionParticles", "MCParticles1"),
-                     KeyValue("InputCollectionSimTrackerHits", "SimTrackerHits"),
-                     KeyValue("InputCollectionTrackerHits", "TrackerHits"),
-                     KeyValue("InputCollectionTracks", "Tracks"),
+                   KeyValues("InputCollectionFloat", {"VectorFloat"}),
+                   KeyValues("InputCollectionParticles", {"MCParticles1"}),
+                   KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
+                   KeyValues("InputCollectionTrackerHits", {"TrackerHits"}),
+                   KeyValues("InputCollectionTracks", {"Tracks"}),
                  }) {}
 
   // This is the function that will be called to transform the data
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index d4ebfc30..09b89322 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -21,28 +21,26 @@
 
 #include "edm4hep/MCParticleCollection.h"
 
-// #include "k4FWCore/Consumer.h"
-#include "k4FWCore/NewConsumer.h"
+#include "k4FWCore/Consumer.h"
 
 #include <memory>
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollections final
-    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& input)> {
+    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
-    : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {}
+      : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& input) const override {
+  void operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input) const override {
     if (input.size() != 3) {
       fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
     }
     for (auto& [key, val] : input) {
-      auto coll = std::dynamic_pointer_cast<edm4hep::MCParticleCollection>(val);
       int  i    = 0;
-      for (const auto& particle : *coll) {
+      for (const auto& particle : *val) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
             (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index faaeec25..d0c37363 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -23,14 +23,13 @@
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
 
-// #include "k4FWCore/Consumer.h"
-#include "k4FWCore/NewConsumer.h"
+#include "k4FWCore/Consumer.h"
 
 #include <memory>
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
-    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& particleMap,
+  : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
                               const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
                               const edm4hep::SimTrackerHitCollection&                              simTrackerHits)> {
   // The pair in KeyValue can be changed from python and it corresponds
@@ -40,7 +39,7 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
     // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& particleMap,
+  void operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
                   const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
                   const edm4hep::SimTrackerHitCollection&                              simTrackerHits) const override {
     info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size() << " track collections"

From 35a3255311d549b17d178085e5cff96fef2a39c6 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 09:28:49 +0100
Subject: [PATCH 015/127] Make some clean up in the consumer

---
 k4FWCore/include/k4FWCore/Consumer.h | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 43dd6eef..3fec60a8 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -19,9 +19,7 @@
 #ifndef FWCORE_CONSUMER_H
 #define FWCORE_CONSUMER_H
 
-#include <GaudiKernel/DataObjectHandle.h>
 #include <GaudiKernel/FunctionalFilterDecision.h>
-#include "FunctionalUtils.h"
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
 
@@ -29,7 +27,7 @@
 
 #include "k4FWCore/FunctionalUtils.h"
 
-#include "GaudiKernel/CommonMessaging.h"
+// #include "GaudiKernel/CommonMessaging.h"
 
 #include <memory>
 #include <type_traits>
@@ -70,13 +68,11 @@ namespace k4FWCore {
                 this,
                 std::get<I>(inputs).first,
                 {DataObjID{std::get<I>(inputs).second[0]}},
-                [this, inputs](Gaudi::Details::PropertyBase&) {
+                [this](Gaudi::Details::PropertyBase&) {
                   if constexpr (!is_map_like<In>::value) {
-                    Gaudi::Algorithm::info() << "Inside the lambda for " << std::get<I>(inputs).first << endmsg;
                     auto& handle  = std::get<I>(m_inputs);
                     auto& ins     = m_inputLocations[I];
                     handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
-                    Gaudi::Algorithm::info() << "Input location (inside lambda): " << ins << endmsg;
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
@@ -86,10 +82,6 @@ namespace k4FWCore {
             m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
 
       {
-        Gaudi::Algorithm::info() << "Creating Consumer with " << sizeof...(In) << " inputs" << endmsg;
-        for (auto& val : std::get<0>(inputs).second) {
-          Gaudi::Algorithm::info() << "Input location: " << val << endmsg;
-        }
         // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
         //   // for (auto& value : std::get<I...>(inputs).second) {
         //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
@@ -116,8 +108,6 @@ namespace k4FWCore {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             
-            Gaudi::Algorithm::info() << "Treating as a map: " << std::get<Index>(handles).objKey() << endmsg;
-              
             using EDM4hepType = typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
             auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
 
@@ -150,9 +140,6 @@ namespace k4FWCore {
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
-        for (auto& prop : m_inputLocations) {
-          Gaudi::Algorithm::info() << "Input location: " << prop << endmsg;
-        }
         try {
           transformAndApplyAlgoAtIndex<0>(this->m_inputs);
           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);

From 59170f0dbdc8ffd10cb8fa5086dc9c7192ab422b Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 09:31:56 +0100
Subject: [PATCH 016/127] Add the python folder

---
 python/CMakeLists.txt             |  6 ++++++
 python/k4FWCore/ApplicationMgr.py | 17 +++++++++++++++++
 python/k4FWCore/IOSvc.py          | 28 ++++++++++++++++++++++++++++
 python/k4FWCore/__init__.py       |  2 ++
 4 files changed, 53 insertions(+)
 create mode 100644 python/CMakeLists.txt
 create mode 100644 python/k4FWCore/ApplicationMgr.py
 create mode 100644 python/k4FWCore/IOSvc.py
 create mode 100644 python/k4FWCore/__init__.py

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
new file mode 100644
index 00000000..b076d15e
--- /dev/null
+++ b/python/CMakeLists.txt
@@ -0,0 +1,6 @@
+install(FILES ${to_install} DESTINATION ${podio_PYTHON_INSTALLDIR})
+
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore
+  DESTINATION python
+  )
+
diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
new file mode 100644
index 00000000..a7df717f
--- /dev/null
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -0,0 +1,17 @@
+from Configurables import ApplicationMgr as AppMgr
+from Configurables import Reader, Writer, IOSvc
+
+# seq = Gaudi__Sequencer("Node", Members=[reader, transformer, writer], Sequential=True, OutputLevel=INFO)
+
+class ApplicationMgr:
+
+    def __init__(self, **kwargs):
+        self._mgr = AppMgr(**kwargs)
+
+        for conf in frozenset(self._mgr.allConfigurables.values()):
+            if isinstance(conf, IOSvc):
+                props = conf.getPropertiesWithDescription()
+                if 'input' in props:
+                    self._mgr.TopAlg =  [Reader("Reader")] + self._mgr.TopAlg
+                if 'output' in props:
+                    self._mgr.TopAlg = self._mgr.TopAlg + [Writer("Writer")]
diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
new file mode 100644
index 00000000..cafb37c5
--- /dev/null
+++ b/python/k4FWCore/IOSvc.py
@@ -0,0 +1,28 @@
+from Configurables import IOSvc as IO
+import os
+
+class IOSvc:
+
+    def __init__(self, *args, **kwargs):
+
+        self._svc = IO(**kwargs)
+
+    def __getattr__(self, attr):
+        return getattr(self._svc, attr)
+
+    def __setattr__(self, attr, value):
+        print(f'IOSvc: {attr} {value}')
+        if attr == '_svc':
+            super().__setattr__(attr, value)
+            return
+
+        if attr == 'input':
+            if isinstance(value, str):
+                value = [value]
+            for inp in value:
+                if os.path.dirname(inp) and not os.path.exists(inp):
+                    os.makedirs(os.path.dirname(inp))
+        if attr == 'output':
+            if os.path.dirname(value) and not os.path.exists(os.path.dirname(value)):
+                os.makedirs(os.path.dirname(value))
+        setattr(self._svc, attr, value)
diff --git a/python/k4FWCore/__init__.py b/python/k4FWCore/__init__.py
new file mode 100644
index 00000000..c8a912de
--- /dev/null
+++ b/python/k4FWCore/__init__.py
@@ -0,0 +1,2 @@
+from .ApplicationMgr import ApplicationMgr
+from .IOSvc import IOSvc

From 98099f7413585f8fcf1dd1e6bf1101757b47ae07 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 10:23:20 +0100
Subject: [PATCH 017/127] Delete another unused test file

---
 .../options/ReadManyCollections.py            | 49 -------------------
 1 file changed, 49 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/ReadManyCollections.py

diff --git a/test/k4FWCoreTest/options/ReadManyCollections.py b/test/k4FWCoreTest/options/ReadManyCollections.py
deleted file mode 100644
index a2be9bed..00000000
--- a/test/k4FWCoreTest/options/ReadManyCollections.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Copyright (c) 2014-2023 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example using a producer with a multiple outputs and saving that to a file
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducerMultiple
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
-
-podioevent = k4DataSvc("EventDataSvc")
-
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_producer_multiple.root"
-# Collections can be dropped
-# out.outputCommands = ["drop *"]
-
-producer = ExampleFunctionalProducerMultiple("ExampleFunctionalProducerMultiple",
-                                             OutputCollectionFloat="VectorFloat",
-                                             OutputCollectionParticles1="MCParticles1",
-                                             OutputCollectionParticles2="MCParticles2",
-                                             OutputCollectionSimTrackerHits="SimTrackerHits",
-                                             OutputCollectionTrackerHits="TrackerHits",
-                                             OutputCollectionTracks="Tracks",
-                                             ExampleInt=5)
-
-ApplicationMgr(TopAlg=[producer, out],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[k4DataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )

From f71df098dcc10090a57e814d6b8eb0c952e1a396 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 10:24:48 +0100
Subject: [PATCH 018/127] Fix a few tests by making all the parameters lists

---
 .../ExampleFunctionalConsumerRuntimeCollections.py     |  2 +-
 ...mpleFunctionalConsumerRuntimeCollectionsMultiple.py |  2 +-
 test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py |  2 +-
 .../options/ExampleFunctionalMultipleMemory.py         | 10 +++++-----
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index ee7a7eaf..f7be2442 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -21,7 +21,7 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumerRuntimeCollections
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducer("Producer0",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 1fe8d6fd..cc81b154 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -21,7 +21,7 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerRuntimeCollectionsMultiple
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducerMultiple("Producer0",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index a4a2d969..ab6c1dfc 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -47,7 +47,7 @@
                                      OutputCollection="MCParticles")
 
 consumer = ExampleFunctionalConsumer("Consumer",
-                                     InputCollection="NewMCParticles",
+                                     InputCollection=["NewMCParticles"],
                                      Offset=10,
                                      )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index 24cafc37..d2e82f41 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -39,11 +39,11 @@
 producer = ExampleFunctionalProducerMultiple("Producer")
 
 consumer = ExampleFunctionalConsumerMultiple("Consumer",
-                                             InputCollectionFloat="VectorFloat",
-                                             InputCollectionParticles="NewMCParticles",
-                                             InputCollectionSimTrackerHits="SimTrackerHits",
-                                             InputCollectionTrackerHits="TrackerHits",
-                                             InputCollectionTracks="Tracks",
+                                             InputCollectionFloat=["VectorFloat"],
+                                             InputCollectionParticles=["NewMCParticles"],
+                                             InputCollectionSimTrackerHits=["SimTrackerHits"],
+                                             InputCollectionTrackerHits=["TrackerHits"],
+                                             InputCollectionTracks=["Tracks"],
                                              Offset=10,
                                              )
 

From 327d252ebdd7a51ac413e528d31000b12bf65423 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:05:42 +0100
Subject: [PATCH 019/127] Add a new transformer

---
 k4FWCore/include/k4FWCore/Producer.h          |  13 +-
 k4FWCore/include/k4FWCore/Transformer.h       | 483 ++++++++++++------
 .../options/ExampleFunctionalFile.py          |   4 +-
 .../options/ExampleFunctionalMTFile.py        |   4 +-
 .../options/ExampleFunctionalMTMemory.py      |   4 +-
 .../ExampleFunctionalMultipleMemory.py        |  14 +-
 .../ExampleFunctionalTransformer.cpp          |   9 +-
 .../ExampleFunctionalTransformerMultiple.cpp  |   8 +-
 8 files changed, 338 insertions(+), 201 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Producer.h b/k4FWCore/include/k4FWCore/Producer.h
index 1b0da23e..277ab6ca 100644
--- a/k4FWCore/include/k4FWCore/Producer.h
+++ b/k4FWCore/include/k4FWCore/Producer.h
@@ -19,20 +19,9 @@
 #ifndef FWCORE_PRODUCER_H
 #define FWCORE_PRODUCER_H
 
-#include "FunctionalUtils.h"
-#include "Gaudi/Functional/details.h"
-#include "Gaudi/Functional/utilities.h"
-#include <GaudiKernel/CommonMessaging.h>
-#include <GaudiKernel/FunctionalFilterDecision.h>
-
-#include "podio/CollectionBase.h"
-
-#include "k4FWCore/FunctionalUtils.h"
 #include "k4FWCore/Transformer.h"
 
-#include <type_traits>
-#include <utility>
-
+#include <tuple>
 
 
 namespace k4FWCore {
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 6f225bf5..64b904be 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -19,193 +19,342 @@
 #ifndef FWCORE_TRANSFORMER_H
 #define FWCORE_TRANSFORMER_H
 
-#include "FunctionalUtils.h"
+#include <GaudiKernel/FunctionalFilterDecision.h>
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
-#include <GaudiKernel/CommonMessaging.h>
-#include <GaudiKernel/FunctionalFilterDecision.h>
 
 #include "podio/CollectionBase.h"
 
 #include "k4FWCore/FunctionalUtils.h"
 
+// #include "GaudiKernel/CommonMessaging.h"
+
+#include <memory>
 #include <type_traits>
 #include <utility>
 
 namespace k4FWCore {
 
-namespace details {
-
-  template <typename Signature, typename Traits_>
-  struct Transformer;
-
-  template <typename Out, typename... In, typename Traits_>
-  struct Transformer<Out(const In&...), Traits_>
-      : Gaudi::Functional::details::DataHandleMixin<std::tuple<decltype(transformType(std::declval<Out>()))>,
-                                                    Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
-    using Gaudi::Functional::details::DataHandleMixin<
-        std::tuple<decltype(transformType(std::declval<Out>()))>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_>::DataHandleMixin;
-
-    // derived classes can NOT implement execute
-    StatusCode execute(const EventContext& ctx) const override final {
-      try {
-        if constexpr (sizeof...(In) == 0) {
-          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), ptrOrCast((*this)()));
-        } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
-          Gaudi::Functional::details::put(std::get<0>(this->m_outputs), (*this)(ctx));
-        } else {
-          Gaudi::Functional::details::put(std::get<0>(this->m_outputs),
-              ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
-                                              );
-        }
-        return Gaudi::Functional::FilterDecision::PASSED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+  namespace details {
+
+    template <typename Signature, typename Traits_> struct Transformer;
+
+    template <typename Out, typename... In, typename Traits_>
+    struct Transformer<Out(const In&...), Traits_>
+        : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
+
+      template <typename T>
+      using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+      template <typename T>
+      using OutputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
+          m_inputs;  //   and make the handles properties instead...
+      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>>
+          m_outputs;  //   and make the handles properties instead...
+      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
+      std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
+      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
+      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, 1> m_outputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_outputLocationsMap;
+
+      using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
+
+      using KeyValue  = typename base_class::KeyValue;
+      using KeyValues = typename base_class::KeyValues;
+      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
+      using OutKeys   = Gaudi::Functional::details::RepeatValues_<KeyValues, 1>;
+
+      template <typename IArgs, typename OArgs, std::size_t... I, std::size_t... J>
+      Transformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>,
+                       const OArgs& outputs, std::index_sequence<J...>)
+          : base_class(std::move(name), locator),
+            m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<I>(inputs).first,
+                {DataObjID{std::get<I>(inputs).second[0]}},
+                [this](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<In>::value) {
+                    auto& handle = std::get<I>(m_inputs);
+                    auto& ins    = m_inputLocations[I];
+                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
+            m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<J>(outputs).first,
+                {DataObjID{std::get<J>(outputs).second[0]}},
+                [this](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<Out>::value) {
+                    auto& handle = std::get<J>(m_outputs);
+                    auto& ins    = m_outputLocations[J];
+                    handle       = OutputHandle_t<decltype(transformType(std::declval<Out>()))>(ins.value()[0], this);
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
+            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
+            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...},
+            m_outputs{OutputHandle_t<decltype(transformType(std::declval<Out>()))>(std::get<J>(outputs).first, this)...}
+
+      {
+        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
+        //   // for (auto& value : std::get<I...>(inputs).second) {
+        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
+        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
+        //   // }
+        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
+        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
+        //   // auto& handles = std::get<0>(m_inputs);
+        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
+        // }
       }
-    }
-
-    // instead they MUST implement this operator
-    virtual Out operator()(const In&...) const = 0;
-  };
-
-  //
-  // general N -> M algorithms
-  //
-  template <typename Signature, typename Traits_>
-  struct MultiTransformer;
-
-  template <typename... Out, typename... In, typename Traits_>
-  struct MultiTransformer<std::tuple<Out...>(const In&...), Traits_>
-      : Gaudi::Functional::details::DataHandleMixin<std::tuple<decltype(transformType(std::declval<Out>()))...>,
-                                                    Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
-    using Gaudi::Functional::details::DataHandleMixin<
-        std::tuple<decltype(transformType(std::declval<Out>()))...>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_>::DataHandleMixin;
-
-    // derived classes can NOT implement execute
-    StatusCode execute(const EventContext& ctx) const override final {
-      try {
-        GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN
-        std::apply(
-            [this, &ctx](auto&... ohandle) {
-              if constexpr (sizeof...(In) == 0) {
-                std::apply(
-                    [&ohandle...](auto&&... data) {
-                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
-                    },
-                    (*this)());
-              } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
-                std::apply(
-                    [&ohandle...](auto&&... data) {
-                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
-                    },
-                    (*this)(ctx));
-              } else {
-                std::apply(
-                    [&ohandle...](auto&&... data) {
-                      // (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
-                      (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
-                    },
-                    filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs));
+
+      constexpr static std::size_t N_in  = sizeof...(In);
+      constexpr static std::size_t N_out = 1;
+
+      Transformer(std::string name, ISvcLocator* locator,
+                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const&  inputs,
+                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_out> const& outputs)
+          : Transformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
+                             std::index_sequence_for<Out>{}) {}
+
+      // When reading multiple collections we assume that the variable used is a
+      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+      // and read the collections in a space separated string
+      template <size_t Index, typename... Handles>
+      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            using EDM4hepType =
+                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+
+            // To be locked
+            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+              auto vec = std::vector<std::string>();
+              vec.reserve(m_inputLocations[Index].value().size());
+              for (auto& val : m_inputLocations[Index].value()) {
+                vec.push_back(val.key());
               }
-            },
-            this->m_outputs);
-        GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
-        return Gaudi::Functional::FilterDecision::PASSED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+            }
+
+            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
+              DataObject* p;
+              auto        sc = this->evtSvc()->retrieveObject(value, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + value, "Transformer", StatusCode::FAILURE);
+              }
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+            }
+            std::get<Index>(handles).put(std::move(map));
+          }
+
+          // Recursive call for the next index
+          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+        }
       }
-    }
-
-    // instead they MUST implement this operator
-    virtual std::tuple<Out...> operator()(const In&...) const = 0;
-  };
-
-  //
-  // general N -> M algorithms with filter functionality
-  //
-  template <typename Signature, typename Traits_, bool isLegacy>
-  struct MultiTransformerFilter;
-
-  template <typename... Out, typename... In, typename Traits_>
-  struct MultiTransformerFilter<std::tuple<Out...>(const In&...), Traits_, true>
-      : Gaudi::Functional::details::DataHandleMixin<std::tuple<Out...>,
-                                                    Gaudi::Functional::details::filter_evtcontext<In...>, Traits_> {
-    using Gaudi::Functional::details::DataHandleMixin<
-        std::tuple<Out...>, Gaudi::Functional::details::filter_evtcontext<In...>, Traits_>::DataHandleMixin;
-
-    // derived classes can NOT implement execute
-    StatusCode execute() override final {
-      try {
-        return std::apply(
-                   [&](auto&... ohandle) {
-                     GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN
-                     return std::apply(
-                         [&ohandle...](bool passed, auto&&... data) {
-                           (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
-                           return passed;
-                         },
-                         Gaudi::Functional::details::filter_evtcontext_t<In...>::apply(*this, this->m_inputs));
-                     GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
-                   },
-                   this->m_outputs)
-            ? Gaudi::Functional::FilterDecision::PASSED
-            : Gaudi::Functional::FilterDecision::FAILED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        try {
+          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          if constexpr (sizeof...(In) == 0) {
+            Gaudi::Functional::details::put(std::get<0>(this->m_outputs), ptrOrCast((*this)()));
+          } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
+            Gaudi::Functional::details::put(std::get<0>(this->m_outputs), (*this)(ctx));
+          } else {
+            Gaudi::Functional::details::put(std::get<0>(this->m_outputs),
+                ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
+                                                );
+          }
+          return Gaudi::Functional::FilterDecision::PASSED;
+        } catch (GaudiException& e) {
+          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+          return e.code();
+        }
       }
-    }
-
-    // instead they MUST implement this operator
-    virtual std::tuple<bool, Out...> operator()(const In&...) const = 0;
-  };
-
-  template <typename... Out, typename... In, typename Traits_>
-  struct MultiTransformerFilter<std::tuple<Out...>(const In&...), Traits_, false>
-      : Gaudi::Functional::details::DataHandleMixin<std::tuple<Out...>,
-                                                    Gaudi::Functional::details::filter_evtcontext<In...>, Traits_> {
-    using Gaudi::Functional::details::DataHandleMixin<
-        std::tuple<Out...>, Gaudi::Functional::details::filter_evtcontext<In...>, Traits_>::DataHandleMixin;
-
-    // derived classes can NOT implement execute
-    StatusCode execute(const EventContext& ctx) const override final {
-      try {
-        return std::apply(
-                   GF_SUPPRESS_SPURIOUS_CLANG_WARNING_BEGIN[&](auto&... ohandle) {
-                     return std::apply(
-                         [&ohandle...](bool passed, auto&&... data) {
-                           (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
-                           return passed;
-                         },
-                         Gaudi::Functional::details::filter_evtcontext_t<In...>::apply(*this, ctx, this->m_inputs));
-                   },
-                   GF_SUPPRESS_SPURIOUS_CLANG_WARNING_END
-
-                   this->m_outputs)
-            ? Gaudi::Functional::FilterDecision::PASSED
-            : Gaudi::Functional::FilterDecision::FAILED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+
+      // ... instead, they must implement the following operator
+      virtual Out operator()(const In&...) const = 0;
+    };
+
+
+
+
+
+
+    template <typename Signature, typename Traits_> struct MultiTransformer;
+
+    template <typename... Out, typename... In, typename Traits_>
+    struct MultiTransformer<std::tuple<Out...>(const In&...), Traits_>
+        : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
+      using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
+
+      template <typename T>
+      using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+      template <typename T>
+      using OutputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
+          m_inputs;  //   and make the handles properties instead...
+      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>...>
+          m_outputs;  //   and make the handles properties instead...
+      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
+      std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
+      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
+      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)> m_outputLocations{};
+      mutable std::map<std::string, std::vector<std::string>>            m_outputLocationsMap;
+
+      using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
+
+      using KeyValue  = typename base_class::KeyValue;
+      using KeyValues = typename base_class::KeyValues;
+      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
+      using OutKeys   = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(Out)>;
+
+      template <typename IArgs, typename OArgs, std::size_t... I, std::size_t... J>
+      MultiTransformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>,
+                       const OArgs& outputs, std::index_sequence<J...>)
+          : base_class(std::move(name), locator),
+            m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<I>(inputs).first,
+                {DataObjID{std::get<I>(inputs).second[0]}},
+                [this](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<In>::value) {
+                    auto& handle = std::get<I>(m_inputs);
+                    auto& ins    = m_inputLocations[I];
+                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
+            m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
+                this,
+                std::get<J>(outputs).first,
+                {DataObjID{std::get<J>(outputs).second[0]}},
+                [this](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<Out>::value) {
+                    auto& handle = std::get<J>(m_outputs);
+                    auto& ins    = m_outputLocations[J];
+                    handle       = OutputHandle_t<decltype(transformType(std::declval<Out>()))>(ins.value()[0], this);
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
+            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
+            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...},
+            m_outputs{OutputHandle_t<decltype(transformType(std::declval<Out>()))>(std::get<J>(outputs).first, this)...}
+
+      {
+        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
+        //   // for (auto& value : std::get<I...>(inputs).second) {
+        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
+        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
+        //   // }
+        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
+        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
+        //   // auto& handles = std::get<0>(m_inputs);
+        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
+        // }
+      }
+
+      constexpr static std::size_t N_in  = sizeof...(In);
+      constexpr static std::size_t N_out = sizeof...(Out);
+
+      MultiTransformer(std::string name, ISvcLocator* locator,
+                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const&  inputs,
+                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_out> const& outputs)
+          : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
+                             std::index_sequence_for<Out...>{}) {}
+
+      // When reading multiple collections we assume that the variable used is a
+      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
+      // and read the collections in a space separated string
+      template <size_t Index, typename... Handles>
+      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            using EDM4hepType =
+                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+
+            // To be locked
+            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+              auto vec = std::vector<std::string>();
+              vec.reserve(m_inputLocations[Index].value().size());
+              for (auto& val : m_inputLocations[Index].value()) {
+                vec.push_back(val.key());
+              }
+              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+            }
+
+            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
+              DataObject* p;
+              auto        sc = this->evtSvc()->retrieveObject(value, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + value, "MultiTransformer", StatusCode::FAILURE);
+              }
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+            }
+            std::get<Index>(handles).put(std::move(map));
+          }
+
+          // Recursive call for the next index
+          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+        }
+      }
+
+      // derived classes are NOT allowed to implement execute ...
+      StatusCode execute(const EventContext& ctx) const override final {
+        try {
+          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          std::apply(
+              [this, &ctx](auto&... ohandle) {
+                if constexpr (sizeof...(In) == 0) {
+                  std::apply(
+                      [&ohandle...](auto&&... data) {
+                        (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
+                      },
+                      (*this)());
+                } else {
+                  std::apply(
+                      [&ohandle...](auto&&... data) {
+                        // (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
+                        (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
+                      },
+                      filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs));
+                }
+              },
+              this->m_outputs);
+          return Gaudi::Functional::FilterDecision::PASSED;
+        } catch (GaudiException& e) {
+          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+          return e.code();
+        }
       }
-    }
 
-    // instead they MUST implement this operator
-    virtual std::tuple<bool, Out...> operator()(const In&...) const = 0;
-  };
-} // namespace details
+      // ... instead, they must implement the following operator
+      virtual std::tuple<Out...> operator()(const In&...) const = 0;
+    };
 
-template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-using Transformer = details::Transformer<Signature, Traits_>;
+  }  // namespace details
 
-template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-using MultiTransformer = details::MultiTransformer<Signature, Traits_>;
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using MultiTransformer = details::MultiTransformer<Signature, Traits_>;
 
-template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-using MultiTransformerFilter = details::MultiTransformerFilter<Signature, Traits_, false>;
+  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
+  using Transformer = details::Transformer<Signature, Traits_>;
 
-} // namespace k4FWCore
+}  // namespace k4FWCore
 
-#endif // FWCORE_TRANSFORMER_H
+#endif  // FWCORE_TRANSFORMER_H
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index dc72581a..87949d08 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -30,8 +30,8 @@
 svc.output = 'functional_transformer.root'
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
+                                           InputCollection=["MCParticles"],
+                                           OutputCollection=["NewMCParticles"])
 
 mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 4d354857..fe8153af 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -45,8 +45,8 @@
 
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
+                                           InputCollection=["MCParticles"],
+                                           OutputCollection=["NewMCParticles"])
 
 mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index ab6c1dfc..a68de06d 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -40,8 +40,8 @@
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
+                                           InputCollection=["MCParticles"],
+                                           OutputCollection=["NewMCParticles"])
 
 producer = ExampleFunctionalProducer("Producer",
                                      OutputCollection="MCParticles")
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index d2e82f41..d148301b 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -26,13 +26,13 @@
 from k4FWCore import ApplicationMgr
 
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
-                                                   InputCollectionFloat="VectorFloat",
-                                                   InputCollectionParticles="MCParticles1",
-                                                   InputCollectionSimTrackerHits="SimTrackerHits",
-                                                   InputCollectionTrackerHits="TrackerHits",
-                                                   InputCollectionTracks="Tracks",
-                                                   OutputCollectionCounter="Counter",
-                                                   OutputCollectionParticles="NewMCParticles",
+                                                   InputCollectionFloat=["VectorFloat"],
+                                                   InputCollectionParticles=["MCParticles1"],
+                                                   InputCollectionSimTrackerHits=["SimTrackerHits"],
+                                                   InputCollectionTrackerHits=["TrackerHits"],
+                                                   InputCollectionTracks=["Tracks"],
+                                                   OutputCollectionCounter=["Counter"],
+                                                   OutputCollectionParticles=["NewMCParticles"],
                                                    Offset=10,
                                                    )
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
index e3df13d0..7f691efa 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
@@ -28,10 +28,10 @@
 #include <string>
 
 struct ExampleFunctionalTransformer final
-    : k4FWCore::Transformer<  edm4hep::MCParticleCollection(const edm4hep::MCParticleCollection&)> {
+    : k4FWCore::Transformer<edm4hep::MCParticleCollection(const edm4hep::MCParticleCollection&)> {
   ExampleFunctionalTransformer(const std::string& name, ISvcLocator* svcLoc)
-      : Transformer(name, svcLoc, KeyValue("InputCollection", "MCParticles"),
-                    KeyValue("OutputCollection", "NewMCParticles")) {}
+    : Transformer(name, svcLoc, {KeyValues("InputCollection", {"MCParticles"})},
+                  {KeyValues("OutputCollection", {"NewMCParticles"})}) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as all pointers to collections
@@ -54,8 +54,7 @@ struct ExampleFunctionalTransformer final
 
 private:
   // integer to add to the dummy values written to the edm
-  Gaudi::Property<int> m_offset{this, "Offset", 10,
-      "Integer to add to the dummy values written to the edm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalTransformer)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
index 1d354812..cf803ad3 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
@@ -55,10 +55,10 @@ struct ExampleFunctionalTransformerMultiple final
   ExampleFunctionalTransformerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : MultiTransformer(
             name, svcLoc,
-            {KeyValue("InputCollectionFloat", "VectorFloat"), KeyValue("InputCollectionParticles", "MCParticles1"),
-             KeyValue("InputCollectionSimTrackerHits", "SimTrackerHits"),
-             KeyValue("InputCollectionTrackerHits", "TrackerHits"), KeyValue("InputCollectionTracks", "Tracks")},
-            {KeyValue("OutputCollectionCounter", "Counter"), KeyValue("OutputCollectionParticles", "NewMCParticles")}) {
+            {KeyValues("InputCollectionFloat", {"VectorFloat"}), KeyValues("InputCollectionParticles", {"MCParticles1"}),
+             KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
+             KeyValues("InputCollectionTrackerHits", {"TrackerHits"}), KeyValues("InputCollectionTracks", {"Tracks"})},
+            {KeyValues("OutputCollectionCounter", {"Counter"}), KeyValues("OutputCollectionParticles", {"NewMCParticles"})}) {
   }
 
   // This is the function that will be called to transform the data

From 245939723fd81a530eadea948ed1cf8316f1e4a9 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 13 Feb 2024 10:36:05 +0100
Subject: [PATCH 020/127] Fix the producers

---
 .../src/components/ExampleFunctionalProducer.cpp         | 2 +-
 .../src/components/ExampleFunctionalProducerMultiple.cpp | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
index 4f4a41c9..c2a2e66f 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
@@ -30,7 +30,7 @@ struct ExampleFunctionalProducer final : k4FWCore::Producer<edm4hep::MCParticleC
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducer(const std::string& name, ISvcLocator* svcLoc)
-      : Producer(name, svcLoc, KeyValue("OutputCollection", "MCParticles")) {}
+    : Producer(name, svcLoc, {}, KeyValues("OutputCollection", {"MCParticles"})) {}
 
   // This is the function that will be called to produce the data
   edm4hep::MCParticleCollection operator()() const override {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index 05c9de28..eb5370d8 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -44,10 +44,11 @@ struct ExampleFunctionalProducerMultiple final
   ExampleFunctionalProducerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : Producer(
             name, svcLoc,
-            {KeyValue("OutputCollectionFloat", "VectorFloat"), KeyValue("OutputCollectionParticles1", "MCParticles1"),
-             KeyValue("OutputCollectionParticles2", "MCParticles2"),
-             KeyValue("OutputCollectionSimTrackerHits", "SimTrackerHits"),
-             KeyValue("OutputCollectionTrackerHits", "TrackerHits"), KeyValue("OutputCollectionTracks", "Tracks")}) {}
+            {},
+            {KeyValues("OutputCollectionFloat", {"VectorFloat"}), KeyValues("OutputCollectionParticles1", {"MCParticles1"}),
+             KeyValues("OutputCollectionParticles2", {"MCParticles2"}),
+             KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}),
+             KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}), KeyValues("OutputCollectionTracks", {"Tracks"})}) {}
 
   // This is the function that will be called to produce the data
   std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track> operator()() const override {

From eaeebb203594faafc9266d37f5c6fe3173397e26 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:45:29 +0100
Subject: [PATCH 021/127] Make all properties not be lists

---
 k4FWCore/include/k4FWCore/Consumer.h          | 34 ++++++++++-------
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 33 ++++++++++++++++
 ...ionalConsumerRuntimeCollectionsMultiple.py | 38 +++++++++----------
 3 files changed, 73 insertions(+), 32 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 3fec60a8..f8fd4539 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -52,6 +52,7 @@ namespace k4FWCore {
       // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
       // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(In)> m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
@@ -60,28 +61,33 @@ namespace k4FWCore {
       using KeyValues = typename base_class::KeyValues;
       using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
 
-      // Consumer(std::string name, ISvcLocator* locator, const std::vector<KeyValues>& inputs)
       template <typename IArgs, std::size_t... I>
       Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
-          : base_class(std::move(name), locator),
+        : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
                 this,
-                std::get<I>(inputs).first,
-                {DataObjID{std::get<I>(inputs).second[0]}},
-                [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<In>::value) {
-                    auto& handle  = std::get<I>(m_inputs);
-                    auto& ins     = m_inputLocations[I];
-                    handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
-                  }
-                },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
+                getName<In>(std::get<I>(inputs),false),
+                {DataObjID{std::get<I>(inputs).second[0]}}
             }
         ...},
-            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
+          m_inputLocationsPair{Gaudi::Property<DataObjID>{
+              this,
+              getName<In>(std::get<I>(inputs),true),
+              DataObjID{std::get<I>(inputs).second[0]},
+              [this](Gaudi::Details::PropertyBase&) {
+                if constexpr (!is_map_like<In>::value) {
+                  auto& handle  = std::get<I>(m_inputs);
+                  auto& ins     = m_inputLocationsPair[I];
+                  handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value(), this);
+                }
+              },
+              Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
+          }
+        ...},
             m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
 
       {
+
         // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
         //   // for (auto& value : std::get<I...>(inputs).second) {
         //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
@@ -95,6 +101,8 @@ namespace k4FWCore {
       }
 
       constexpr static std::size_t N_in = sizeof...(In);
+      constexpr static std::size_t N_inMap = CountType<In...>::value;
+      constexpr static std::size_t N_inPair = N_in - N_inMap;
 
       Consumer(std::string name, ISvcLocator* locator,
                Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const& inputs)
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 00de7d56..20012c2e 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -29,6 +29,39 @@ namespace k4FWCore {
 
     template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
 
+    static std::atomic<int> i = 0;
+
+    template <typename T, typename K>
+    std::string getName(const K& first, bool pair=false) {
+      // Gaudi::Algorithm::info() << "T is " << typeid(T).name() << endmsg;
+      // Gaudi::Algorithm::info() << "getName " << first.first << endmsg;
+      if constexpr (is_map_like<T>::value) {
+        if (pair) {
+          // Gaudi::Algorithm::info() << "returning " << "_" + std::to_string(i) << endmsg;
+          return "_" + std::to_string(i++);
+        }
+        // Gaudi::Algorithm::info() << "returning " << first.first << endmsg;
+        return first.first;
+      }
+      if (pair) {
+        // Gaudi::Algorithm::info() << "returning " << first.first << endmsg;
+        return first.first;
+      }
+      // Gaudi::Algorithm::info() << "returning " << "_" + std::to_string(i) << endmsg;
+      return "_" + std::to_string(i++);
+    }
+
+
+    template <typename... Args>
+    struct CountType {
+      static constexpr size_t value = 0;
+    };
+
+    template <typename First, typename... Rest>
+    struct CountType<First, Rest...> {
+      static constexpr size_t value = (is_map_like<First>::value ? 1 : 0) + CountType<Rest...>::value;
+    };
+
     template <typename T> struct ExtractInnerType;
 
     // Specialization for DataObjectReadHandle with map and shared_ptr
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index cc81b154..042bf7df 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -25,37 +25,37 @@
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducerMultiple("Producer0",
-                                              OutputCollectionFloat="VectorFloat0",
-                                              OutputCollectionParticles1="MCParticles0",
-                                              OutputCollectionParticles2="MCParticles1",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
-                                              OutputCollectionTrackerHits="TrackerHits0",
-                                              OutputCollectionTracks="Tracks0",
+                                              OutputCollectionFloat=["VectorFloat0"],
+                                              OutputCollectionParticles1=["MCParticles0"],
+                                              OutputCollectionParticles2=["MCParticles1"],
+                                              OutputCollectionSimTrackerHits=["SimTrackerHits0"],
+                                              OutputCollectionTrackerHits=["TrackerHits0"],
+                                              OutputCollectionTracks=["Tracks0"],
                                               ExampleInt=5
                                               )
 producer1 = ExampleFunctionalProducerMultiple("Producer1",
-                                              OutputCollectionFloat="VectorFloat1",
-                                              OutputCollectionParticles1="MCParticles2",
-                                              OutputCollectionParticles2="MCParticles3",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
-                                              OutputCollectionTrackerHits="TrackerHits1",
-                                              OutputCollectionTracks="Tracks1",
+                                              OutputCollectionFloat=["VectorFloat1"],
+                                              OutputCollectionParticles1=["MCParticles2"],
+                                              OutputCollectionParticles2=["MCParticles3"],
+                                              OutputCollectionSimTrackerHits=["SimTrackerHits1"],
+                                              OutputCollectionTrackerHits=["TrackerHits1"],
+                                              OutputCollectionTracks=["Tracks1"],
                                               ExampleInt=5
                                               )
 producer2 = ExampleFunctionalProducerMultiple("Producer2",
-                                              OutputCollectionFloat="VectorFloat2",
-                                              OutputCollectionParticles1="MCParticles4",
-                                              OutputCollectionParticles2="MCParticles5",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
-                                              OutputCollectionTrackerHits="TrackerHits2",
-                                              OutputCollectionTracks="Tracks2",
+                                              OutputCollectionFloat=["VectorFloat2"],
+                                              OutputCollectionParticles1=["MCParticles4"],
+                                              OutputCollectionParticles2=["MCParticles5"],
+                                              OutputCollectionSimTrackerHits=["SimTrackerHits2"],
+                                              OutputCollectionTrackerHits=["TrackerHits2"],
+                                              OutputCollectionTracks=["Tracks2"],
                                               ExampleInt=5
                                               )
 
 consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple("Consumer",
                                                                Particles=["MCParticles0", "MCParticles1", "MCParticles2", "MCParticles3", "MCParticles4"],
                                                                Tracks=["Tracks0", "Tracks1", "Tracks2"],
-                                                               SimTrackerHits=["SimTrackerHits0"],
+                                                               SimTrackerHits="SimTrackerHits0",
                                                                Offset=0,
                                                                )
 

From 6469a629e52c5bc1459abd1f3275ab013a6e7674 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:39:53 +0100
Subject: [PATCH 022/127] Add a hash function for hidden properties

---
 k4FWCore/include/k4FWCore/Consumer.h          | 52 +++++++------------
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 31 ++++-------
 ...pleFunctionalConsumerRuntimeCollections.py |  6 +--
 3 files changed, 31 insertions(+), 58 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index f8fd4539..273be974 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -27,8 +27,6 @@
 
 #include "k4FWCore/FunctionalUtils.h"
 
-// #include "GaudiKernel/CommonMessaging.h"
-
 #include <memory>
 #include <type_traits>
 #include <utility>
@@ -47,12 +45,11 @@ namespace k4FWCore {
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...> m_inputs;  //   and make the handles properties instead...
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
+          m_inputs;
       std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
-      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
-      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(In)> m_inputLocationsPair{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>              m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
@@ -63,31 +60,22 @@ namespace k4FWCore {
 
       template <typename IArgs, std::size_t... I>
       Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
-        : base_class(std::move(name), locator),
+          : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                getName<In>(std::get<I>(inputs),false),
-                {DataObjID{std::get<I>(inputs).second[0]}}
-            }
-        ...},
-          m_inputLocationsPair{Gaudi::Property<DataObjID>{
-              this,
-              getName<In>(std::get<I>(inputs),true),
-              DataObjID{std::get<I>(inputs).second[0]},
-              [this](Gaudi::Details::PropertyBase&) {
-                if constexpr (!is_map_like<In>::value) {
-                  auto& handle  = std::get<I>(m_inputs);
-                  auto& ins     = m_inputLocationsPair[I];
-                  handle        = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value(), this);
-                }
-              },
-              Gaudi::Details::Property::ImmediatelyInvokeHandler{true}
-          }
-        ...},
+                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
+            m_inputLocationsPair{Gaudi::Property<DataObjID>{
+                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
+                [this](Gaudi::Details::PropertyBase&) {
+                  if constexpr (!is_map_like<In>::value) {
+                    auto& handle = std::get<I>(m_inputs);
+                    auto& ins    = m_inputLocationsPair[I];
+                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value(), this);
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
             m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
 
       {
-
         // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
         //   // for (auto& value : std::get<I...>(inputs).second) {
         //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
@@ -100,12 +88,8 @@ namespace k4FWCore {
         // }
       }
 
-      constexpr static std::size_t N_in = sizeof...(In);
-      constexpr static std::size_t N_inMap = CountType<In...>::value;
-      constexpr static std::size_t N_inPair = N_in - N_inMap;
-
       Consumer(std::string name, ISvcLocator* locator,
-               Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const& inputs)
+               Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
           : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
 
       // When reading multiple collections we assume that the variable used is a
@@ -115,8 +99,8 @@ namespace k4FWCore {
       void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            
-            using EDM4hepType = typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            using EDM4hepType =
+                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
             auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
 
             // To be locked
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 20012c2e..159b06e4 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -2,13 +2,13 @@
 
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
-#include "GaudiKernel/CommonMessaging.h"
 
 #include "podio/CollectionBase.h"
 
 #include <stdexcept>
 #include <tuple>
 #include <type_traits>
+#include <unordered_map>
 
 namespace k4FWCore {
 
@@ -29,39 +29,28 @@ namespace k4FWCore {
 
     template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
 
-    static std::atomic<int> i = 0;
+    inline std::string hash_string(const std::string& str) {
+      std::string result = "_";
+      for (auto& c : std::to_string(std::hash<std::string>{}(str))) {
+        result += 'a' + (c - '0');
+      }
+      return result;
+    }
 
     template <typename T, typename K>
     std::string getName(const K& first, bool pair=false) {
-      // Gaudi::Algorithm::info() << "T is " << typeid(T).name() << endmsg;
-      // Gaudi::Algorithm::info() << "getName " << first.first << endmsg;
       if constexpr (is_map_like<T>::value) {
         if (pair) {
-          // Gaudi::Algorithm::info() << "returning " << "_" + std::to_string(i) << endmsg;
-          return "_" + std::to_string(i++);
+          return hash_string(first.first);
         }
-        // Gaudi::Algorithm::info() << "returning " << first.first << endmsg;
         return first.first;
       }
       if (pair) {
-        // Gaudi::Algorithm::info() << "returning " << first.first << endmsg;
         return first.first;
       }
-      // Gaudi::Algorithm::info() << "returning " << "_" + std::to_string(i) << endmsg;
-      return "_" + std::to_string(i++);
+      return hash_string(first.first);
     }
 
-
-    template <typename... Args>
-    struct CountType {
-      static constexpr size_t value = 0;
-    };
-
-    template <typename First, typename... Rest>
-    struct CountType<First, Rest...> {
-      static constexpr size_t value = (is_map_like<First>::value ? 1 : 0) + CountType<Rest...>::value;
-    };
-
     template <typename T> struct ExtractInnerType;
 
     // Specialization for DataObjectReadHandle with map and shared_ptr
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index f7be2442..addc8a09 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -25,13 +25,13 @@
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducer("Producer0",
-                                     OutputCollection="MCParticles0",
+                                     OutputCollection=["MCParticles0"],
                                      )
 producer1 = ExampleFunctionalProducer("Producer1",
-                                     OutputCollection="MCParticles1",
+                                     OutputCollection=["MCParticles1"],
                                      )
 producer2 = ExampleFunctionalProducer("Producer2",
-                                     OutputCollection="MCParticles2",
+                                     OutputCollection=["MCParticles2"],
                                      )
 consumer = ExampleFunctionalConsumerRuntimeCollections("Consumer",
                                      # InputCollection="MCParticles0 MCParticles1 MCParticles2",

From ec210e6e73e30fb746c3630b748c48064e238c12 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 14 Feb 2024 08:41:15 +0100
Subject: [PATCH 023/127] Simplify and clean up the consumer

---
 k4FWCore/include/k4FWCore/Consumer.h | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 273be974..bf0b2464 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -45,18 +45,15 @@ namespace k4FWCore {
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
-          m_inputs;
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                 m_inputs;
       std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>              m_inputLocationsPair{};
-      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                        m_inputLocations{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                     m_inputLocationsPair{};
+      mutable std::map<std::string, std::vector<std::string>>                                   m_inputLocationsMap;
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
-      using KeyValue  = typename base_class::KeyValue;
       using KeyValues = typename base_class::KeyValues;
-      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
 
       template <typename IArgs, std::size_t... I>
       Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
@@ -69,7 +66,7 @@ namespace k4FWCore {
                   if constexpr (!is_map_like<In>::value) {
                     auto& handle = std::get<I>(m_inputs);
                     auto& ins    = m_inputLocationsPair[I];
-                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value(), this);
+                    handle       = {ins.value(), this};
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
@@ -92,11 +89,8 @@ namespace k4FWCore {
                Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
           : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
 
-      // When reading multiple collections we assume that the variable used is a
-      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-      // and read the collections in a space separated string
       template <size_t Index, typename... Handles>
-      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+      void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             using EDM4hepType =
@@ -126,14 +120,14 @@ namespace k4FWCore {
           }
 
           // Recursive call for the next index
-          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+          readMapInputs<Index + 1>(handles);
         }
       }
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
+          readMapInputs<0>(this->m_inputs);
           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {

From 2f8884cfe10dbb17fabc10e091576c7fb002f799 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 14 Feb 2024 10:03:47 +0100
Subject: [PATCH 024/127] Fix tests to not only work with lists

---
 ...pleFunctionalConsumerRuntimeCollections.py |  6 ++--
 ...ionalConsumerRuntimeCollectionsMultiple.py | 36 +++++++++----------
 .../options/ExampleFunctionalFile.py          |  4 +--
 .../options/ExampleFunctionalMTFile.py        |  4 +--
 .../options/ExampleFunctionalMTMemory.py      |  6 ++--
 .../ExampleFunctionalMultipleMemory.py        | 24 ++++++-------
 6 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index addc8a09..f7be2442 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -25,13 +25,13 @@
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducer("Producer0",
-                                     OutputCollection=["MCParticles0"],
+                                     OutputCollection="MCParticles0",
                                      )
 producer1 = ExampleFunctionalProducer("Producer1",
-                                     OutputCollection=["MCParticles1"],
+                                     OutputCollection="MCParticles1",
                                      )
 producer2 = ExampleFunctionalProducer("Producer2",
-                                     OutputCollection=["MCParticles2"],
+                                     OutputCollection="MCParticles2",
                                      )
 consumer = ExampleFunctionalConsumerRuntimeCollections("Consumer",
                                      # InputCollection="MCParticles0 MCParticles1 MCParticles2",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 042bf7df..87ee0b43 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -25,30 +25,30 @@
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducerMultiple("Producer0",
-                                              OutputCollectionFloat=["VectorFloat0"],
-                                              OutputCollectionParticles1=["MCParticles0"],
-                                              OutputCollectionParticles2=["MCParticles1"],
-                                              OutputCollectionSimTrackerHits=["SimTrackerHits0"],
-                                              OutputCollectionTrackerHits=["TrackerHits0"],
-                                              OutputCollectionTracks=["Tracks0"],
+                                              OutputCollectionFloat="VectorFloat0",
+                                              OutputCollectionParticles1="MCParticles0",
+                                              OutputCollectionParticles2="MCParticles1",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
+                                              OutputCollectionTrackerHits="TrackerHits0",
+                                              OutputCollectionTracks="Tracks0",
                                               ExampleInt=5
                                               )
 producer1 = ExampleFunctionalProducerMultiple("Producer1",
-                                              OutputCollectionFloat=["VectorFloat1"],
-                                              OutputCollectionParticles1=["MCParticles2"],
-                                              OutputCollectionParticles2=["MCParticles3"],
-                                              OutputCollectionSimTrackerHits=["SimTrackerHits1"],
-                                              OutputCollectionTrackerHits=["TrackerHits1"],
-                                              OutputCollectionTracks=["Tracks1"],
+                                              OutputCollectionFloat="VectorFloat1",
+                                              OutputCollectionParticles1="MCParticles2",
+                                              OutputCollectionParticles2="MCParticles3",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
+                                              OutputCollectionTrackerHits="TrackerHits1",
+                                              OutputCollectionTracks="Tracks1",
                                               ExampleInt=5
                                               )
 producer2 = ExampleFunctionalProducerMultiple("Producer2",
-                                              OutputCollectionFloat=["VectorFloat2"],
-                                              OutputCollectionParticles1=["MCParticles4"],
-                                              OutputCollectionParticles2=["MCParticles5"],
-                                              OutputCollectionSimTrackerHits=["SimTrackerHits2"],
-                                              OutputCollectionTrackerHits=["TrackerHits2"],
-                                              OutputCollectionTracks=["Tracks2"],
+                                              OutputCollectionFloat="VectorFloat2",
+                                              OutputCollectionParticles1="MCParticles4",
+                                              OutputCollectionParticles2="MCParticles5",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
+                                              OutputCollectionTrackerHits="TrackerHits2",
+                                              OutputCollectionTracks="Tracks2",
                                               ExampleInt=5
                                               )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 87949d08..dc72581a 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -30,8 +30,8 @@
 svc.output = 'functional_transformer.root'
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection=["MCParticles"],
-                                           OutputCollection=["NewMCParticles"])
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
 
 mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index fe8153af..4d354857 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -45,8 +45,8 @@
 
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection=["MCParticles"],
-                                           OutputCollection=["NewMCParticles"])
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
 
 mgr = ApplicationMgr(TopAlg=[transformer],
                EvtSel="NONE",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index a68de06d..a4a2d969 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -40,14 +40,14 @@
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 
 transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection=["MCParticles"],
-                                           OutputCollection=["NewMCParticles"])
+                                           InputCollection="MCParticles",
+                                           OutputCollection="NewMCParticles")
 
 producer = ExampleFunctionalProducer("Producer",
                                      OutputCollection="MCParticles")
 
 consumer = ExampleFunctionalConsumer("Consumer",
-                                     InputCollection=["NewMCParticles"],
+                                     InputCollection="NewMCParticles",
                                      Offset=10,
                                      )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index d148301b..24cafc37 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -26,24 +26,24 @@
 from k4FWCore import ApplicationMgr
 
 transformer = ExampleFunctionalTransformerMultiple("Transformer",
-                                                   InputCollectionFloat=["VectorFloat"],
-                                                   InputCollectionParticles=["MCParticles1"],
-                                                   InputCollectionSimTrackerHits=["SimTrackerHits"],
-                                                   InputCollectionTrackerHits=["TrackerHits"],
-                                                   InputCollectionTracks=["Tracks"],
-                                                   OutputCollectionCounter=["Counter"],
-                                                   OutputCollectionParticles=["NewMCParticles"],
+                                                   InputCollectionFloat="VectorFloat",
+                                                   InputCollectionParticles="MCParticles1",
+                                                   InputCollectionSimTrackerHits="SimTrackerHits",
+                                                   InputCollectionTrackerHits="TrackerHits",
+                                                   InputCollectionTracks="Tracks",
+                                                   OutputCollectionCounter="Counter",
+                                                   OutputCollectionParticles="NewMCParticles",
                                                    Offset=10,
                                                    )
 
 producer = ExampleFunctionalProducerMultiple("Producer")
 
 consumer = ExampleFunctionalConsumerMultiple("Consumer",
-                                             InputCollectionFloat=["VectorFloat"],
-                                             InputCollectionParticles=["NewMCParticles"],
-                                             InputCollectionSimTrackerHits=["SimTrackerHits"],
-                                             InputCollectionTrackerHits=["TrackerHits"],
-                                             InputCollectionTracks=["Tracks"],
+                                             InputCollectionFloat="VectorFloat",
+                                             InputCollectionParticles="NewMCParticles",
+                                             InputCollectionSimTrackerHits="SimTrackerHits",
+                                             InputCollectionTrackerHits="TrackerHits",
+                                             InputCollectionTracks="Tracks",
                                              Offset=10,
                                              )
 

From 00664a52faa8309cb5b048749e56ff20c659ac8e Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 15 Feb 2024 09:06:26 +0100
Subject: [PATCH 025/127] Add reading and writing from any to any in the
 transformer

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |   7 +-
 k4FWCore/include/k4FWCore/Transformer.h       | 216 ++++++++++--------
 test/k4FWCoreTest/CMakeLists.txt              |   4 +-
 ...pleFunctionalProducerRuntimeCollections.py |   3 +-
 ...FunctionalTransformerRuntimeCollections.py |  61 +++++
 ...alTransformerRuntimeCollectionsMultiple.py | 101 ++++++++
 .../ExampleFunctionalProducerMultiple.cpp     |  36 ++-
 ...leFunctionalProducerRuntimeCollections.cpp |  16 +-
 ...unctionalTransformerRuntimeCollections.cpp |  57 +++++
 ...lTransformerRuntimeCollectionsMultiple.cpp | 198 ++++++++++++++++
 10 files changed, 571 insertions(+), 128 deletions(-)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 159b06e4..2b133e22 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -5,6 +5,7 @@
 
 #include "podio/CollectionBase.h"
 
+#include <memory>
 #include <stdexcept>
 #include <tuple>
 #include <type_traits>
@@ -53,12 +54,16 @@ namespace k4FWCore {
 
     template <typename T> struct ExtractInnerType;
 
-    // Specialization for DataObjectReadHandle with map and shared_ptr
     template <typename Value>
     struct ExtractInnerType<DataObjectReadHandle<std::map<std::string, std::shared_ptr<Value>>>> {
       using type = Value;
     };
 
+    template <typename Value>
+    struct ExtractInnerType<std::map<std::string, std::shared_ptr<Value>>> {
+      using type = Value;
+    };
+
     template <typename T>
     std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(
         const T& arg) {
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 64b904be..62732873 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -47,53 +47,47 @@ namespace k4FWCore {
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
       template <typename T>
-      using OutputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+      using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
-          m_inputs;  //   and make the handles properties instead...
-      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>>
-          m_outputs;  //   and make the handles properties instead...
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
+      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>>                   m_outputs;
       std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
       std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
-      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
-      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
-      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, 1> m_outputLocations{};
-      mutable std::map<std::string, std::vector<std::string>>            m_outputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
+      mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, 1>                                     m_outputLocations{};
+      std::array<Gaudi::Property<DataObjID>, 1>               m_outputLocationsPair{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
-      using KeyValue  = typename base_class::KeyValue;
       using KeyValues = typename base_class::KeyValues;
-      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
-      using OutKeys   = Gaudi::Functional::details::RepeatValues_<KeyValues, 1>;
 
       template <typename IArgs, typename OArgs, std::size_t... I, std::size_t... J>
       Transformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>,
-                       const OArgs& outputs, std::index_sequence<J...>)
+                  const OArgs& outputs, std::index_sequence<J...>)
           : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                std::get<I>(inputs).first,
-                {DataObjID{std::get<I>(inputs).second[0]}},
+                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
+            m_inputLocationsPair{Gaudi::Property<DataObjID>{
+                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
                 [this](Gaudi::Details::PropertyBase&) {
                   if constexpr (!is_map_like<In>::value) {
                     auto& handle = std::get<I>(m_inputs);
-                    auto& ins    = m_inputLocations[I];
-                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                    auto& ins    = m_inputLocationsPair[I];
+                    handle       = {ins.value(), this};
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
             m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                std::get<J>(outputs).first,
-                {DataObjID{std::get<J>(outputs).second[0]}},
+                this, getName<Out>(std::get<J>(outputs), false), {DataObjID{std::get<J>(outputs).second[0]}}}...},
+            m_outputLocationsPair{Gaudi::Property<DataObjID>{
+                this, getName<Out>(std::get<J>(outputs), true), DataObjID{std::get<J>(outputs).second[0]},
                 [this](Gaudi::Details::PropertyBase&) {
                   if constexpr (!is_map_like<Out>::value) {
                     auto& handle = std::get<J>(m_outputs);
-                    auto& ins    = m_outputLocations[J];
-                    handle       = OutputHandle_t<decltype(transformType(std::declval<Out>()))>(ins.value()[0], this);
+                    auto& ins    = m_outputLocationsPair[J];
+                    handle       = {ins.value(), this};
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
@@ -118,16 +112,12 @@ namespace k4FWCore {
       constexpr static std::size_t N_out = 1;
 
       Transformer(std::string name, ISvcLocator* locator,
-                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const&  inputs,
-                       Gaudi::Functional::details::RepeatValues_<KeyValues, N_out> const& outputs)
+                  Gaudi::Functional::details::RepeatValues_<KeyValues, N_in> const&  inputs,
+                  Gaudi::Functional::details::RepeatValues_<KeyValues, N_out> const& outputs)
           : Transformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
-                             std::index_sequence_for<Out>{}) {}
+                        std::index_sequence_for<Out>{}) {}
 
-      // When reading multiple collections we assume that the variable used is a
-      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-      // and read the collections in a space separated string
-      template <size_t Index, typename... Handles>
-      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             using EDM4hepType =
@@ -157,22 +147,59 @@ namespace k4FWCore {
           }
 
           // Recursive call for the next index
-          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+          readMapInputs<Index + 1>(handles);
+        }
+      }
+
+      template <size_t Index, typename... Handles> void putMapInputs(const std::tuple<Handles...>& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            using EDM4hepType =
+                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+
+            // To be locked
+            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+              auto vec = std::vector<std::string>();
+              vec.reserve(m_inputLocations[Index].value().size());
+              for (auto& val : m_inputLocations[Index].value()) {
+                vec.push_back(val.key());
+              }
+              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+            }
+
+            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
+              DataObject* p;
+              auto        sc = this->evtSvc()->retrieveObject(value, p);
+              if (!sc.isSuccess()) {
+                throw GaudiException("Failed to retrieve object " + value, "Transformer", StatusCode::FAILURE);
+              }
+              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+            }
+            std::get<Index>(handles).put(std::move(map));
+          }
+
+          // Recursive call for the next index
+          readMapInputs<Index + 1>(handles);
         }
       }
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
-          if constexpr (sizeof...(In) == 0) {
-            Gaudi::Functional::details::put(std::get<0>(this->m_outputs), ptrOrCast((*this)()));
-          } else if constexpr (std::tuple_size_v<Gaudi::Functional::details::filter_evtcontext<In...>> == 0) {
-            Gaudi::Functional::details::put(std::get<0>(this->m_outputs), (*this)(ctx));
+          readMapInputs<0>(this->m_inputs);
+          if constexpr (is_map_like<Out>::value) {
+            for (auto [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
+              auto        shared = std::dynamic_pointer_cast<podio::CollectionBase>(val);
+              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));
+              DataObject* p      = w;
+              auto        sc     = this->evtSvc()->registerObject(key, p);
+            }
           } else {
-            Gaudi::Functional::details::put(std::get<0>(this->m_outputs),
-                ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)))
-                                                );
+            Gaudi::Functional::details::put(
+                std::get<0>(this->m_outputs),
+                ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
           }
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
@@ -185,11 +212,6 @@ namespace k4FWCore {
       virtual Out operator()(const In&...) const = 0;
     };
 
-
-
-
-
-
     template <typename Signature, typename Traits_> struct MultiTransformer;
 
     template <typename... Out, typename... In, typename Traits_>
@@ -200,57 +222,50 @@ namespace k4FWCore {
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
       template <typename T>
-      using OutputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+      using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>
-          m_inputs;  //   and make the handles properties instead...
-      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>...>
-          m_outputs;  //   and make the handles properties instead...
+      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
+      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>...>                m_outputs;
       std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
       std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
-      // Gaudi::Property<std::vector<DataObjID>>                                      m_inputLocations;
-      // std::map<std::string, Gaudi::Property<std::vector<DataObjID>>>                                      m_inputLocations;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)> m_inputLocations{};
-      mutable std::map<std::string, std::vector<std::string>>            m_inputLocationsMap;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)> m_outputLocations{};
-      mutable std::map<std::string, std::vector<std::string>>            m_outputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
+      mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)>                        m_outputLocations{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(Out)>  m_outputLocationsPair{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
-      using KeyValue  = typename base_class::KeyValue;
       using KeyValues = typename base_class::KeyValues;
-      using InKeys    = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)>;
-      using OutKeys   = Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(Out)>;
 
       template <typename IArgs, typename OArgs, std::size_t... I, std::size_t... J>
       MultiTransformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>,
                        const OArgs& outputs, std::index_sequence<J...>)
           : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                std::get<I>(inputs).first,
-                {DataObjID{std::get<I>(inputs).second[0]}},
+                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
+            m_inputLocationsPair{Gaudi::Property<DataObjID>{
+                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
                 [this](Gaudi::Details::PropertyBase&) {
                   if constexpr (!is_map_like<In>::value) {
                     auto& handle = std::get<I>(m_inputs);
-                    auto& ins    = m_inputLocations[I];
-                    handle       = InputHandle_t<decltype(transformType(std::declval<In>()))>(ins.value()[0], this);
+                    auto& ins    = m_inputLocationsPair[I];
+                    handle       = {ins.value(), this};
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
             m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this,
-                std::get<J>(outputs).first,
-                {DataObjID{std::get<J>(outputs).second[0]}},
+                this, getName<Out>(std::get<J>(outputs), false), {DataObjID{std::get<J>(outputs).second[0]}}}...},
+            m_outputLocationsPair{Gaudi::Property<DataObjID>{
+                this, getName<Out>(std::get<J>(outputs), true), DataObjID{std::get<J>(outputs).second[0]},
                 [this](Gaudi::Details::PropertyBase&) {
                   if constexpr (!is_map_like<Out>::value) {
                     auto& handle = std::get<J>(m_outputs);
-                    auto& ins    = m_outputLocations[J];
-                    handle       = OutputHandle_t<decltype(transformType(std::declval<Out>()))>(ins.value()[0], this);
+                    auto& ins    = m_outputLocationsPair[J];
+                    handle       = {ins.value(), this};
                   }
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
-            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
             m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...},
             m_outputs{OutputHandle_t<decltype(transformType(std::declval<Out>()))>(std::get<J>(outputs).first, this)...}
 
@@ -276,11 +291,7 @@ namespace k4FWCore {
           : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                              std::index_sequence_for<Out...>{}) {}
 
-      // When reading multiple collections we assume that the variable used is a
-      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-      // and read the collections in a space separated string
-      template <size_t Index, typename... Handles>
-      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
+      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             using EDM4hepType =
@@ -310,32 +321,43 @@ namespace k4FWCore {
           }
 
           // Recursive call for the next index
-          transformAndApplyAlgoAtIndex<Index + 1>(handles);
+          readMapInputs<Index + 1>(handles);
+        }
+      }
+
+      template <size_t Index, typename... Handles> void putMapOutputs(std::tuple<Handles...>&& handles) const {
+        if constexpr (Index < sizeof...(Handles)) {
+          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
+            using EDM4hepType =
+                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
+            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+
+            for (auto [key, val] : std::get<Index>(handles)) {
+              auto        shared = std::dynamic_pointer_cast<podio::CollectionBase>(val);
+              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));
+              DataObject* p      = w;
+              auto        sc     = this->evtSvc()->registerObject(key, p);
+            }
+
+          }
+          else {
+            Gaudi::Functional::details::put(
+                std::get<Index>(m_outputs),
+                ptrOrCast(std::move(std::get<Index>(handles))));
+          }
+
+          // Recursive call for the next index
+          putMapOutputs<Index + 1>(std::move(handles));
         }
       }
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
-          std::apply(
-              [this, &ctx](auto&... ohandle) {
-                if constexpr (sizeof...(In) == 0) {
-                  std::apply(
-                      [&ohandle...](auto&&... data) {
-                        (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
-                      },
-                      (*this)());
-                } else {
-                  std::apply(
-                      [&ohandle...](auto&&... data) {
-                        // (Gaudi::Functional::details::put(ohandle, std::forward<decltype(data)>(data)), ...);
-                        (Gaudi::Functional::details::put(ohandle, ptrOrCast(std::forward<decltype(data)>(data))), ...);
-                      },
-                      filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs));
-                }
-              },
-              this->m_outputs);
+          readMapInputs<0>(this->m_inputs);
+
+          auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+          putMapOutputs<0>(std::move(tmp));
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index c6a89158..2b276569 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -121,6 +121,8 @@ add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS F
 add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
 add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
-add_test_with_env(FunctionalRuntimeCollections options/ExampleFunctionalRuntimeCollections.py)
+add_test_with_env(FunctionalProducerRuntimeCollections options/ExampleFunctionalProducerRuntimeCollections.py)
+add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctionalTransformerRuntimeCollections.py)
+add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
index 9e6ccf41..1d9ac227 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -25,7 +25,8 @@
 from Configurables import EventDataSvc
 
 producer = ExampleFunctionalProducerRuntimeCollections("Producer",
-                                                       NumberOfCollections=3)
+                                                       OutputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
+                                                       )
 
 consumer0 = ExampleFunctionalConsumer("Consumer0",
                                      InputCollection="MCParticles0",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
new file mode 100644
index 00000000..5802d48c
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformerRuntimeCollections, ExampleFunctionalConsumer, ExampleFunctionalProducer
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+producer0 = ExampleFunctionalProducer("Producer0",
+                                     OutputCollection="MCParticles0",
+                                     )
+producer1 = ExampleFunctionalProducer("Producer1",
+                                     OutputCollection="MCParticles1",
+                                     )
+producer2 = ExampleFunctionalProducer("Producer2",
+                                     OutputCollection="MCParticles2",
+                                     )
+
+transformer = ExampleFunctionalTransformerRuntimeCollections("Transformer",
+                                                       InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
+                                                       OutputCollections=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
+                                                       )
+
+consumer0 = ExampleFunctionalConsumer("Consumer0",
+                                     InputCollection="NewMCParticles0",
+                                     Offset=0,
+                                     )
+consumer1 = ExampleFunctionalConsumer("Consumer1",
+                                     InputCollection="NewMCParticles1",
+                                     Offset=0,
+                                     )
+consumer2 = ExampleFunctionalConsumer("Consumer2",
+                                     InputCollection="NewMCParticles2",
+                                     Offset=0,
+                                     )
+
+
+ApplicationMgr(TopAlg=[producer0, producer1, producer2, transformer, consumer0, consumer1, consumer2],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
new file mode 100644
index 00000000..8caaee21
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
@@ -0,0 +1,101 @@
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalTransformerRuntimeCollectionsMultiple, ExampleFunctionalConsumerMultiple, ExampleFunctionalProducerMultiple
+from Configurables import ApplicationMgr
+from Configurables import EventDataSvc
+
+producer0 = ExampleFunctionalProducerMultiple("Producer0",
+                                              OutputCollectionFloat="VectorFloat0",
+                                              OutputCollectionParticles1="MCParticles0",
+                                              OutputCollectionParticles2="MCParticles1",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
+                                              OutputCollectionTrackerHits="TrackerHits0",
+                                              OutputCollectionTracks="Tracks0",
+                                              ExampleInt=5
+                                              )
+producer1 = ExampleFunctionalProducerMultiple("Producer1",
+                                              OutputCollectionFloat="VectorFloat1",
+                                              OutputCollectionParticles1="MCParticles2",
+                                              OutputCollectionParticles2="MCParticles3",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
+                                              OutputCollectionTrackerHits="TrackerHits1",
+                                              OutputCollectionTracks="Tracks1",
+                                              ExampleInt=5
+                                              )
+producer2 = ExampleFunctionalProducerMultiple("Producer2",
+                                              OutputCollectionFloat="VectorFloat2",
+                                              OutputCollectionParticles1="MCParticles4",
+                                              OutputCollectionParticles2="MCParticles5",
+                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
+                                              OutputCollectionTrackerHits="TrackerHits2",
+                                              OutputCollectionTracks="Tracks2",
+                                              ExampleInt=5
+                                              )
+
+transformer = ExampleFunctionalTransformerRuntimeCollectionsMultiple("Transformer",
+                                                       InputCollectionFloat=["VectorFloat0", "VectorFloat1", "VectorFloat2"],
+                                                       InputCollectionParticles=["MCParticles0", "MCParticles2", "MCParticles4"],
+                                                       InputCollectionSimTrackerHits=["SimTrackerHits0", "SimTrackerHits1", "SimTrackerHits2"],
+                                                       InputCollectionTrackerHits=["TrackerHits0", "TrackerHits1", "TrackerHits2"],
+                                                       InputCollectionTracks=["Tracks0", "Tracks1", "Tracks2"],
+                                                       OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],              
+                                                       OutputCollectionParticles1=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
+                                                       OutputCollectionParticles2=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
+                                                       OutputCollectionSimTrackerHits=["NewSimTrackerHits0", "NewSimTrackerHits1", "NewSimTrackerHits2"],           
+                                                       OutputCollectionTrackerHits=["NewTrackerHits0", "NewTrackerHits1", "NewTrackerHits2"],
+                                                       OutputCollectionTracks=["NewTracks0", "NewTracks1", "NewTracks2"],
+                                                       Offset=0,
+                                                       )
+
+consumer0 = ExampleFunctionalConsumerMultiple("Consumer0",
+                                             InputCollectionFloat="NewVectorFloat0",
+                                             InputCollectionParticles="NewMCParticles0",
+                                             InputCollectionSimTrackerHits="NewSimTrackerHits0",
+                                             InputCollectionTrackerHits="NewTrackerHits0",
+                                             InputCollectionTracks="NewTracks0",
+                                             )
+
+consumer1 = ExampleFunctionalConsumerMultiple("Consumer1",
+                                                InputCollectionFloat="NewVectorFloat1",
+                                                InputCollectionParticles="NewMCParticles2",
+                                                InputCollectionSimTrackerHits="NewSimTrackerHits1",
+                                                InputCollectionTrackerHits="NewTrackerHits1",
+                                                InputCollectionTracks="NewTracks1",
+                                                )
+
+consumer2 = ExampleFunctionalConsumerMultiple("Consumer2",
+                                                InputCollectionFloat="NewVectorFloat2",
+                                                InputCollectionParticles="NewMCParticles4",
+                                                InputCollectionSimTrackerHits="NewSimTrackerHits2",
+                                                InputCollectionTrackerHits="NewTrackerHits2",
+                                                InputCollectionTracks="NewTracks2",
+                                                )
+
+
+ApplicationMgr(TopAlg=[producer0, producer1, producer2, transformer,
+                       consumer0, consumer1, consumer2],
+               EvtSel="NONE",
+               EvtMax=10,
+               ExtSvc=[EventDataSvc("EventDataSvc")],
+               OutputLevel=INFO,
+               )
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index eb5370d8..fe521b49 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -31,27 +31,25 @@
 #include <string>
 
 // Which type of collections we are producing
-using Float         = podio::UserDataCollection<float>;
-using Particle      = edm4hep::MCParticleCollection;
-using SimTrackerHit = edm4hep::SimTrackerHitCollection;
-using TrackerHit    = edm4hep::TrackerHitCollection;
-using Track         = edm4hep::TrackCollection;
-
-struct ExampleFunctionalProducerMultiple final
-  : k4FWCore::Producer<std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track>()> {
+
+using retType =
+    std::tuple<podio::UserDataCollection<float>, edm4hep::MCParticleCollection, edm4hep::MCParticleCollection,
+               edm4hep::SimTrackerHitCollection, edm4hep::TrackerHitCollection, edm4hep::TrackCollection>;
+
+struct ExampleFunctionalProducerMultiple final : k4FWCore::Producer<retType()> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the output collections
   ExampleFunctionalProducerMultiple(const std::string& name, ISvcLocator* svcLoc)
-      : Producer(
-            name, svcLoc,
-            {},
-            {KeyValues("OutputCollectionFloat", {"VectorFloat"}), KeyValues("OutputCollectionParticles1", {"MCParticles1"}),
-             KeyValues("OutputCollectionParticles2", {"MCParticles2"}),
-             KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}),
-             KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}), KeyValues("OutputCollectionTracks", {"Tracks"})}) {}
+      : Producer(name, svcLoc, {},
+                 {KeyValues("OutputCollectionFloat", {"VectorFloat"}),
+                  KeyValues("OutputCollectionParticles1", {"MCParticles1"}),
+                  KeyValues("OutputCollectionParticles2", {"MCParticles2"}),
+                  KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}),
+                  KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}),
+                  KeyValues("OutputCollectionTracks", {"Tracks"})}) {}
 
   // This is the function that will be called to produce the data
-  std::tuple<Float, Particle, Particle, SimTrackerHit, TrackerHit, Track> operator()() const override {
+  retType operator()() const override {
     // The following was copied and adapted from the
     // k4FWCoreTest_CreateExampleEventData test
 
@@ -60,7 +58,7 @@ struct ExampleFunctionalProducerMultiple final
     floatVector.push_back(25.);
     floatVector.push_back(m_event);
 
-    auto particles = edm4hep::MCParticleCollection();
+    auto              particles = edm4hep::MCParticleCollection();
     edm4hep::Vector3d v{0, 0, 0};
     edm4hep::Vector3f vv{0, 0, 0};
     particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, vv);
@@ -91,8 +89,8 @@ struct ExampleFunctionalProducerMultiple final
     track.addToTrackerHits(trackerHit);
     track.addToTracks(track2);
 
-    return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits),
-                           std::move(trackerHits), std::move(tracks));
+    return std::make_tuple(std::move(floatVector), std::move(particles), edm4hep::MCParticleCollection(),
+                           std::move(simTrackerHits), std::move(trackerHits), std::move(tracks));
   }
 
 private:
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
index 01b4f0fc..c3055a7d 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
@@ -18,30 +18,28 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Producer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 
-// #include "k4FWCore/ProducerMany.h"
-#include "k4FWCore/TransformerMany.h"
+#include "k4FWCore/Producer.h"
 
 #include <string>
 
-struct ExampleFunctionalProducerRuntimeCollections final : k4FWCore::ProducerMany<std::map<std::string, std::shared_ptr<podio::CollectionBase>>()> {
+struct ExampleFunctionalProducerRuntimeCollections final : k4FWCore::Producer<std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>()> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
-      : ProducerMany(name, svcLoc, KeyValue("OutputCollection", "MCParticles")) {}
+    : Producer(name, svcLoc, {}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
-  std::map<std::string, std::shared_ptr<podio::CollectionBase>> operator()() const override {
-    std::map<std::string, std::shared_ptr<podio::CollectionBase>> m_outputCollections;
+  std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> operator()() const override {
+    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
     for (int i = 0; i < m_numberOfCollections; ++i) {
       std::string name = "MCParticles" + std::to_string(i);
       auto coll = std::make_shared<edm4hep::MCParticleCollection>();
       coll->create(1, 2, 3, 4.f, 5.f, 6.f);
       coll->create(2, 3, 4, 5.f, 6.f, 7.f);
-      m_outputCollections[name] = std::dynamic_pointer_cast<podio::CollectionBase>(coll);
+      m_outputCollections[name] = coll;
     }
     return m_outputCollections;
 
@@ -50,7 +48,7 @@ struct ExampleFunctionalProducerRuntimeCollections final : k4FWCore::ProducerMan
 private:
   // We can define any property we want that can be set from python
   // and use it inside operator()
-  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 1, "Example int that can be used in the algorithm"};
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3, "Example int that can be used in the algorithm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalProducerRuntimeCollections)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
new file mode 100644
index 00000000..ecf6b491
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+
+#include "edm4hep/MCParticleCollection.h"
+
+#include "k4FWCore/Transformer.h"
+
+#include <string>
+
+using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
+
+struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const mapType& input)> {
+  // The pair in KeyValue can be changed from python and it corresponds
+  // to the name of the output collection
+  ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
+    : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
+
+  // This is the function that will be called to produce the data
+  mapType operator()(const mapType& input) const override {
+    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
+    for (int i = 0; i < input.size(); ++i) {
+      std::string name = "NewMCParticles" + std::to_string(i);
+      auto old_coll = input.at("MCParticles" + std::to_string(i));
+      auto coll = std::make_shared<edm4hep::MCParticleCollection>();
+      coll->push_back(old_coll->at(0).clone());
+      coll->push_back(old_coll->at(1).clone());
+      m_outputCollections[name] = coll;
+    }
+    return m_outputCollections;
+
+  }
+
+private:
+  // We can define any property we want that can be set from python
+  // and use it inside operator()
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3, "Example int that can be used in the algorithm"};
+};
+
+DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeCollections)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
new file mode 100644
index 00000000..60790743
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+
+#include "edm4hep/MCParticleCollection.h"
+#include "edm4hep/SimTrackerHitCollection.h"
+#include "edm4hep/TrackCollection.h"
+#include "edm4hep/TrackerHitCollection.h"
+#include "podio/UserDataCollection.h"
+
+#include "k4FWCore/Transformer.h"
+
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+// Which type of collection we are reading
+using FloatColl         = std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>;
+using ParticleColl      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
+using SimTrackerHitColl = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>;
+using TrackerHitColl    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>;
+using TrackColl         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>;
+
+using retType = std::tuple<std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>>;
+
+struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
+    : k4FWCore::MultiTransformer<retType(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
+                                         const TrackerHitColl&, const TrackColl&)> {
+  // The pairs in KeyValue can be changed from python and they correspond
+  // to the names of the input collection
+  ExampleFunctionalTransformerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc)
+      : MultiTransformer(name, svcLoc,
+                         {
+                             KeyValues("InputCollectionFloat", {"VectorFloat"}),
+                             KeyValues("InputCollectionParticles", {"MCParticles1"}),
+                             KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
+                             KeyValues("InputCollectionTrackerHits", {"TrackerHits"}),
+                             KeyValues("InputCollectionTracks", {"Tracks"}),
+                         },
+                         {
+                             KeyValues("OutputCollectionFloat", {"VectorFloat"}),
+                             KeyValues("OutputCollectionParticles1", {"MCParticles1"}),
+                             KeyValues("OutputCollectionParticles2", {"MCParticles2"}),
+                             KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}),
+                             KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}),
+                             KeyValues("OutputCollectionTracks", {"Tracks"}),
+                         }) {}
+
+  // This is the function that will be called to transform the data
+  // Note that the function has to be const, as well as the collections
+  // we get from the input
+  retType operator()(const FloatColl& floatMap, const ParticleColl& particlesMap,
+                     const SimTrackerHitColl& simTrackerHitMap, const TrackerHitColl& trackerHitMap,
+                     const TrackColl& trackMap) const override {
+    auto floatMapOut         = std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>();
+    auto particleMapOut      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
+    auto particle2MapOut     = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
+    auto simTrackerHitMapOut = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>();
+    auto trackerHitMapOut    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>();
+    auto trackMapOut         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>();
+
+    if (floatMap.size() != 3) {
+      throw std::runtime_error("Wrong size of the floatVector collection map, expected 3, got " +
+                               std::to_string(floatMap.size()) + "");
+    }
+    for (const auto& [key, floatVector] : floatMap) {
+      if (floatVector->size() != 3) {
+        throw std::runtime_error("Wrong size of floatVector collection, expected 3, got " +
+                                 std::to_string(floatVector->size()) + "");
+      }
+      if ((floatVector->vec()[0] != 125) || (floatVector->vec()[1] != 25) || (floatVector->vec()[2] != 0)) {
+        std::stringstream error;
+        error << "Wrong data in floatVector collection, expected 125, 25, " << 0 << " got " << floatVector->vec()[0]
+              << ", " << floatVector->vec()[1] << ", " << floatVector->vec()[2] << "";
+        throw std::runtime_error(error.str());
+      }
+      auto ptr = std::make_shared<podio::UserDataCollection<float>>();
+      ptr->push_back(floatVector->vec()[0]);
+      ptr->push_back(floatVector->vec()[1]);
+      ptr->push_back(floatVector->vec()[2]);
+      floatMapOut["New" + key] = floatVector;
+    }
+
+    if (particlesMap.size() != 3) {
+      throw std::runtime_error("Wrong size of the particleMap map, expected 3, got " +
+                               std::to_string(particleMapOut.size()) + "");
+    }
+
+    for (auto& [key, particles] : particlesMap) {
+      auto ptr = std::make_shared<edm4hep::MCParticleCollection>();
+      int  i   = 0;
+      for (const auto& particle : *particles) {
+        if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+            (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
+            (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
+          std::stringstream error;
+          error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset
+                << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", "
+                << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", "
+                << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
+                << particle.getMass() << "";
+          throw std::runtime_error(error.str());
+          ptr->push_back(particle.clone());
+        }
+        i++;
+        particleMapOut["New" + key] = ptr;
+      }
+    }
+
+    if (simTrackerHitMap.size() != 3) {
+      throw std::runtime_error("Wrong size of the simTrackerHitMap map, expected 3, got " +
+                               std::to_string(simTrackerHitMapOut.size()) + "");
+    }
+
+    for (auto& [key, simTrackerHits] : simTrackerHitMap) {
+      auto ptr = std::make_shared<edm4hep::SimTrackerHitCollection>();
+      if ((simTrackerHits->at(0).getPosition()[0] != 3) || (simTrackerHits->at(0).getPosition()[1] != 4) ||
+          (simTrackerHits->at(0).getPosition()[2] != 5)) {
+        std::stringstream error;
+        error << "Wrong data in simTrackerHits collection, expected 3, 4, 5 got "
+              << simTrackerHits->at(0).getPosition()[0] << ", " << simTrackerHits->at(0).getPosition()[1] << ", "
+              << simTrackerHits->at(0).getPosition()[2] << "";
+        throw std::runtime_error(error.str());
+      }
+      ptr->push_back(simTrackerHits->at(0).clone());
+      simTrackerHitMapOut["New" + key] = ptr;
+    }
+
+    if (trackerHitMap.size() != 3) {
+      throw std::runtime_error("Wrong size of the trackerHitMap map, expected 3, got " +
+                               std::to_string(trackerHitMapOut.size()) + "");
+    }
+
+    for (auto& [key, trackerHits] : trackerHitMap) {
+      auto ptr = std::make_shared<edm4hep::TrackerHitCollection>();
+      if ((trackerHits->at(0).getPosition()[0] != 3) || (trackerHits->at(0).getPosition()[1] != 4) ||
+          (trackerHits->at(0).getPosition()[2] != 5)) {
+        std::stringstream error;
+        error << "Wrong data in trackerHits collection, expected 3, 4, 5 got " << trackerHits->at(0).getPosition()[0]
+              << ", " << trackerHits->at(0).getPosition()[1] << ", " << trackerHits->at(0).getPosition()[2] << "";
+        throw std::runtime_error(error.str());
+      }
+      ptr->push_back(trackerHits->at(0).clone());
+      trackerHitMapOut["New" + key] = ptr;
+    }
+
+    if (trackMap.size() != 3) {
+      throw std::runtime_error("Wrong size of the trackMap map, expected 3, got " + std::to_string(trackMapOut.size()) +
+                               "");
+    }
+
+    for (auto& [key, tracks] : trackMap) {
+      auto ptr = std::make_shared<edm4hep::TrackCollection>();
+      if ((tracks->at(0).getType() != 1) || (std::abs(tracks->at(0).getChi2() - 2.1) > 1e-6) ||
+          (tracks->at(0).getNdf() != 3) || (std::abs(tracks->at(0).getDEdx() - 4.1) > 1e-6) ||
+          (std::abs(tracks->at(0).getDEdxError() - 5.1) > 1e-6) ||
+          (std::abs(tracks->at(0).getRadiusOfInnermostHit() - 6.1) > 1e-6)) {
+        std::stringstream error;
+        error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks->at(0).getType()
+              << ", " << tracks->at(0).getChi2() << ", " << tracks->at(0).getNdf() << ", " << tracks->at(0).getDEdx()
+              << ", " << tracks->at(0).getDEdxError() << ", " << tracks->at(0).getRadiusOfInnermostHit() << "";
+        throw std::runtime_error(error.str());
+      }
+      ptr->push_back(tracks->at(0).clone());
+      trackMapOut["New" + key] = ptr;
+    }
+
+    return std::make_tuple(std::move(floatMapOut), std::move(particleMapOut), std::move(particle2MapOut),
+                           std::move(simTrackerHitMapOut), std::move(trackerHitMapOut), std::move(trackMapOut));
+  }
+
+private:
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
+};
+
+DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeCollectionsMultiple)

From 1c87a648e299092f5d7256262fe0cbe0e2cfd66f Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 15 Feb 2024 09:12:13 +0100
Subject: [PATCH 026/127] Use ROOTWriter

---
 k4FWCore/components/IIOSvc.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 367fd044..fe767612 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -13,7 +13,7 @@
 #include "GaudiKernel/IInterface.h"
 
 #include "podio/CollectionBase.h"
-#include "podio/ROOTFrameWriter.h"
+#include "podio/ROOTWriter.h"
 
 #include <exception>
 #include <memory>
@@ -36,7 +36,7 @@ class IIOSvc : virtual public IInterface {
   virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next( ) = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
-  virtual std::shared_ptr<podio::ROOTFrameWriter> getWriter() = 0;
+  virtual std::shared_ptr<podio::ROOTWriter> getWriter() = 0;
   virtual void deleteWriter() = 0;
   virtual void deleteReader() = 0;
   virtual bool writeCollection( const std::string& collName) = 0;

From 7ba44197277e7d065855f357624bc435c27db592 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 15 Feb 2024 14:23:47 +0100
Subject: [PATCH 027/127] Format

---
 k4FWCore/components/IIOSvc.h   | 49 +++++++++++++++++++++-------------
 k4FWCore/components/Reader.cpp |  2 +-
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index fe767612..4ac88d0c 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -1,14 +1,23 @@
-/*****************************************************************************\
-* (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration      *
-*                                                                             *
-* This software is distributed under the terms of the GNU General Public      *
-* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   *
-*                                                                             *
-* In applying this licence, CERN does not waive the privileges and immunities *
-* granted to it by virtue of its status as an Intergovernmental Organization  *
-* or submit itself to any jurisdiction.                                       *
-\*****************************************************************************/
-#pragma once
+/*
+ * Copyright (c) 2014-2023 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FWCORE_IIOSVC_H
+#define FWCORE_IIOSVC_H
 
 #include "GaudiKernel/IInterface.h"
 
@@ -23,21 +32,23 @@
  * The interface implemented by any class making IO and reading RawEvent Data
  */
 class IIOSvc : virtual public IInterface {
-
 public:
   struct EndOfInput : std::logic_error {
-    EndOfInput() : logic_error( "Reached end of input while more data were expected" ){};
+    EndOfInput() : logic_error("Reached end of input while more data were expected"){};
   };
 
 public:
   /// InterfaceID
-  DeclareInterfaceID( IIOSvc, 1, 0 );
+  DeclareInterfaceID(IIOSvc, 1, 0);
 
-  virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next( ) = 0;
+  virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame>
+                                                    next()                     = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
-  virtual std::shared_ptr<podio::ROOTWriter> getWriter() = 0;
-  virtual void deleteWriter() = 0;
-  virtual void deleteReader() = 0;
-  virtual bool writeCollection( const std::string& collName) = 0;
+  virtual std::shared_ptr<podio::ROOTWriter> getWriter()                                  = 0;
+  virtual void                               deleteWriter()                               = 0;
+  virtual void                               deleteReader()                               = 0;
+  virtual bool                               writeCollection(const std::string& collName) = 0;
 };
+
+#endif
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index f8aaa640..4f691cf9 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -23,11 +23,11 @@
 #include "GaudiKernel/AnyDataWrapper.h"
 #include "GaudiKernel/IDataProviderSvc.h"
 
-#include "IIOSvc.h"
 
 #include "podio/CollectionBase.h"
 #include "podio/Frame.h"
 
+#include "IIOSvc.h"
 #include "k4FWCore/FunctionalUtils.h"
 
 #include <memory>

From d1b2770843db1d7ddf71a01d4e91134180d5ddbd Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 15 Feb 2024 15:48:33 +0100
Subject: [PATCH 028/127] Comment out the RNTuple readers and writers

---
 k4FWCore/components/IOSvc.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 54cf66a8..6acc83be 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -28,8 +28,8 @@
 
 #include "podio/ROOTReader.h"
 #include "podio/ROOTWriter.h"
-#include "podio/RNTupleReader.h"
-#include "podio/RNTupleWriter.h"
+// #include "podio/RNTupleReader.h"
+// #include "podio/RNTupleWriter.h"
 
 #include "k4FWCore/KeepDropSwitch.h"
 

From 21d5b3b76689445647966969f0f7fe44c0b0d4df Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 08:31:26 +0100
Subject: [PATCH 029/127] Use PROJECT_NAME

---
 test/k4FWCoreTest/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 2b276569..46bee8a3 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -29,8 +29,8 @@ set(K4RUN ${PROJECT_SOURCE_DIR}/k4FWCore/scripts/k4run)
 
 function(set_test_env testname)
   set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "ROOT_INCLUDE_PATH=$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>/../include>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>/../include>:$ENV{ROOT_INCLUDE_PATH}")
-  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${CMAKE_PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
-  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${CMAKE_PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
+  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
+  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
 endfunction()
 
 function(add_test_with_env testname)

From d617e8815a97bcc00a0fda820fee1d2965f4e572 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 08:31:47 +0100
Subject: [PATCH 030/127] Change the order of the python directory for tests

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index df46b4d3..a0b68b67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,11 +73,11 @@ endif()
 
 add_subdirectory(k4FWCore)
 add_subdirectory(k4Interface)
+add_subdirectory(python)
 if(BUILD_TESTING)
   add_subdirectory(test/k4FWCoreTest)
 endif()
 
-add_subdirectory(python)
 
 option(ENABLE_CPACK "Whether or not to use cpack config" OFF)
 if(ENABLE_CPACK)

From 926f7c26e3448c8772054c9d30563bbe9f681709 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 08:43:35 +0100
Subject: [PATCH 031/127] Install the python files to one of the build
 directories

---
 python/CMakeLists.txt | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index b076d15e..6b4c6f72 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -1,6 +1,10 @@
-install(FILES ${to_install} DESTINATION ${podio_PYTHON_INSTALLDIR})
-
-install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore
-  DESTINATION python
-  )
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
 
+# The following is done to make the tests work without installing the files in
+# the installation directory. The k4FWCore in the build directory is populated by
+# Gaudi so we can't just make a k4FWCore folder and put the python files there.
+# We put everything in one of the folders generated by Gaudi so that importing from
+# k4FWCore will give the right result
+file(GLOB_RECURSE python_files ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/*.py
+     EXCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/__init__.py)
+install(FILES ${python_files} DESTINATION ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)

From 2b05dda31916f180aaade2f60ea5c412e586d03d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 09:02:19 +0100
Subject: [PATCH 032/127] Use CMAKE_CURRENT_LIST_DIR for this test to make
 dependent tests pass

---
 test/k4FWCoreTest/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 46bee8a3..6432abe4 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -71,6 +71,7 @@ set_property(TEST ReadExampleEventData APPEND PROPERTY DEPENDS CreateExampleEven
 add_test_with_env(ReadExampleDataFromNthEvent options/readExampleDataFromNthEvent.py PROPERTIES DEPENDS CreateExampleEventData)
 
 add_test_with_env(AlgorithmWithTFile options/TestAlgorithmWithTFile.py)
+set_property(TEST AlgorithmWithTFile WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
 add_test(NAME AlgorithmWithTFileCheckFrameworkOutput
          WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
          COMMAND  python scripts/check_TestAlgorithmWithTFile_framework_nonempty.py)

From 783debb4e29a66fff4735e9f8fedd3308ec6ce26 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 09:03:13 +0100
Subject: [PATCH 033/127] Fix syntax

---
 test/k4FWCoreTest/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 6432abe4..599deb00 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -71,7 +71,7 @@ set_property(TEST ReadExampleEventData APPEND PROPERTY DEPENDS CreateExampleEven
 add_test_with_env(ReadExampleDataFromNthEvent options/readExampleDataFromNthEvent.py PROPERTIES DEPENDS CreateExampleEventData)
 
 add_test_with_env(AlgorithmWithTFile options/TestAlgorithmWithTFile.py)
-set_property(TEST AlgorithmWithTFile WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+set_property(TEST AlgorithmWithTFile PROPERTY WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
 add_test(NAME AlgorithmWithTFileCheckFrameworkOutput
          WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
          COMMAND  python scripts/check_TestAlgorithmWithTFile_framework_nonempty.py)

From cc07d96f0e5a859d148ba179e5185f56357eaa9c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 09:04:13 +0100
Subject: [PATCH 034/127] Use getAvailableCollections

---
 test/k4FWCoreTest/options/CheckOutputFiles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index d26b9233..b65a99a7 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -3,7 +3,7 @@
 def check_collections(filename, names):
     podio_reader = podio.root_io.Reader(filename)
     for frame in podio_reader.get('events'):
-        available = set(frame.collections)
+        available = set(frame.getAvailableCollections())
         if available != set(names):
             print(f'These collections should be in the frame but are not: {set(names) - available}')
             print(f'These collections are in the frame but should not be: {available - set(names)}')

From 1e259262c877923d65b282124c628d6da9a5c4f1 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 09:42:08 +0100
Subject: [PATCH 035/127] Fix test

---
 test/k4FWCoreTest/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 599deb00..edc420ae 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -91,7 +91,7 @@ PROPERTIES PASS_REGULAR_EXPRESSION "TwasBrilligAndTheSlithyToves"
 )
 add_test(NAME checkKeepDropSwitch
          WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-         COMMAND python scripts/check_KeepDropSwitch.py)
+         COMMAND python scripts/check_KeepDropSwitch.py ${CMAKE_BINARY_DIR}/test/k4FWCoreTest/output_k4test_exampledata_2.root)
 set_test_env(checkKeepDropSwitch)
 set_property(TEST checkKeepDropSwitch APPEND PROPERTY DEPENDS ReadExampleEventData)
 add_test_with_env(TestUniqueIDGenSvc options/TestUniqueIDGenSvc.py)

From d261c2b6794e24e6099457f7362341b7ea35c64e Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:06:24 +0100
Subject: [PATCH 036/127] Fix tests

---
 python/CMakeLists.txt            | 2 +-
 test/k4FWCoreTest/CMakeLists.txt | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6b4c6f72..f0c66da6 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -7,4 +7,4 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
 # k4FWCore will give the right result
 file(GLOB_RECURSE python_files ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/*.py
      EXCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/__init__.py)
-install(FILES ${python_files} DESTINATION ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)
+install(FILES ${python_files} DESTINATION ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index edc420ae..0d06f112 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -101,6 +101,7 @@ add_test(NAME TestExec WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND pytho
 set_test_env(TestExec)
 
 add_test(NAME Testk4runNoArgumentsHelp COMMAND ${K4RUN})
+set_test_env(Testk4runNoArgumentsHelp)
 set_tests_properties(Testk4runNoArgumentsHelp PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run <options_file.py>, use --help to get a complete list of arguments")
 add_test_with_env(Testk4runCustomArguments options/TestArgs.py --foo=42 PROPERTIES PASS_REGULAR_EXPRESSION "The answer is 42")
 add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIES PASS_REGULAR_EXPRESSION " VERBOSE ")

From f01c0eb1cec42704b103cd9f2353621be00f4ec7 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:13:41 +0100
Subject: [PATCH 037/127] Don't install

---
 python/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index f0c66da6..7ac8c731 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -7,4 +7,5 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
 # k4FWCore will give the right result
 file(GLOB_RECURSE python_files ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/*.py
      EXCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/__init__.py)
-install(FILES ${python_files} DESTINATION ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)
+add_custom_target(copy_python_files ALL COMMAND
+  ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)

From 365cfda086ad70eba54584cc113ac47673daad92 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 11:01:02 +0100
Subject: [PATCH 038/127] Comment out

---
 python/CMakeLists.txt | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 7ac8c731..639721d5 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -7,5 +7,12 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
 # k4FWCore will give the right result
 file(GLOB_RECURSE python_files ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/*.py
      EXCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/__init__.py)
-add_custom_target(copy_python_files ALL COMMAND
-  ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)
+
+# add_custom_command(
+#     OUTPUT ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt
+#     COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore
+#     COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt
+#     DEPENDS ${python_files}
+# )
+
+# add_custom_target(copy_python_files ALL DEPENDS ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt)

From be277ba1a5a8766b0f48e4874fa52b95af2d2223 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:36:25 +0100
Subject: [PATCH 039/127] Copy some files for tests

---
 python/CMakeLists.txt | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 639721d5..234f23bf 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -5,14 +5,6 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
 # Gaudi so we can't just make a k4FWCore folder and put the python files there.
 # We put everything in one of the folders generated by Gaudi so that importing from
 # k4FWCore will give the right result
-file(GLOB_RECURSE python_files ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/*.py
-     EXCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/__init__.py)
-
-# add_custom_command(
-#     OUTPUT ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt
-#     COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore
-#     COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt
-#     DEPENDS ${python_files}
-# )
-
-# add_custom_target(copy_python_files ALL DEPENDS ${CMAKE_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/copied.txt)
+if(BUILD_TESTING)
+  file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/ DESTINATION ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/)
+endif()

From 4d97dbc169f46deb8fe1f08c7570d3cb3746f4a6 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:37:55 +0100
Subject: [PATCH 040/127] Change order

---
 CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a0b68b67..74c2e338 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,10 +73,11 @@ endif()
 
 add_subdirectory(k4FWCore)
 add_subdirectory(k4Interface)
-add_subdirectory(python)
 if(BUILD_TESTING)
   add_subdirectory(test/k4FWCoreTest)
 endif()
+# It has to go after tests since some files are copied if the tests are built
+add_subdirectory(python)
 
 
 option(ENABLE_CPACK "Whether or not to use cpack config" OFF)

From 18778dde7407463515df06658902d6dedd28cdda Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:49:20 +0100
Subject: [PATCH 041/127] Try this

---
 CMakeLists.txt                   | 3 +--
 python/CMakeLists.txt            | 9 ---------
 test/k4FWCoreTest/CMakeLists.txt | 9 +++++++++
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 74c2e338..a0b68b67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,11 +73,10 @@ endif()
 
 add_subdirectory(k4FWCore)
 add_subdirectory(k4Interface)
+add_subdirectory(python)
 if(BUILD_TESTING)
   add_subdirectory(test/k4FWCoreTest)
 endif()
-# It has to go after tests since some files are copied if the tests are built
-add_subdirectory(python)
 
 
 option(ENABLE_CPACK "Whether or not to use cpack config" OFF)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 234f23bf..c191738a 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -1,10 +1 @@
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
-
-# The following is done to make the tests work without installing the files in
-# the installation directory. The k4FWCore in the build directory is populated by
-# Gaudi so we can't just make a k4FWCore folder and put the python files there.
-# We put everything in one of the folders generated by Gaudi so that importing from
-# k4FWCore will give the right result
-if(BUILD_TESTING)
-  file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore/ DESTINATION ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/)
-endif()
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 0d06f112..5ca1cf35 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -128,3 +128,12 @@ add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctio
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
+
+# The following is done to make the tests work without installing the files in
+# the installation directory. The k4FWCore in the build directory is populated by
+# Gaudi so we can't just make a k4FWCore folder and put the python files there.
+# We put everything in one of the folders generated by Gaudi so that importing from
+# k4FWCore will give the right result
+add_custom_command(TARGET k4FWCoreTestPlugins POST_BUILD
+                   COMMAND ${CMAKE_COMMAND} -E copy_directory
+                   ${PROJECT_SOURCE_DIR}/python/k4FWCore/ ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore)

From e06e489477f334202267b3334be7ef49930fda71 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:57:00 +0100
Subject: [PATCH 042/127] Format

---
 k4FWCore/components/IIOSvc.h                  |   2 +-
 k4FWCore/components/IOSvc.cpp                 |  34 ++----
 k4FWCore/components/IOSvc.h                   |  23 ++--
 k4FWCore/components/Reader.cpp                | 115 ++++++++----------
 k4FWCore/components/Writer.cpp                |   5 +-
 k4FWCore/include/k4FWCore/BaseClass.h         |   6 +-
 k4FWCore/include/k4FWCore/Consumer.h          |   3 +-
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |   6 +-
 k4FWCore/include/k4FWCore/OldConsumer.h       |   2 +-
 k4FWCore/include/k4FWCore/Producer.h          |  13 +-
 k4FWCore/include/k4FWCore/Transformer.h       |  11 +-
 test/k4FWCoreTest/CMakeLists.txt              |   3 +
 .../ExampleFunctionalConsumerMemory.py        |   2 +-
 ...ExampleFunctionalConsumerMultipleMemory.py |   2 +-
 .../options/ExampleFunctionalFile.py          |   2 +-
 .../options/ExampleFunctionalFileMultiple.py  |   2 +-
 .../options/ExampleFunctionalMTFile.py        |   2 +-
 .../options/ExampleFunctionalMTMemory.py      |   2 +-
 .../ExampleFunctionalMultipleMemory.py        |   2 +-
 .../ExampleFunctionalOutputCommands.py        |   2 +-
 .../components/ExampleEventHeaderConsumer.cpp |   5 +-
 .../components/ExampleFunctionalConsumer.cpp  |  23 ++--
 .../ExampleFunctionalConsumerMultiple.cpp     |  35 +++---
 ...leFunctionalConsumerRuntimeCollections.cpp |   2 +-
 ...onalConsumerRuntimeCollectionsMultiple.cpp |  31 ++---
 .../components/ExampleFunctionalProducer.cpp  |   2 +-
 ...leFunctionalProducerRuntimeCollections.cpp |  11 +-
 .../ExampleFunctionalTransformer.cpp          |   4 +-
 .../ExampleFunctionalTransformerMultiple.cpp  |  15 ++-
 ...unctionalTransformerRuntimeCollections.cpp |  13 +-
 30 files changed, 176 insertions(+), 204 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 4ac88d0c..be3a353a 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023 Key4hep-Project.
+ * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 50484575..607c7024 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023 Key4hep-Project.
+ * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
@@ -59,26 +59,25 @@ StatusCode IOSvc::initialize() {
 StatusCode IOSvc::finalize() { return Service::finalize(); }
 
 std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> IOSvc::next() {
-
   podio::Frame frame;
   {
     std::scoped_lock<std::mutex> lock(m_changeBufferLock);
     info() << "m_nextEntry = " << m_nextEntry << " m_entries = " << m_entries << endmsg;
     if (m_nextEntry < m_entries) {
       frame = podio::Frame(std::move(m_reader->readEntry(podio::Category::Event, m_nextEntry)));
-    }
-    else {
-      return std::make_tuple(std::vector<std::shared_ptr<podio::CollectionBase>>(), std::vector<std::string>(), std::move(frame));
+    } else {
+      return std::make_tuple(std::vector<std::shared_ptr<podio::CollectionBase>>(), std::vector<std::string>(),
+                             std::move(frame));
     }
     m_nextEntry++;
     if (m_collectionNames.empty()) {
-        m_collectionNames = frame.getAvailableCollections();
+      m_collectionNames = frame.getAvailableCollections();
     }
   }
 
   if (m_nextEntry >= m_entries) {
-  // if (true) {
-    auto ep = serviceLocator()->as<IEventProcessor>();
+    // if (true) {
+    auto       ep = serviceLocator()->as<IEventProcessor>();
     StatusCode sc = ep->stopRun();
     if (sc.isFailure()) {
       error() << "Error when stopping run" << endmsg;
@@ -100,8 +99,7 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
 // After every event if there is still a frame in the TES
 // that means it hasn't been written so the collections inside the Frame
 // should be removed so that they are deleted when the Frame is deleted
-void IOSvc::handle( const Incident& incident ) {
-
+void IOSvc::handle(const Incident& incident) {
   StatusCode code;
   if (m_hiveWhiteBoard) {
     if (!incident.context().valid()) {
@@ -115,7 +113,7 @@ void IOSvc::handle( const Incident& incident ) {
       throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
     }
   }
-  DataObject *p;
+  DataObject* p;
   code = m_dataSvc->retrieveObject("/Event/_Frame", p);
   if (code.isFailure()) {
     info() << "No frame found" << endmsg;
@@ -124,7 +122,7 @@ void IOSvc::handle( const Incident& incident ) {
 
   auto frame = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
   for (const auto& coll : frame->getData().getAvailableCollections()) {
-    DataObject *collPtr;
+    DataObject* collPtr;
     code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);
     if (code.isSuccess()) {
       info() << "Removing collection: " << coll << endmsg;
@@ -136,16 +134,10 @@ void IOSvc::handle( const Incident& incident ) {
   }
 }
 
-void IOSvc::setReadingCollectionNames(const std::vector<std::string>& names) {
-  m_collectionNames = names;
-}
+void IOSvc::setReadingCollectionNames(const std::vector<std::string>& names) { m_collectionNames = names; }
 
-void IOSvc::setReadingFileNames(const std::vector<std::string>& names) {
-  m_readingFileNames = names;
-}
+void IOSvc::setReadingFileNames(const std::vector<std::string>& names) { m_readingFileNames = names; }
 
-bool IOSvc::writeCollection(const std::string& collName) {
-  return m_switch.isOn(collName);
-}
+bool IOSvc::writeCollection(const std::string& collName) { return m_switch.isOn(collName); }
 
 DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 6acc83be..f869e96b 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023 Key4hep-Project.
+ * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
@@ -20,11 +20,11 @@
 #define FWCORE_IOSVC_H
 
 #include "Gaudi/Property.h"
-#include "GaudiKernel/Service.h"
-#include "GaudiKernel/IIncidentListener.h"
-#include "GaudiKernel/IIncidentSvc.h"
 #include "GaudiKernel/IDataProviderSvc.h"
 #include "GaudiKernel/IHiveWhiteBoard.h"
+#include "GaudiKernel/IIncidentListener.h"
+#include "GaudiKernel/IIncidentSvc.h"
+#include "GaudiKernel/Service.h"
 
 #include "podio/ROOTReader.h"
 #include "podio/ROOTWriter.h"
@@ -41,6 +41,7 @@
 
 class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   using extends::extends;
+
 public:
   // Gaudi doesn't run the destructor of the Services so we have to
   // manually ask for the writer to be deleted so it will call finish()
@@ -49,7 +50,8 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   StatusCode initialize() override;
   StatusCode finalize() override;
 
-  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next() override;
+  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame> next()
+      override;
 
   std::shared_ptr<std::vector<std::string>> getCollectionNames() const override {
     return std::make_shared<std::vector<std::string>>(m_collectionNames);
@@ -59,7 +61,6 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   void setReadingFileNames(const std::vector<std::string>& names);
 
 protected:
-
   Gaudi::Property<unsigned int> m_bufferNbEvents{
       this, "BufferNbEvents", 20000,
       "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
@@ -67,7 +68,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
   Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
-  Gaudi::Property<std::string> m_writingFileName{this, "output", {}, "List of files to read"};
+  Gaudi::Property<std::string>              m_writingFileName{this, "output", {}, "List of files to read"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
   Gaudi::Property<std::string> m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"};
@@ -98,13 +99,13 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   }
 
   SmartIF<IDataProviderSvc> m_dataSvc;
-  SmartIF<IIncidentSvc> m_incidentSvc;
-  SmartIF<IHiveWhiteBoard> m_hiveWhiteBoard;
-  void handle(const Incident& incident) override;
+  SmartIF<IIncidentSvc>     m_incidentSvc;
+  SmartIF<IHiveWhiteBoard>  m_hiveWhiteBoard;
+  void                      handle(const Incident& incident) override;
 
   int m_entries{0};
   int m_nextEntry{0};
-  
+
   bool writeCollection(const std::string& collName) override;
 };
 
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 4f691cf9..cbbd0582 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023 Key4hep-Project.
+ * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
@@ -18,11 +18,10 @@
  */
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
-#include "GaudiKernel/FunctionalFilterDecision.h"
-#include "GaudiKernel/StatusCode.h"
 #include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/FunctionalFilterDecision.h"
 #include "GaudiKernel/IDataProviderSvc.h"
-
+#include "GaudiKernel/StatusCode.h"
 
 #include "podio/CollectionBase.h"
 #include "podio/Frame.h"
@@ -32,75 +31,65 @@
 
 #include <memory>
 
-template <typename Container>
-using vector_of_ = std::vector<Container>;
-template <typename Container>
-using vector_of_optional_ = std::vector<std::optional<Container>>;
-
+template <typename Container> using vector_of_          = std::vector<Container>;
+template <typename Container> using vector_of_optional_ = std::vector<std::optional<Container>>;
 
 class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::Functional::Traits::useDefaults> {
-    using Traits_ = Gaudi::Functional::Traits::useDefaults;
-    using Out = std::shared_ptr<podio::CollectionBase>;
-    using base_class = Gaudi::Functional::details::BaseClass_t<Traits_>;
-    static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
-
-  public:
-    CollectionPusher(std::string name, ISvcLocator* locator) :
-      base_class(std::move(name), locator)
-    {}
-
-    // derived classes can NOT implement execute
-    StatusCode execute(const EventContext&) const override final {
-      try {
-        auto out = (*this)();
-
-        auto outColls = std::get<0>(out);
-        auto outputLocations = std::get<1>(out);
-
-        // if (out.size() != m_outputs.size()) {
-        //   throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
-        //                            " containers, got " + std::to_string(out.size()) + " instead",
-        //                        this->name(), StatusCode::FAILURE);
-        // }
-        for (unsigned i = 0; i != outColls.size(); ++i) {
-          auto objectp = std::make_unique<AnyDataWrapper<Out>>( std::move( outColls[i] ) );
-          if ( auto sc = m_dataSvc->registerObject( outputLocations[i], objectp.get() ); sc.isFailure() ) {
-          }
-          // The store has the ownership so we shouldn't delete the object
-          auto ptr = objectp.release();
+  using Traits_    = Gaudi::Functional::Traits::useDefaults;
+  using Out        = std::shared_ptr<podio::CollectionBase>;
+  using base_class = Gaudi::Functional::details::BaseClass_t<Traits_>;
+  static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
+
+public:
+  CollectionPusher(std::string name, ISvcLocator* locator) : base_class(std::move(name), locator) {}
+
+  // derived classes can NOT implement execute
+  StatusCode execute(const EventContext&) const override final {
+    try {
+      auto out = (*this)();
+
+      auto outColls        = std::get<0>(out);
+      auto outputLocations = std::get<1>(out);
+
+      // if (out.size() != m_outputs.size()) {
+      //   throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
+      //                            " containers, got " + std::to_string(out.size()) + " instead",
+      //                        this->name(), StatusCode::FAILURE);
+      // }
+      for (unsigned i = 0; i != outColls.size(); ++i) {
+        auto objectp = std::make_unique<AnyDataWrapper<Out>>(std::move(outColls[i]));
+        if (auto sc = m_dataSvc->registerObject(outputLocations[i], objectp.get()); sc.isFailure()) {
         }
-        return Gaudi::Functional::FilterDecision::PASSED;
-      } catch (GaudiException& e) {
-        (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-        return e.code();
+        // The store has the ownership so we shouldn't delete the object
+        auto ptr = objectp.release();
       }
+      return Gaudi::Functional::FilterDecision::PASSED;
+    } catch (GaudiException& e) {
+      (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
+      return e.code();
     }
+  }
 
-    virtual std::tuple<vector_of_<Out>, std::vector<std::string>> operator()() const = 0;
-
-  private:
-    // if In is a pointer, it signals optional (as opposed to mandatory) input
-    // template <typename T>
-    // using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
-    // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
-    // TODO/FIXME: replace vector of DataObjID property + call-back with a
-    //             vector<handle> property ... as soon as declareProperty can deal with that.
-    ServiceHandle<IDataProviderSvc> m_dataSvc{this, "EventDataSvc", "EventDataSvc"};
+  virtual std::tuple<vector_of_<Out>, std::vector<std::string>> operator()() const = 0;
 
-  };
+private:
+  // if In is a pointer, it signals optional (as opposed to mandatory) input
+  // template <typename T>
+  // using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
+  // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
+  // TODO/FIXME: replace vector of DataObjID property + call-back with a
+  //             vector<handle> property ... as soon as declareProperty can deal with that.
+  ServiceHandle<IDataProviderSvc> m_dataSvc{this, "EventDataSvc", "EventDataSvc"};
+};
 
 class Reader final : public CollectionPusher {
 public:
-  Reader(const std::string& name, ISvcLocator* svcLoc) :
-    CollectionPusher(name, svcLoc) {
-  }
+  Reader(const std::string& name, ISvcLocator* svcLoc) : CollectionPusher(name, svcLoc) {}
 
   // Gaudi doesn't run the destructor of the Services so we have to
   // manually ask for the reader to be deleted so it will call finish()
   // See https://gitlab.cern.ch/gaudi/Gaudi/-/issues/169
-  ~Reader() {
-    iosvc->deleteReader();
-  }
+  ~Reader() { iosvc->deleteReader(); }
 
   ServiceHandle<IIOSvc> iosvc{this, "IOSvc", "IOSvc"};
 
@@ -123,18 +112,18 @@ class Reader final : public CollectionPusher {
   // The IOSvc takes care of reading and passing the data
   // By convention the Frame is pushed to the store
   // so that it's deleted at the right time
-  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>> operator()() const override {
+  std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>> operator()()
+      const override {
     auto val = iosvc->next();
 
-    auto eds = eventSvc().as<IDataProviderSvc>();
+    auto eds   = eventSvc().as<IDataProviderSvc>();
     auto frame = std::move(std::get<2>(val));
 
-    auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
+    auto tmp  = new AnyDataWrapper<podio::Frame>(std::move(frame));
     auto code = eds->registerObject("/Event" + k4FWCore::frameLocation, tmp);
 
     return std::make_tuple(std::get<0>(val), std::get<1>(val));
   }
-
 };
 
 DECLARE_COMPONENT(Reader)
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index b5e1d441..9c47aafd 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2014-2023 Key4hep-Project.
+/* * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
@@ -71,7 +71,6 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
   }
 
   StatusCode finalize() override {
-
     podio::Frame config_metadata_frame;
 
     //// prepare job options metadata ///////////////////////
@@ -92,7 +91,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     // and have to be traversed like this. Note that Gaudi!577 will improve this.
     for (const auto* name : {"NTupleSvc"}) {
       std::stringstream config_stream;
-      auto svc = service<IProperty>(name);
+      auto              svc = service<IProperty>(name);
       if (!svc.isValid())
         continue;
       for (const auto* property : svc->getProperties()) {
diff --git a/k4FWCore/include/k4FWCore/BaseClass.h b/k4FWCore/include/k4FWCore/BaseClass.h
index 02732d60..7032d574 100644
--- a/k4FWCore/include/k4FWCore/BaseClass.h
+++ b/k4FWCore/include/k4FWCore/BaseClass.h
@@ -20,13 +20,15 @@
 #ifndef K4FWCORE_FUNCTIONALUTILS_H
 #define K4FWCORE_FUNCTIONALUTILS_H
 
-#include "Gaudi/Algorithm.h"
+#include "GaudiAlg/GaudiAlgorithm.h"
 #include "GaudiKernel/DataObjectHandle.h"
 #include "k4FWCore/DataWrapper.h"
 
 // Base class used for the Traits template argument of the
 // Gaudi::Functional algorithms
-struct [[deprecated("Functional algorithms using the BaseClass.h header are deprecated and will be removed in the future")]] BaseClass_t {
+struct [[deprecated(
+    "Functional algorithms using the BaseClass.h header are deprecated and will be removed in the "
+    "future")]] BaseClass_t {
   template <typename T> using InputHandle  = DataObjectReadHandle<DataWrapper<T>>;
   template <typename T> using OutputHandle = DataObjectWriteHandle<DataWrapper<T>>;
 
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index bf0b2464..cca927e6 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -89,8 +89,7 @@ namespace k4FWCore {
                Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
           : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
 
-      template <size_t Index, typename... Handles>
-      void readMapInputs(const std::tuple<Handles...>& handles) const {
+      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             using EDM4hepType =
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 2b133e22..28afece5 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -38,8 +38,7 @@ namespace k4FWCore {
       return result;
     }
 
-    template <typename T, typename K>
-    std::string getName(const K& first, bool pair=false) {
+    template <typename T, typename K> std::string getName(const K& first, bool pair = false) {
       if constexpr (is_map_like<T>::value) {
         if (pair) {
           return hash_string(first.first);
@@ -59,8 +58,7 @@ namespace k4FWCore {
       using type = Value;
     };
 
-    template <typename Value>
-    struct ExtractInnerType<std::map<std::string, std::shared_ptr<Value>>> {
+    template <typename Value> struct ExtractInnerType<std::map<std::string, std::shared_ptr<Value>>> {
       using type = Value;
     };
 
diff --git a/k4FWCore/include/k4FWCore/OldConsumer.h b/k4FWCore/include/k4FWCore/OldConsumer.h
index 5000a5d0..860fe89b 100644
--- a/k4FWCore/include/k4FWCore/OldConsumer.h
+++ b/k4FWCore/include/k4FWCore/OldConsumer.h
@@ -64,7 +64,7 @@ namespace k4FWCore {
             std::string        token;
             while (ss >> token) {
               DataObject* p;
-              auto sc = this->evtSvc()->retrieveObject(token, p);
+              auto        sc = this->evtSvc()->retrieveObject(token, p);
               if (!sc.isSuccess()) {
                 throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
               }
diff --git a/k4FWCore/include/k4FWCore/Producer.h b/k4FWCore/include/k4FWCore/Producer.h
index 277ab6ca..a5e404ef 100644
--- a/k4FWCore/include/k4FWCore/Producer.h
+++ b/k4FWCore/include/k4FWCore/Producer.h
@@ -23,29 +23,26 @@
 
 #include <tuple>
 
-
 namespace k4FWCore {
 
   namespace details {
 
-    template <typename Signature, typename Traits_>
-    struct Producer;
+    template <typename Signature, typename Traits_> struct Producer;
 
     template <typename... Out, typename Traits_>
     struct Producer<std::tuple<Out...>(), Traits_> : MultiTransformer<std::tuple<Out...>(), Traits_> {
       using MultiTransformer<std::tuple<Out...>(), Traits_>::MultiTransformer;
     };
 
-    template <typename Out, typename Traits_>
-    struct Producer<Out(), Traits_> : Transformer<Out(), Traits_> {
+    template <typename Out, typename Traits_> struct Producer<Out(), Traits_> : Transformer<Out(), Traits_> {
       using Transformer<Out(), Traits_>::Transformer;
     };
 
-  } // namespace details
+  }  // namespace details
 
   template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
   using Producer = details::Producer<Signature, Traits_>;
 
-} // namespace k4FWCore
+}  // namespace k4FWCore
 
-#endif // FWCORE_PRODUCER_H
+#endif  // FWCORE_PRODUCER_H
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 62732873..2d9b6b95 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -57,7 +57,7 @@ namespace k4FWCore {
       std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
       std::array<Gaudi::Property<std::vector<DataObjID>>, 1>                                     m_outputLocations{};
-      std::array<Gaudi::Property<DataObjID>, 1>               m_outputLocationsPair{};
+      std::array<Gaudi::Property<DataObjID>, 1> m_outputLocationsPair{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
@@ -232,7 +232,7 @@ namespace k4FWCore {
       std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)>                        m_outputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(Out)>  m_outputLocationsPair{};
+      std::array<Gaudi::Property<DataObjID>, sizeof...(Out)> m_outputLocationsPair{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
@@ -339,11 +339,8 @@ namespace k4FWCore {
               auto        sc     = this->evtSvc()->registerObject(key, p);
             }
 
-          }
-          else {
-            Gaudi::Functional::details::put(
-                std::get<Index>(m_outputs),
-                ptrOrCast(std::move(std::get<Index>(handles))));
+          } else {
+            Gaudi::Functional::details::put(std::get<Index>(m_outputs), ptrOrCast(std::move(std::get<Index>(handles))));
           }
 
           // Recursive call for the next index
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 5ca1cf35..4f618a0a 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -33,6 +33,9 @@ function(set_test_env testname)
   set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
 endfunction()
 
+message(WARNING "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
+message(WARNING "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
+
 function(add_test_with_env testname)
   foreach(arg ${ARGN})
     if(arg STREQUAL "PROPERTIES")
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
index d5630a84..a62197e6 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
index ea2eb698..9ccf78d3 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index dc72581a..58436c7f 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
index ea68c898..57cf1c32 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 4d354857..e7de43b5 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index a4a2d969..6b9753bc 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index 24cafc37..ee2f1284 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
index b8e4c7de..8611665c 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2023 Key4hep-Project.
+# Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
 # See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
index 09c139b5..0915fa38 100644
--- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp
@@ -33,10 +33,9 @@
 #include <stdexcept>
 #include <string>
 
-struct ExampleEventHeaderConsumer final
-    : k4FWCore::Consumer<void(const edm4hep::EventHeaderCollection&)> {
+struct ExampleEventHeaderConsumer final : k4FWCore::Consumer<void(const edm4hep::EventHeaderCollection&)> {
   ExampleEventHeaderConsumer(const std::string& name, ISvcLocator* svcLoc)
-    : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::EventHeaderName})}) {}
+      : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::EventHeaderName})}) {}
 
   void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const {
     if (evtHeaderColl.empty()) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index 3631b5ae..3bc56cd7 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -31,7 +31,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the input collection
   ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc)
-    : Consumer(name, svcLoc, KeyValues("InputCollection", {"MCParticles"})) {}
+      : Consumer(name, svcLoc, KeyValues("InputCollection", {"MCParticles"})) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as the collections
@@ -43,25 +43,22 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
     }
     int i = 0;
     for (const auto& particle : input) {
-      if ((particle.getPDG() != 1 + i + m_offset) ||
-          (particle.getGeneratorStatus() != 2 + i + m_offset) ||
-          (particle.getSimulatorStatus() != 3 + i + m_offset) ||
-          (particle.getCharge() != 4 + i + m_offset) || (particle.getTime() != 5 + i + m_offset) ||
-          (particle.getMass() != 6 + i + m_offset)) {
+      if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+          (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
+          (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
         std::stringstream error;
-        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", "
-              << 2 + i + m_offset << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset
-              << ", " << 5 + i + m_offset << ", " << 6 + i + m_offset << " got " << particle.getPDG()
-              << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
-              << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
+        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset
+              << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", "
+              << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", "
+              << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
+              << particle.getMass() << "";
         throw std::runtime_error(error.str());
       }
       i++;
     }
   }
 
-  Gaudi::Property<int> m_offset{this, "Offset", 10,
-      "Integer to add to the dummy values written to the edm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalConsumer)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
index b1e9d846..bb546f06 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
@@ -46,18 +46,18 @@ using TrackerHitColl    = edm4hep::TrackerHit3DCollection;
 using TrackColl         = edm4hep::TrackCollection;
 
 struct ExampleFunctionalConsumerMultiple final
-    : k4FWCore::Consumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
-                                       const TrackerHitColl&, const TrackColl&)> {
+    : k4FWCore::Consumer<void(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&, const TrackerHitColl&,
+                              const TrackColl&)> {
   // The pairs in KeyValue can be changed from python and they correspond
   // to the names of the input collection
   ExampleFunctionalConsumerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : Consumer(name, svcLoc,
                  {
-                   KeyValues("InputCollectionFloat", {"VectorFloat"}),
-                   KeyValues("InputCollectionParticles", {"MCParticles1"}),
-                   KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
-                   KeyValues("InputCollectionTrackerHits", {"TrackerHits"}),
-                   KeyValues("InputCollectionTracks", {"Tracks"}),
+                     KeyValues("InputCollectionFloat", {"VectorFloat"}),
+                     KeyValues("InputCollectionParticles", {"MCParticles1"}),
+                     KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
+                     KeyValues("InputCollectionTrackerHits", {"TrackerHits"}),
+                     KeyValues("InputCollectionTracks", {"Tracks"}),
                  }) {}
 
   // This is the function that will be called to transform the data
@@ -78,17 +78,15 @@ struct ExampleFunctionalConsumerMultiple final
 
     int i = 0;
     for (const auto& particle : particles) {
-      if ((particle.getPDG() != 1 + i + m_offset) ||
-          (particle.getGeneratorStatus() != 2 + i + m_offset) ||
-          (particle.getSimulatorStatus() != 3 + i + m_offset) ||
-          (particle.getCharge() != 4 + i + m_offset) || (particle.getTime() != 5 + i + m_offset) ||
-          (particle.getMass() != 6 + i + m_offset)) {
+      if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
+          (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
+          (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
         std::stringstream error;
-        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", "
-              << 2 + i + m_offset << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset
-              << ", " << 5 + i + m_offset << ", " << 6 + i + m_offset << " got " << particle.getPDG()
-              << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", "
-              << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << "";
+        error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset
+              << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", "
+              << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", "
+              << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
+              << particle.getMass() << "";
         throw std::runtime_error(error.str());
       }
       i++;
@@ -122,8 +120,7 @@ struct ExampleFunctionalConsumerMultiple final
   }
 
 private:
-  Gaudi::Property<int> m_offset{this, "Offset", 10,
-      "Integer to add to the dummy values written to the edm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalConsumerMultiple)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index 09b89322..fc0683d5 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -39,7 +39,7 @@ struct ExampleFunctionalConsumerRuntimeCollections final
       fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
     }
     for (auto& [key, val] : input) {
-      int  i    = 0;
+      int i = 0;
       for (const auto& particle : *val) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index d0c37363..8ec11409 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -29,27 +29,29 @@
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
-  : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
-                              const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
-                              const edm4hep::SimTrackerHitCollection&                              simTrackerHits)> {
+    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
+                              const std::map<std::string, std::shared_ptr<podio::CollectionBase>>&         trackMap,
+                              const edm4hep::SimTrackerHitCollection& simTrackerHits)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc)
-    : Consumer(name, svcLoc, {KeyValues("Particles", {"MCParticles"}), KeyValues("Tracks", {"MCParticles"}), KeyValues("SimTrackerHits", {"MCParticles"})}) {}
-    // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
+      : Consumer(name, svcLoc,
+                 {KeyValues("Particles", {"MCParticles"}), KeyValues("Tracks", {"MCParticles"}),
+                  KeyValues("SimTrackerHits", {"MCParticles"})}) {}
+  // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
 
   // This is the function that will be called to produce the data
   void operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
-                  const std::map<std::string, std::shared_ptr<podio::CollectionBase>>& trackMap,
-                  const edm4hep::SimTrackerHitCollection&                              simTrackerHits) const override {
-    info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size() << " track collections"
-           << endmsg;
+                  const std::map<std::string, std::shared_ptr<podio::CollectionBase>>&         trackMap,
+                  const edm4hep::SimTrackerHitCollection& simTrackerHits) const override {
+    info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size()
+           << " track collections" << endmsg;
     if (particleMap.size() != 5) {
       fatal() << "Wrong size of the particleMap map, expected 5, got " << particleMap.size() << endmsg;
     }
     for (auto& [key, val] : particleMap) {
       const auto& particles = *std::static_pointer_cast<edm4hep::MCParticleCollection>(val);
-      int  i    = 0;
+      int         i         = 0;
       for (const auto& particle : particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
@@ -83,10 +85,9 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
   }
 
 private:
-// We can define any property we want that can be set from python
-// and use it inside operator()
-Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
-}
-;
+  // We can define any property we want that can be set from python
+  // and use it inside operator()
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
+};
 
 DECLARE_COMPONENT(ExampleFunctionalConsumerRuntimeCollectionsMultiple)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
index c2a2e66f..afa2222b 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
@@ -30,7 +30,7 @@ struct ExampleFunctionalProducer final : k4FWCore::Producer<edm4hep::MCParticleC
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducer(const std::string& name, ISvcLocator* svcLoc)
-    : Producer(name, svcLoc, {}, KeyValues("OutputCollection", {"MCParticles"})) {}
+      : Producer(name, svcLoc, {}, KeyValues("OutputCollection", {"MCParticles"})) {}
 
   // This is the function that will be called to produce the data
   edm4hep::MCParticleCollection operator()() const override {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
index c3055a7d..2f512f73 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
@@ -25,30 +25,31 @@
 
 #include <string>
 
-struct ExampleFunctionalProducerRuntimeCollections final : k4FWCore::Producer<std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>()> {
+struct ExampleFunctionalProducerRuntimeCollections final
+    : k4FWCore::Producer<std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>()> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
-    : Producer(name, svcLoc, {}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
+      : Producer(name, svcLoc, {}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
   std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> operator()() const override {
     std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
     for (int i = 0; i < m_numberOfCollections; ++i) {
       std::string name = "MCParticles" + std::to_string(i);
-      auto coll = std::make_shared<edm4hep::MCParticleCollection>();
+      auto        coll = std::make_shared<edm4hep::MCParticleCollection>();
       coll->create(1, 2, 3, 4.f, 5.f, 6.f);
       coll->create(2, 3, 4, 5.f, 6.f, 7.f);
       m_outputCollections[name] = coll;
     }
     return m_outputCollections;
-
   }
 
 private:
   // We can define any property we want that can be set from python
   // and use it inside operator()
-  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3, "Example int that can be used in the algorithm"};
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3,
+                                             "Example int that can be used in the algorithm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalProducerRuntimeCollections)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
index 7f691efa..c4233151 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
@@ -30,8 +30,8 @@
 struct ExampleFunctionalTransformer final
     : k4FWCore::Transformer<edm4hep::MCParticleCollection(const edm4hep::MCParticleCollection&)> {
   ExampleFunctionalTransformer(const std::string& name, ISvcLocator* svcLoc)
-    : Transformer(name, svcLoc, {KeyValues("InputCollection", {"MCParticles"})},
-                  {KeyValues("OutputCollection", {"NewMCParticles"})}) {}
+      : Transformer(name, svcLoc, {KeyValues("InputCollection", {"MCParticles"})},
+                    {KeyValues("OutputCollection", {"NewMCParticles"})}) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as all pointers to collections
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
index cf803ad3..a27d2b71 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp
@@ -49,17 +49,17 @@ using Counter  = podio::UserDataCollection<int>;
 using Particle = edm4hep::MCParticleCollection;
 
 struct ExampleFunctionalTransformerMultiple final
-    : k4FWCore::MultiTransformer<std::tuple<Counter, Particle>(const FloatColl&, const ParticleColl&,
-                                                                        const SimTrackerHitColl&, const TrackerHitColl&,
-                                                                        const TrackColl&)> {
+    : k4FWCore::MultiTransformer<std::tuple<Counter, Particle>(
+          const FloatColl&, const ParticleColl&, const SimTrackerHitColl&, const TrackerHitColl&, const TrackColl&)> {
   ExampleFunctionalTransformerMultiple(const std::string& name, ISvcLocator* svcLoc)
       : MultiTransformer(
             name, svcLoc,
-            {KeyValues("InputCollectionFloat", {"VectorFloat"}), KeyValues("InputCollectionParticles", {"MCParticles1"}),
+            {KeyValues("InputCollectionFloat", {"VectorFloat"}),
+             KeyValues("InputCollectionParticles", {"MCParticles1"}),
              KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}),
              KeyValues("InputCollectionTrackerHits", {"TrackerHits"}), KeyValues("InputCollectionTracks", {"Tracks"})},
-            {KeyValues("OutputCollectionCounter", {"Counter"}), KeyValues("OutputCollectionParticles", {"NewMCParticles"})}) {
-  }
+            {KeyValues("OutputCollectionCounter", {"Counter"}),
+             KeyValues("OutputCollectionParticles", {"NewMCParticles"})}) {}
 
   // This is the function that will be called to transform the data
   // Note that the function has to be const, as well as the collections
@@ -92,8 +92,7 @@ struct ExampleFunctionalTransformerMultiple final
     return std::make_tuple(std::move(counter), std::move(newParticlesColl));
   }
 
-  Gaudi::Property<int> m_offset{this, "Offset", 10,
-      "Integer to add to the dummy values written to the edm"};
+  Gaudi::Property<int> m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalTransformerMultiple)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
index ecf6b491..c3ffccd1 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -31,27 +31,28 @@ struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transfor
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
-    : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
+      : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})},
+                    {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
   mapType operator()(const mapType& input) const override {
     std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
     for (int i = 0; i < input.size(); ++i) {
-      std::string name = "NewMCParticles" + std::to_string(i);
-      auto old_coll = input.at("MCParticles" + std::to_string(i));
-      auto coll = std::make_shared<edm4hep::MCParticleCollection>();
+      std::string name     = "NewMCParticles" + std::to_string(i);
+      auto        old_coll = input.at("MCParticles" + std::to_string(i));
+      auto        coll     = std::make_shared<edm4hep::MCParticleCollection>();
       coll->push_back(old_coll->at(0).clone());
       coll->push_back(old_coll->at(1).clone());
       m_outputCollections[name] = coll;
     }
     return m_outputCollections;
-
   }
 
 private:
   // We can define any property we want that can be set from python
   // and use it inside operator()
-  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3, "Example int that can be used in the algorithm"};
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3,
+                                             "Example int that can be used in the algorithm"};
 };
 
 DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeCollections)

From 72496cbb308c249dfbd7175351933d1741af39f2 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 14:18:40 +0100
Subject: [PATCH 043/127] Remove a couple of files

---
 .../options/AvalancheSchedulerSimpleTest.py   | 74 -----------------
 test/k4FWCoreTest/options/SimpleTest.py       | 79 -------------------
 2 files changed, 153 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
 delete mode 100644 test/k4FWCoreTest/options/SimpleTest.py

diff --git a/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py b/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
deleted file mode 100644
index b7d42d8b..00000000
--- a/test/k4FWCoreTest/options/AvalancheSchedulerSimpleTest.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Test that we can get reproducible random numbers in a multithreaded environment
-# given that the seed is the same (obtained, for example, with the UniqueIDGenSvc)
-
-from Configurables import AlgResourcePool, AvalancheSchedulerSvc
-from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard
-from Configurables import ApplicationMgr, Gaudi__Sequencer
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, Writer
-from Gaudi.Configuration import INFO, DEBUG, WARNING
-
-evtslots = 1
-threads = 1
-# -------------------------------------------------------------------------------
-
-# The configuration of the whiteboard ------------------------------------------
-# It is useful to call it EventDataSvc to replace the usual data service with
-# the whiteboard transparently.
-
-whiteboard = HiveWhiteBoard("EventDataSvc",
-                            EventSlots=evtslots,
-                            ForceLeaves=True,
-                            )
-
-# Event Loop Manager -----------------------------------------------------------
-# It's called slim since it has less functionalities overall than the good-old
-# event loop manager. Here we just set its outputlevel to DEBUG.
-
-slimeventloopmgr = HiveSlimEventLoopMgr(
-    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
-)
-
-# AvalancheScheduler -----------------------------------------------------------
-# We just decide how many algorithms in flight we want to have and how many
-# threads in the pool. The default value is -1, which is for TBB equivalent
-# to take over the whole machine.
-
-scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
-
-# Algo Resource Pool -----------------------------------------------------------
-# Nothing special here, we just set the debug level.
-AlgResourcePool(OutputLevel=DEBUG)
-
-a1 = ExampleFunctionalProducer("First")
-a2 = ExampleFunctionalConsumer("Consumer")
-# a2 = ExampleFunctionalProducer("Second", OutputCollection="MySecondCollection")
-# a3 = ExampleFunctionalProducer("Third", OutputCollection="MyThirdCollection")
-
-
-svc = IOSvc("IOSvc")
-# svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/output_k4test_exampledata_producer.root']
-svc.FileNames = ['/home/juanmi/Key4hep/Workarea/build/output_k4test_exampledata_producer.root']
-svc.CollectionNames = ['MCParticles']
-
-io = Reader("Reader",
-           OutputLocations=['MCParticles'])
-io.Cardinality = 1
-
-save = Writer("Writer")
-save.CollectionNames = ['MCParticles']
-save.Cardinality = 1
-
-node = Gaudi__Sequencer("Node", Members=[io, a1], Sequential=True, OutputLevel=INFO)
-
-app = ApplicationMgr(
-    EvtMax=1e7,
-    EvtSel="NONE",
-    ExtSvc=[whiteboard],
-    EventLoop=slimeventloopmgr,
-    TopAlg=[node],
-    MessageSvcType="InertMessageSvc",
-    OutputLevel=INFO,
-)
-
-
-print(app)
diff --git a/test/k4FWCoreTest/options/SimpleTest.py b/test/k4FWCoreTest/options/SimpleTest.py
deleted file mode 100644
index 761958f6..00000000
--- a/test/k4FWCoreTest/options/SimpleTest.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Test that we can get reproducible random numbers in a multithreaded environment
-# given that the seed is the same (obtained, for example, with the UniqueIDGenSvc)
-
-from Configurables import AlgResourcePool, AvalancheSchedulerSvc
-from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard
-from Configurables import ApplicationMgr, Gaudi__Sequencer
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer, Reader, IOSvc, Writer, EventDataSvc
-from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerMultiple
-from Gaudi.Configuration import INFO, DEBUG, WARNING
-
-evtslots = 1
-threads = 1
-# -------------------------------------------------------------------------------
-
-# The configuration of the whiteboard ------------------------------------------
-# It is useful to call it EventDataSvc to replace the usual data service with
-# the whiteboard transparently.
-
-# whiteboard = HiveWhiteBoard("EventDataSvc",
-#                             EventSlots=evtslots,
-#                             ForceLeaves=True,
-#                             )
-
-# Event Loop Manager -----------------------------------------------------------
-# It's called slim since it has less functionalities overall than the good-old
-# event loop manager. Here we just set its outputlevel to DEBUG.
-
-# slimeventloopmgr = HiveSlimEventLoopMgr(
-#     SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
-# )
-
-# AvalancheScheduler -----------------------------------------------------------
-# We just decide how many algorithms in flight we want to have and how many
-# threads in the pool. The default value is -1, which is for TBB equivalent
-# to take over the whole machine.
-
-scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
-
-# Algo Resource Pool -----------------------------------------------------------
-# Nothing special here, we just set the debug level.
-AlgResourcePool(OutputLevel=DEBUG)
-
-a1 = ExampleFunctionalProducer("FirstSingle")
-a2 = ExampleFunctionalConsumer("ConsumerSingle")
-b1 = ExampleFunctionalProducerMultiple("First")
-b2 = ExampleFunctionalConsumerMultiple("Consumer")
-# a2 = ExampleFunctionalProducer("Second", OutputCollection="MySecondCollection")
-# a3 = ExampleFunctionalProducer("Third", OutputCollection="MyThirdCollection")
-
-datasvc = EventDataSvc("EventDataSvc")
-
-
-svc = IOSvc("IOSvc")
-# svc.FileNames = ['/home/juanmi/Key4hep/Algorithm-validation/Digitisation/output_k4test_exampledata_producer.root']
-svc.FileNames = ['/home/juanmi/Key4hep/Workarea/build/output_k4test_exampledata_producer.root']
-svc.CollectionNames = ['MCParticles']
-
-io = Reader("Reader",
-           OutputLocations=['MCParticles'])
-io.Cardinality = 1
-
-save = Writer("Writer")
-save.CollectionNames = ['MCParticles']
-save.Cardinality = 1
-
-# node = Gaudi__Sequencer("Node", Members=[io, a1], Sequential=True, OutputLevel=INFO)
-
-app = ApplicationMgr(
-    EvtMax=10,
-    EvtSel="NONE",
-    ExtSvc=[datasvc],
-    TopAlg=[b1, b2],
-    # TopAlg=[io, a2],
-    MessageSvcType="InertMessageSvc",
-    OutputLevel=INFO,
-)
-
-
-print(app)

From 2686d8b3283cec1bfe10f0332d00e6887f575b6c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 14:28:55 +0100
Subject: [PATCH 044/127] More format

---
 k4FWCore/components/Writer.cpp                 |  3 ++-
 k4FWCore/include/k4FWCore/FunctionalUtils.h    | 18 ++++++++++++++++++
 python/CMakeLists.txt                          | 18 ++++++++++++++++++
 python/k4FWCore/ApplicationMgr.py              | 18 ++++++++++++++++++
 python/k4FWCore/IOSvc.py                       | 18 ++++++++++++++++++
 python/k4FWCore/__init__.py                    | 18 ++++++++++++++++++
 test/k4FWCoreTest/options/CheckOutputFiles.py  | 18 ++++++++++++++++++
 ...mpleFunctionalConsumerRuntimeCollections.py |  1 +
 ...tionalConsumerRuntimeCollectionsMultiple.py |  1 +
 .../options/ExampleFunctionalFile.py           |  2 --
 .../options/ExampleFunctionalMTFile.py         |  4 ++--
 ...mpleFunctionalProducerRuntimeCollections.py |  1 +
 ...eFunctionalTransformerRuntimeCollections.py |  1 +
 ...nalTransformerRuntimeCollectionsMultiple.py |  5 +++--
 14 files changed, 119 insertions(+), 7 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 9c47aafd..0726543a 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -1,4 +1,5 @@
-/* * Copyright (c) 2014-2024 Key4hep-Project.
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
  *
  * This file is part of Key4hep.
  * See https://key4hep.github.io/key4hep-doc/ for further info.
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 28afece5..7af3c523 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -1,3 +1,21 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 #pragma once
 
 #include "Gaudi/Functional/details.h"
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index c191738a..d370a894 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -1 +1,19 @@
+#[[
+Copyright (c) 2014-2024 Key4hep-Project.
+
+This file is part of Key4hep.
+See https://key4hep.github.io/key4hep-doc/ for further info.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python)
diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index a7df717f..57de660f 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -1,3 +1,21 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from Configurables import ApplicationMgr as AppMgr
 from Configurables import Reader, Writer, IOSvc
 
diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
index cafb37c5..be9224f7 100644
--- a/python/k4FWCore/IOSvc.py
+++ b/python/k4FWCore/IOSvc.py
@@ -1,3 +1,21 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from Configurables import IOSvc as IO
 import os
 
diff --git a/python/k4FWCore/__init__.py b/python/k4FWCore/__init__.py
index c8a912de..a3db4cf6 100644
--- a/python/k4FWCore/__init__.py
+++ b/python/k4FWCore/__init__.py
@@ -1,2 +1,20 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from .ApplicationMgr import ApplicationMgr
 from .IOSvc import IOSvc
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index b65a99a7..1b20ee8b 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -1,3 +1,21 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 import podio
 
 def check_collections(filename, names):
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index f7be2442..e3c7532a 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -1,3 +1,4 @@
+#
 # Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 87ee0b43..5dcc2abc 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -1,3 +1,4 @@
+#
 # Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 58436c7f..3336c010 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -40,5 +40,3 @@
                OutputLevel=INFO,
                )
 
-
-    
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index e7de43b5..e0ba337c 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -52,8 +52,8 @@
                EvtSel="NONE",
                EvtMax=-1,
                ExtSvc=[whiteboard],
-               EventLoop=slimeventloopmgr,      
+               EventLoop=slimeventloopmgr,
                MessageSvcType="InertMessageSvc",
                OutputLevel=INFO,
                )
-    
+
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
index 1d9ac227..bf2a1dbe 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -1,3 +1,4 @@
+#
 # Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
index 5802d48c..c4cbef5a 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
@@ -1,3 +1,4 @@
+#
 # Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
index 8caaee21..71e6fa81 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
@@ -1,3 +1,4 @@
+#
 # Copyright (c) 2014-2024 Key4hep-Project.
 #
 # This file is part of Key4hep.
@@ -58,10 +59,10 @@
                                                        InputCollectionSimTrackerHits=["SimTrackerHits0", "SimTrackerHits1", "SimTrackerHits2"],
                                                        InputCollectionTrackerHits=["TrackerHits0", "TrackerHits1", "TrackerHits2"],
                                                        InputCollectionTracks=["Tracks0", "Tracks1", "Tracks2"],
-                                                       OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],              
+                                                       OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],
                                                        OutputCollectionParticles1=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
                                                        OutputCollectionParticles2=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
-                                                       OutputCollectionSimTrackerHits=["NewSimTrackerHits0", "NewSimTrackerHits1", "NewSimTrackerHits2"],           
+                                                       OutputCollectionSimTrackerHits=["NewSimTrackerHits0", "NewSimTrackerHits1", "NewSimTrackerHits2"],
                                                        OutputCollectionTrackerHits=["NewTrackerHits0", "NewTrackerHits1", "NewTrackerHits2"],
                                                        OutputCollectionTracks=["NewTracks0", "NewTracks1", "NewTracks2"],
                                                        Offset=0,

From 9307ba821205a126efa3e76ecf268549dba0403e Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 14:38:29 +0100
Subject: [PATCH 045/127] Format

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 7af3c523..1ba758b2 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -40,9 +40,7 @@ namespace k4FWCore {
 
     template <typename T, typename P>
       requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
-    const auto& maybeTransformToEDM4hep(const P& arg) {
-      return static_cast<const T&>(*arg);
-    }
+    const auto& maybeTransformToEDM4hep(const P& arg) { return static_cast<const T&>(*arg); }
 
     template <typename T> struct is_map_like : std::false_type {};
 

From 62aaa446b399a5d60469a1b04160467361aef904 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 14:45:45 +0100
Subject: [PATCH 046/127] Format

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 1ba758b2..9652127c 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -39,7 +39,7 @@ namespace k4FWCore {
     template <typename T, typename P> const auto& maybeTransformToEDM4hep(const P& arg) { return arg; }
 
     template <typename T, typename P>
-      requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
+    requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
     const auto& maybeTransformToEDM4hep(const P& arg) { return static_cast<const T&>(*arg); }
 
     template <typename T> struct is_map_like : std::false_type {};

From cac24cb8bd82ae12e0496a464e353c8cc8d371c8 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 22:20:53 +0100
Subject: [PATCH 047/127] Add the genConfDir that seems to be necessary
 sometimes

for running the tests without installing
---
 test/k4FWCoreTest/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 4f618a0a..ed987c88 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -29,7 +29,7 @@ set(K4RUN ${PROJECT_SOURCE_DIR}/k4FWCore/scripts/k4run)
 
 function(set_test_env testname)
   set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "ROOT_INCLUDE_PATH=$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>/../include>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>/../include>:$ENV{ROOT_INCLUDE_PATH}")
-  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
+  set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
   set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
 endfunction()
 

From ee4519e6e3b15b48391a89c1b494c53a410d28f2 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 16 Feb 2024 22:31:27 +0100
Subject: [PATCH 048/127] Improve test command and test output when it fails

---
 test/k4FWCoreTest/CMakeLists.txt              | 2 +-
 test/k4FWCoreTest/options/CheckOutputFiles.py | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index ed987c88..34b5f407 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -94,7 +94,7 @@ PROPERTIES PASS_REGULAR_EXPRESSION "TwasBrilligAndTheSlithyToves"
 )
 add_test(NAME checkKeepDropSwitch
          WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-         COMMAND python scripts/check_KeepDropSwitch.py ${CMAKE_BINARY_DIR}/test/k4FWCoreTest/output_k4test_exampledata_2.root)
+         COMMAND python scripts/check_KeepDropSwitch.py ${PROJECT_BINARY_DIR}/test/k4FWCoreTest/output_k4test_exampledata_2.root)
 set_test_env(checkKeepDropSwitch)
 set_property(TEST checkKeepDropSwitch APPEND PROPERTY DEPENDS ReadExampleEventData)
 add_test_with_env(TestUniqueIDGenSvc options/TestUniqueIDGenSvc.py)
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index 1b20ee8b..63548748 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -16,7 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import podio
+try:
+    import podio
+except ImportError:
+    import os
+    print(f'PYTHONPATH={os.environ["PYTHONPATH"]}')
+    raise
 
 def check_collections(filename, names):
     podio_reader = podio.root_io.Reader(filename)

From 487c2a61a4728627cb13d53fbc56364cae8b38f3 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sat, 17 Feb 2024 09:32:54 +0100
Subject: [PATCH 049/127] Don't try to create any directories for inputs

---
 python/k4FWCore/IOSvc.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
index be9224f7..b927713a 100644
--- a/python/k4FWCore/IOSvc.py
+++ b/python/k4FWCore/IOSvc.py
@@ -37,9 +37,6 @@ def __setattr__(self, attr, value):
         if attr == 'input':
             if isinstance(value, str):
                 value = [value]
-            for inp in value:
-                if os.path.dirname(inp) and not os.path.exists(inp):
-                    os.makedirs(os.path.dirname(inp))
         if attr == 'output':
             if os.path.dirname(value) and not os.path.exists(os.path.dirname(value)):
                 os.makedirs(os.path.dirname(value))

From 9ae5f734f28369a74e08ee817942b69cb24accde Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sat, 17 Feb 2024 11:28:58 +0100
Subject: [PATCH 050/127] Remove info message

---
 k4FWCore/components/IOSvc.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 607c7024..d24dd016 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -116,7 +116,6 @@ void IOSvc::handle(const Incident& incident) {
   DataObject* p;
   code = m_dataSvc->retrieveObject("/Event/_Frame", p);
   if (code.isFailure()) {
-    info() << "No frame found" << endmsg;
     return;
   }
 

From f0bb6e0d1ccc82f264bdcc5223835df66f108eb8 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sat, 17 Feb 2024 16:06:02 +0100
Subject: [PATCH 051/127] Use a SmartIF for the HiveWhiteBoard instead of a
 ServiceHandle

since we don't always want to have it and it emits a warning when it
can't find the HiveWhiteBoard
---
 k4FWCore/components/Writer.cpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 0726543a..69ee6be0 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -52,7 +52,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
   mutable std::mutex m_mutex;
 
   ServiceHandle<IIOSvc>          iosvc{this, "IOSvc", "IOSvc"};
-  ServiceHandle<IHiveWhiteBoard> m_hiveWhiteBoard{this, "EventDataSvc", "EventDataSvc"};
+  SmartIF<IHiveWhiteBoard> m_hiveWhiteBoard;
   SmartIF<IDataProviderSvc>      m_dataSvc;
   mutable bool                   m_first{true};
 
@@ -68,6 +68,12 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       return StatusCode::FAILURE;
     }
 
+    m_hiveWhiteBoard = service<IHiveWhiteBoard>("EventDataSvc", true);
+    if (!m_hiveWhiteBoard) {
+      debug() << "Unable to locate IHiveWhiteBoard interface. This isn't a problem if we are not running in a multi-threaded environment"
+             << endmsg;
+    }
+
     return StatusCode::SUCCESS;
   }
 
@@ -252,7 +258,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       }
     }
 
-    info() << "Writing frame, with time: "
+    debug() << "Writing frame, with time: "
            << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
                   .count()
            << endmsg;

From 24db3a4068f04d134f4c0356d453167ae3cddd31 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sat, 17 Feb 2024 16:13:44 +0100
Subject: [PATCH 052/127] Add a sequencer for writing

---
 python/k4FWCore/ApplicationMgr.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 57de660f..7e4ca9ea 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -18,8 +18,7 @@
 #
 from Configurables import ApplicationMgr as AppMgr
 from Configurables import Reader, Writer, IOSvc
-
-# seq = Gaudi__Sequencer("Node", Members=[reader, transformer, writer], Sequential=True, OutputLevel=INFO)
+from Configurables import Gaudi__Sequencer
 
 class ApplicationMgr:
 
@@ -32,4 +31,4 @@ def __init__(self, **kwargs):
                 if 'input' in props:
                     self._mgr.TopAlg =  [Reader("Reader")] + self._mgr.TopAlg
                 if 'output' in props:
-                    self._mgr.TopAlg = self._mgr.TopAlg + [Writer("Writer")]
+                    self._mgr.TopAlg = [Gaudi__Sequencer("Sequencer", Members=[*self._mgr.TopAlg, Writer("Writer")], Sequential=True)]

From cb92395c0e13b670817d807891262e00a70f5d29 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 21 Feb 2024 19:23:27 +0100
Subject: [PATCH 053/127] Remove brackets

---
 test/k4FWCoreTest/options/ExampleFunctionalMTFile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index e0ba337c..f204f60e 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -40,7 +40,7 @@
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
-svc.input = ['output_k4test_exampledata_producer.root']
+svc.input = 'output_k4test_exampledata_producer.root'
 svc.output = 'functional_transformerMT.root'
 
 

From 5954d05af17259fab7a2dbcb4d5f24f0eced0604 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 22 Feb 2024 22:27:45 +0100
Subject: [PATCH 054/127] Format

---
 k4FWCore/components/Writer.cpp | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 69ee6be0..7b1fb334 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -51,10 +51,10 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
   mutable std::mutex m_mutex;
 
-  ServiceHandle<IIOSvc>          iosvc{this, "IOSvc", "IOSvc"};
-  SmartIF<IHiveWhiteBoard> m_hiveWhiteBoard;
-  SmartIF<IDataProviderSvc>      m_dataSvc;
-  mutable bool                   m_first{true};
+  ServiceHandle<IIOSvc>     iosvc{this, "IOSvc", "IOSvc"};
+  SmartIF<IHiveWhiteBoard>  m_hiveWhiteBoard;
+  SmartIF<IDataProviderSvc> m_dataSvc;
+  mutable bool              m_first{true};
 
   StatusCode initialize() override {
     if (!iosvc.isValid()) {
@@ -70,8 +70,9 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
     m_hiveWhiteBoard = service<IHiveWhiteBoard>("EventDataSvc", true);
     if (!m_hiveWhiteBoard) {
-      debug() << "Unable to locate IHiveWhiteBoard interface. This isn't a problem if we are not running in a multi-threaded environment"
-             << endmsg;
+      debug() << "Unable to locate IHiveWhiteBoard interface. This isn't a problem if we are not running in a "
+                 "multi-threaded environment"
+              << endmsg;
     }
 
     return StatusCode::SUCCESS;
@@ -259,9 +260,10 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     }
 
     debug() << "Writing frame, with time: "
-           << std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
-                  .count()
-           << endmsg;
+            << std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+                   .count()
+            << endmsg;
     iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
   }
 };

From e27742a7ea30a85380d90035414c7f322619b689 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 29 Feb 2024 10:10:00 +0100
Subject: [PATCH 055/127] Format

---
 test/k4FWCoreTest/options/CheckOutputFiles.py |  38 +++-
 .../ExampleFunctionalConsumerMemory.py        |  27 +--
 ...ExampleFunctionalConsumerMultipleMemory.py |  56 +++---
 ...pleFunctionalConsumerRuntimeCollections.py |  50 +++--
 ...ionalConsumerRuntimeCollectionsMultiple.py |  94 +++++----
 .../options/ExampleFunctionalFile.py          |  24 +--
 .../options/ExampleFunctionalFileMultiple.py  |  26 +--
 .../options/ExampleFunctionalMTFile.py        |  41 ++--
 .../options/ExampleFunctionalMTMemory.py      |  48 +++--
 .../ExampleFunctionalMultipleMemory.py        |  57 +++---
 .../ExampleFunctionalOutputCommands.py        |  34 ++--
 .../ExampleFunctionalProducerAbsolutePath.py  |  13 +-
 ...pleFunctionalProducerRuntimeCollections.py |  52 ++---
 ...FunctionalTransformerRuntimeCollections.py |  84 ++++----
 ...alTransformerRuntimeCollectionsMultiple.py | 179 +++++++++++-------
 .../options/runEventHeaderCheck.py            |   4 +-
 test/k4FWCoreTest/options/runFunctionalMix.py |  22 ++-
 .../ExampleFunctionalProducerMultiple.cpp     |  11 +-
 ...lTransformerRuntimeCollectionsMultiple.cpp |  15 +-
 19 files changed, 522 insertions(+), 353 deletions(-)

diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index 63548748..0219aaf0 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -20,19 +20,41 @@
     import podio
 except ImportError:
     import os
+
     print(f'PYTHONPATH={os.environ["PYTHONPATH"]}')
     raise
 
+
 def check_collections(filename, names):
     podio_reader = podio.root_io.Reader(filename)
-    for frame in podio_reader.get('events'):
+    for frame in podio_reader.get("events"):
         available = set(frame.getAvailableCollections())
         if available != set(names):
-            print(f'These collections should be in the frame but are not: {set(names) - available}')
-            print(f'These collections are in the frame but should not be: {available - set(names)}')
-            raise RuntimeError('Collections in frame do not match expected collections')
+            print(
+                f"These collections should be in the frame but are not: {set(names) - available}"
+            )
+            print(
+                f"These collections are in the frame but should not be: {available - set(names)}"
+            )
+            raise RuntimeError("Collections in frame do not match expected collections")
+
 
-check_collections('functional_transformer.root', ['MCParticles', 'NewMCParticles'])
-check_collections('functional_transformer_multiple.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits', 'Tracks', 'Counter', 'NewMCParticles'])
-check_collections('functional_transformer_multiple_output_commands.root', ['VectorFloat', 'MCParticles1', 'MCParticles2', 'SimTrackerHits', 'TrackerHits'])
-check_collections('/tmp/a/b/c/output_k4test_exampledata_producer.root', ['MCParticles'])
+check_collections("functional_transformer.root", ["MCParticles", "NewMCParticles"])
+check_collections(
+    "functional_transformer_multiple.root",
+    [
+        "VectorFloat",
+        "MCParticles1",
+        "MCParticles2",
+        "SimTrackerHits",
+        "TrackerHits",
+        "Tracks",
+        "Counter",
+        "NewMCParticles",
+    ],
+)
+check_collections(
+    "functional_transformer_multiple_output_commands.root",
+    ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"],
+)
+check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
index a62197e6..559e1ccc 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
@@ -25,17 +25,20 @@
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr
 
-producer = ExampleFunctionalProducer("ExampleFunctionalProducer",
-                                     OutputCollection="MCParticles",
-                                     )
+producer = ExampleFunctionalProducer(
+    "ExampleFunctionalProducer",
+    OutputCollection="MCParticles",
+)
 
-consumer = ExampleFunctionalConsumer("ExampleFunctionalConsumer",
-                                     InputCollection="MCParticles",
-                                     )
+consumer = ExampleFunctionalConsumer(
+    "ExampleFunctionalConsumer",
+    InputCollection="MCParticles",
+)
 
-ApplicationMgr(TopAlg=[producer, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
index 9ccf78d3..3ea8afe9 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
@@ -21,33 +21,39 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerMultiple
+from Configurables import (
+    ExampleFunctionalProducerMultiple,
+    ExampleFunctionalConsumerMultiple,
+)
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr
 
-producer = ExampleFunctionalProducerMultiple("Producer",
-                                             OutputCollectionFloat="VectorFloat",
-                                             OutputCollectionParticles1="MCParticles1",
-                                             OutputCollectionParticles2="MCParticles2",
-                                             OutputCollectionSimTrackerHits="SimTrackerHits",
-                                             OutputCollectionTrackerHits="TrackerHits",
-                                             OutputCollectionTracks="Tracks",
-                                             ExampleInt=5
-                                             )
+producer = ExampleFunctionalProducerMultiple(
+    "Producer",
+    OutputCollectionFloat="VectorFloat",
+    OutputCollectionParticles1="MCParticles1",
+    OutputCollectionParticles2="MCParticles2",
+    OutputCollectionSimTrackerHits="SimTrackerHits",
+    OutputCollectionTrackerHits="TrackerHits",
+    OutputCollectionTracks="Tracks",
+    ExampleInt=5,
+)
 
-consumer = ExampleFunctionalConsumerMultiple("Consumer",
-                                             InputCollectionFloat="VectorFloat",
-                                             InputCollectionParticles1="MCParticles1",
-                                             InputCollectionParticles2="MCParticles2",
-                                             InputCollectionSimTrackerHits="SimTrackerHits",
-                                             InputCollectionTrackerHits="TrackerHits",
-                                             InputCollectionTracks="Tracks",
-                                             ExampleInt=5
-                                             )
+consumer = ExampleFunctionalConsumerMultiple(
+    "Consumer",
+    InputCollectionFloat="VectorFloat",
+    InputCollectionParticles1="MCParticles1",
+    InputCollectionParticles2="MCParticles2",
+    InputCollectionSimTrackerHits="SimTrackerHits",
+    InputCollectionTrackerHits="TrackerHits",
+    InputCollectionTracks="Tracks",
+    ExampleInt=5,
+)
 
-ApplicationMgr(TopAlg=[producer, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index e3c7532a..0109faaf 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -21,29 +21,37 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumerRuntimeCollections
+from Configurables import (
+    ExampleFunctionalProducer,
+    ExampleFunctionalConsumerRuntimeCollections,
+)
 from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
-producer0 = ExampleFunctionalProducer("Producer0",
-                                     OutputCollection="MCParticles0",
-                                     )
-producer1 = ExampleFunctionalProducer("Producer1",
-                                     OutputCollection="MCParticles1",
-                                     )
-producer2 = ExampleFunctionalProducer("Producer2",
-                                     OutputCollection="MCParticles2",
-                                     )
-consumer = ExampleFunctionalConsumerRuntimeCollections("Consumer",
-                                     # InputCollection="MCParticles0 MCParticles1 MCParticles2",
-                                     InputCollection=["MCParticles0", "MCParticles1", "MCParticles2"],
-                                     Offset=0,
-                                     )
+producer0 = ExampleFunctionalProducer(
+    "Producer0",
+    OutputCollection="MCParticles0",
+)
+producer1 = ExampleFunctionalProducer(
+    "Producer1",
+    OutputCollection="MCParticles1",
+)
+producer2 = ExampleFunctionalProducer(
+    "Producer2",
+    OutputCollection="MCParticles2",
+)
+consumer = ExampleFunctionalConsumerRuntimeCollections(
+    "Consumer",
+    # InputCollection="MCParticles0 MCParticles1 MCParticles2",
+    InputCollection=["MCParticles0", "MCParticles1", "MCParticles2"],
+    Offset=0,
+)
 
 
-ApplicationMgr(TopAlg=[producer0, producer1, producer2, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer0, producer1, producer2, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 5dcc2abc..b7706925 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -21,49 +21,63 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalConsumerRuntimeCollectionsMultiple
+from Configurables import (
+    ExampleFunctionalProducerMultiple,
+    ExampleFunctionalConsumerRuntimeCollectionsMultiple,
+)
 from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
-producer0 = ExampleFunctionalProducerMultiple("Producer0",
-                                              OutputCollectionFloat="VectorFloat0",
-                                              OutputCollectionParticles1="MCParticles0",
-                                              OutputCollectionParticles2="MCParticles1",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
-                                              OutputCollectionTrackerHits="TrackerHits0",
-                                              OutputCollectionTracks="Tracks0",
-                                              ExampleInt=5
-                                              )
-producer1 = ExampleFunctionalProducerMultiple("Producer1",
-                                              OutputCollectionFloat="VectorFloat1",
-                                              OutputCollectionParticles1="MCParticles2",
-                                              OutputCollectionParticles2="MCParticles3",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
-                                              OutputCollectionTrackerHits="TrackerHits1",
-                                              OutputCollectionTracks="Tracks1",
-                                              ExampleInt=5
-                                              )
-producer2 = ExampleFunctionalProducerMultiple("Producer2",
-                                              OutputCollectionFloat="VectorFloat2",
-                                              OutputCollectionParticles1="MCParticles4",
-                                              OutputCollectionParticles2="MCParticles5",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
-                                              OutputCollectionTrackerHits="TrackerHits2",
-                                              OutputCollectionTracks="Tracks2",
-                                              ExampleInt=5
-                                              )
+producer0 = ExampleFunctionalProducerMultiple(
+    "Producer0",
+    OutputCollectionFloat="VectorFloat0",
+    OutputCollectionParticles1="MCParticles0",
+    OutputCollectionParticles2="MCParticles1",
+    OutputCollectionSimTrackerHits="SimTrackerHits0",
+    OutputCollectionTrackerHits="TrackerHits0",
+    OutputCollectionTracks="Tracks0",
+    ExampleInt=5,
+)
+producer1 = ExampleFunctionalProducerMultiple(
+    "Producer1",
+    OutputCollectionFloat="VectorFloat1",
+    OutputCollectionParticles1="MCParticles2",
+    OutputCollectionParticles2="MCParticles3",
+    OutputCollectionSimTrackerHits="SimTrackerHits1",
+    OutputCollectionTrackerHits="TrackerHits1",
+    OutputCollectionTracks="Tracks1",
+    ExampleInt=5,
+)
+producer2 = ExampleFunctionalProducerMultiple(
+    "Producer2",
+    OutputCollectionFloat="VectorFloat2",
+    OutputCollectionParticles1="MCParticles4",
+    OutputCollectionParticles2="MCParticles5",
+    OutputCollectionSimTrackerHits="SimTrackerHits2",
+    OutputCollectionTrackerHits="TrackerHits2",
+    OutputCollectionTracks="Tracks2",
+    ExampleInt=5,
+)
 
-consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple("Consumer",
-                                                               Particles=["MCParticles0", "MCParticles1", "MCParticles2", "MCParticles3", "MCParticles4"],
-                                                               Tracks=["Tracks0", "Tracks1", "Tracks2"],
-                                                               SimTrackerHits="SimTrackerHits0",
-                                                               Offset=0,
-                                                               )
+consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple(
+    "Consumer",
+    Particles=[
+        "MCParticles0",
+        "MCParticles1",
+        "MCParticles2",
+        "MCParticles3",
+        "MCParticles4",
+    ],
+    Tracks=["Tracks0", "Tracks1", "Tracks2"],
+    SimTrackerHits="SimTrackerHits0",
+    Offset=0,
+)
 
 
-ApplicationMgr(TopAlg=[producer0, producer1, producer2, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer0, producer1, producer2, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index 3336c010..bcf60a42 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -26,17 +26,17 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = 'output_k4test_exampledata_producer.root'
-svc.output = 'functional_transformer.root'
+svc.input = "output_k4test_exampledata_producer.root"
+svc.output = "functional_transformer.root"
 
-transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
-
-mgr = ApplicationMgr(TopAlg=[transformer],
-               EvtSel="NONE",
-               EvtMax=-1,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+transformer = ExampleFunctionalTransformer(
+    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+)
 
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
index 57cf1c32..211165ba 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
@@ -26,17 +26,19 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ['output_k4test_exampledata_producer_multiple.root']
-svc.output = 'functional_transformer_multiple.root'
+svc.input = ["output_k4test_exampledata_producer_multiple.root"]
+svc.output = "functional_transformer_multiple.root"
 
-transformer = ExampleFunctionalTransformerMultiple("Transformer",
-                                           # InputCollection="MCParticles",
-                                           # OutputCollection="NewMCParticles")
-                                                   )
+transformer = ExampleFunctionalTransformerMultiple(
+    "Transformer",
+    # InputCollection="MCParticles",
+    # OutputCollection="NewMCParticles")
+)
 
-mgr = ApplicationMgr(TopAlg=[transformer],
-               EvtSel="NONE",
-               EvtMax=-1,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index f204f60e..218ef7fe 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -28,32 +28,33 @@
 evtslots = 2
 threads = 2
 
-whiteboard = HiveWhiteBoard("EventDataSvc",
-                            EventSlots=evtslots,
-                            ForceLeaves=True,
-                            )
+whiteboard = HiveWhiteBoard(
+    "EventDataSvc",
+    EventSlots=evtslots,
+    ForceLeaves=True,
+)
 
-slimeventloopmgr = HiveSlimEventLoopMgr("HiveSlimEventLoopMgr",
-    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
+slimeventloopmgr = HiveSlimEventLoopMgr(
+    "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
 )
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
-svc.input = 'output_k4test_exampledata_producer.root'
-svc.output = 'functional_transformerMT.root'
-
+svc.input = "output_k4test_exampledata_producer.root"
+svc.output = "functional_transformerMT.root"
 
-transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
 
-mgr = ApplicationMgr(TopAlg=[transformer],
-               EvtSel="NONE",
-               EvtMax=-1,
-               ExtSvc=[whiteboard],
-               EventLoop=slimeventloopmgr,
-               MessageSvcType="InertMessageSvc",
-               OutputLevel=INFO,
-               )
+transformer = ExampleFunctionalTransformer(
+    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+)
 
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[whiteboard],
+    EventLoop=slimeventloopmgr,
+    MessageSvcType="InertMessageSvc",
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index 6b9753bc..e7044b77 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -21,17 +21,22 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO, WARNING
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalTransformer, ExampleFunctionalConsumer
+from Configurables import (
+    ExampleFunctionalProducer,
+    ExampleFunctionalTransformer,
+    ExampleFunctionalConsumer,
+)
 from Configurables import HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc
 from k4FWCore import ApplicationMgr
 
 evtslots = 5
 threads = 3
 
-whiteboard = HiveWhiteBoard("EventDataSvc",
-                            EventSlots=evtslots,
-                            ForceLeaves=True,
-                            )
+whiteboard = HiveWhiteBoard(
+    "EventDataSvc",
+    EventSlots=evtslots,
+    ForceLeaves=True,
+)
 
 slimeventloopmgr = HiveSlimEventLoopMgr(
     SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
@@ -39,23 +44,24 @@
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 
-transformer = ExampleFunctionalTransformer("Transformer",
-                                           InputCollection="MCParticles",
-                                           OutputCollection="NewMCParticles")
+transformer = ExampleFunctionalTransformer(
+    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+)
 
-producer = ExampleFunctionalProducer("Producer",
-                                     OutputCollection="MCParticles")
+producer = ExampleFunctionalProducer("Producer", OutputCollection="MCParticles")
 
-consumer = ExampleFunctionalConsumer("Consumer",
-                                     InputCollection="NewMCParticles",
-                                     Offset=10,
-                                     )
+consumer = ExampleFunctionalConsumer(
+    "Consumer",
+    InputCollection="NewMCParticles",
+    Offset=10,
+)
 
 
-ApplicationMgr(TopAlg=[producer, transformer, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               EventLoop=slimeventloopmgr,
-               ExtSvc=[whiteboard],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer, transformer, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    EventLoop=slimeventloopmgr,
+    ExtSvc=[whiteboard],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index ee2f1284..2773dbbc 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -21,36 +21,43 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducerMultiple, ExampleFunctionalTransformerMultiple, ExampleFunctionalConsumerMultiple
+from Configurables import (
+    ExampleFunctionalProducerMultiple,
+    ExampleFunctionalTransformerMultiple,
+    ExampleFunctionalConsumerMultiple,
+)
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr
 
-transformer = ExampleFunctionalTransformerMultiple("Transformer",
-                                                   InputCollectionFloat="VectorFloat",
-                                                   InputCollectionParticles="MCParticles1",
-                                                   InputCollectionSimTrackerHits="SimTrackerHits",
-                                                   InputCollectionTrackerHits="TrackerHits",
-                                                   InputCollectionTracks="Tracks",
-                                                   OutputCollectionCounter="Counter",
-                                                   OutputCollectionParticles="NewMCParticles",
-                                                   Offset=10,
-                                                   )
+transformer = ExampleFunctionalTransformerMultiple(
+    "Transformer",
+    InputCollectionFloat="VectorFloat",
+    InputCollectionParticles="MCParticles1",
+    InputCollectionSimTrackerHits="SimTrackerHits",
+    InputCollectionTrackerHits="TrackerHits",
+    InputCollectionTracks="Tracks",
+    OutputCollectionCounter="Counter",
+    OutputCollectionParticles="NewMCParticles",
+    Offset=10,
+)
 
 producer = ExampleFunctionalProducerMultiple("Producer")
 
-consumer = ExampleFunctionalConsumerMultiple("Consumer",
-                                             InputCollectionFloat="VectorFloat",
-                                             InputCollectionParticles="NewMCParticles",
-                                             InputCollectionSimTrackerHits="SimTrackerHits",
-                                             InputCollectionTrackerHits="TrackerHits",
-                                             InputCollectionTracks="Tracks",
-                                             Offset=10,
-                                             )
+consumer = ExampleFunctionalConsumerMultiple(
+    "Consumer",
+    InputCollectionFloat="VectorFloat",
+    InputCollectionParticles="NewMCParticles",
+    InputCollectionSimTrackerHits="SimTrackerHits",
+    InputCollectionTrackerHits="TrackerHits",
+    InputCollectionTracks="Tracks",
+    Offset=10,
+)
 
 
-ApplicationMgr(TopAlg=[producer, transformer, consumer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer, transformer, consumer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
index 8611665c..b87343f3 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
@@ -26,20 +26,24 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ['output_k4test_exampledata_producer_multiple.root']
-svc.output = 'functional_transformer_multiple_output_commands.root'
-svc.outputCommands = ["drop Tracks",
-                      "drop Counter",
-                      "drop NewMCParticles",]
+svc.input = ["output_k4test_exampledata_producer_multiple.root"]
+svc.output = "functional_transformer_multiple_output_commands.root"
+svc.outputCommands = [
+    "drop Tracks",
+    "drop Counter",
+    "drop NewMCParticles",
+]
 
-transformer = ExampleFunctionalTransformerMultiple("Transformer",
-                                           # InputCollection="MCParticles",
-                                           # OutputCollection="NewMCParticles")
-                                                   )
+transformer = ExampleFunctionalTransformerMultiple(
+    "Transformer",
+    # InputCollection="MCParticles",
+    # OutputCollection="NewMCParticles")
+)
 
-mgr = ApplicationMgr(TopAlg=[transformer],
-               EvtSel="NONE",
-               EvtMax=-1,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+mgr = ApplicationMgr(
+    TopAlg=[transformer],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
index 32b02c3e..98cf71a3 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py
@@ -29,9 +29,10 @@
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 
-ApplicationMgr(TopAlg=[producer],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
index bf2a1dbe..6e55dfdf 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -21,31 +21,39 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducerRuntimeCollections, ExampleFunctionalConsumer
+from Configurables import (
+    ExampleFunctionalProducerRuntimeCollections,
+    ExampleFunctionalConsumer,
+)
 from Configurables import ApplicationMgr
 from Configurables import EventDataSvc
 
-producer = ExampleFunctionalProducerRuntimeCollections("Producer",
-                                                       OutputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
-                                                       )
+producer = ExampleFunctionalProducerRuntimeCollections(
+    "Producer",
+    OutputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
+)
 
-consumer0 = ExampleFunctionalConsumer("Consumer0",
-                                     InputCollection="MCParticles0",
-                                     Offset=0,
-                                     )
-consumer1 = ExampleFunctionalConsumer("Consumer1",
-                                     InputCollection="MCParticles1",
-                                     Offset=0,
-                                     )
-consumer2 = ExampleFunctionalConsumer("Consumer2",
-                                     InputCollection="MCParticles2",
-                                     Offset=0,
-                                     )
+consumer0 = ExampleFunctionalConsumer(
+    "Consumer0",
+    InputCollection="MCParticles0",
+    Offset=0,
+)
+consumer1 = ExampleFunctionalConsumer(
+    "Consumer1",
+    InputCollection="MCParticles1",
+    Offset=0,
+)
+consumer2 = ExampleFunctionalConsumer(
+    "Consumer2",
+    InputCollection="MCParticles2",
+    Offset=0,
+)
 
 
-ApplicationMgr(TopAlg=[producer, consumer0, consumer1, consumer2],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[producer, consumer0, consumer1, consumer2],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
index c4cbef5a..b05afeda 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
@@ -21,42 +21,62 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalTransformerRuntimeCollections, ExampleFunctionalConsumer, ExampleFunctionalProducer
+from Configurables import (
+    ExampleFunctionalTransformerRuntimeCollections,
+    ExampleFunctionalConsumer,
+    ExampleFunctionalProducer,
+)
 from Configurables import ApplicationMgr
 from Configurables import EventDataSvc
 
-producer0 = ExampleFunctionalProducer("Producer0",
-                                     OutputCollection="MCParticles0",
-                                     )
-producer1 = ExampleFunctionalProducer("Producer1",
-                                     OutputCollection="MCParticles1",
-                                     )
-producer2 = ExampleFunctionalProducer("Producer2",
-                                     OutputCollection="MCParticles2",
-                                     )
+producer0 = ExampleFunctionalProducer(
+    "Producer0",
+    OutputCollection="MCParticles0",
+)
+producer1 = ExampleFunctionalProducer(
+    "Producer1",
+    OutputCollection="MCParticles1",
+)
+producer2 = ExampleFunctionalProducer(
+    "Producer2",
+    OutputCollection="MCParticles2",
+)
 
-transformer = ExampleFunctionalTransformerRuntimeCollections("Transformer",
-                                                       InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
-                                                       OutputCollections=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
-                                                       )
+transformer = ExampleFunctionalTransformerRuntimeCollections(
+    "Transformer",
+    InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
+    OutputCollections=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
+)
 
-consumer0 = ExampleFunctionalConsumer("Consumer0",
-                                     InputCollection="NewMCParticles0",
-                                     Offset=0,
-                                     )
-consumer1 = ExampleFunctionalConsumer("Consumer1",
-                                     InputCollection="NewMCParticles1",
-                                     Offset=0,
-                                     )
-consumer2 = ExampleFunctionalConsumer("Consumer2",
-                                     InputCollection="NewMCParticles2",
-                                     Offset=0,
-                                     )
+consumer0 = ExampleFunctionalConsumer(
+    "Consumer0",
+    InputCollection="NewMCParticles0",
+    Offset=0,
+)
+consumer1 = ExampleFunctionalConsumer(
+    "Consumer1",
+    InputCollection="NewMCParticles1",
+    Offset=0,
+)
+consumer2 = ExampleFunctionalConsumer(
+    "Consumer2",
+    InputCollection="NewMCParticles2",
+    Offset=0,
+)
 
 
-ApplicationMgr(TopAlg=[producer0, producer1, producer2, transformer, consumer0, consumer1, consumer2],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[
+        producer0,
+        producer1,
+        producer2,
+        transformer,
+        consumer0,
+        consumer1,
+        consumer2,
+    ],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
index 71e6fa81..c84f9883 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
@@ -21,82 +21,121 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalTransformerRuntimeCollectionsMultiple, ExampleFunctionalConsumerMultiple, ExampleFunctionalProducerMultiple
+from Configurables import (
+    ExampleFunctionalTransformerRuntimeCollectionsMultiple,
+    ExampleFunctionalConsumerMultiple,
+    ExampleFunctionalProducerMultiple,
+)
 from Configurables import ApplicationMgr
 from Configurables import EventDataSvc
 
-producer0 = ExampleFunctionalProducerMultiple("Producer0",
-                                              OutputCollectionFloat="VectorFloat0",
-                                              OutputCollectionParticles1="MCParticles0",
-                                              OutputCollectionParticles2="MCParticles1",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits0",
-                                              OutputCollectionTrackerHits="TrackerHits0",
-                                              OutputCollectionTracks="Tracks0",
-                                              ExampleInt=5
-                                              )
-producer1 = ExampleFunctionalProducerMultiple("Producer1",
-                                              OutputCollectionFloat="VectorFloat1",
-                                              OutputCollectionParticles1="MCParticles2",
-                                              OutputCollectionParticles2="MCParticles3",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits1",
-                                              OutputCollectionTrackerHits="TrackerHits1",
-                                              OutputCollectionTracks="Tracks1",
-                                              ExampleInt=5
-                                              )
-producer2 = ExampleFunctionalProducerMultiple("Producer2",
-                                              OutputCollectionFloat="VectorFloat2",
-                                              OutputCollectionParticles1="MCParticles4",
-                                              OutputCollectionParticles2="MCParticles5",
-                                              OutputCollectionSimTrackerHits="SimTrackerHits2",
-                                              OutputCollectionTrackerHits="TrackerHits2",
-                                              OutputCollectionTracks="Tracks2",
-                                              ExampleInt=5
-                                              )
+producer0 = ExampleFunctionalProducerMultiple(
+    "Producer0",
+    OutputCollectionFloat="VectorFloat0",
+    OutputCollectionParticles1="MCParticles0",
+    OutputCollectionParticles2="MCParticles1",
+    OutputCollectionSimTrackerHits="SimTrackerHits0",
+    OutputCollectionTrackerHits="TrackerHits0",
+    OutputCollectionTracks="Tracks0",
+    ExampleInt=5,
+)
+producer1 = ExampleFunctionalProducerMultiple(
+    "Producer1",
+    OutputCollectionFloat="VectorFloat1",
+    OutputCollectionParticles1="MCParticles2",
+    OutputCollectionParticles2="MCParticles3",
+    OutputCollectionSimTrackerHits="SimTrackerHits1",
+    OutputCollectionTrackerHits="TrackerHits1",
+    OutputCollectionTracks="Tracks1",
+    ExampleInt=5,
+)
+producer2 = ExampleFunctionalProducerMultiple(
+    "Producer2",
+    OutputCollectionFloat="VectorFloat2",
+    OutputCollectionParticles1="MCParticles4",
+    OutputCollectionParticles2="MCParticles5",
+    OutputCollectionSimTrackerHits="SimTrackerHits2",
+    OutputCollectionTrackerHits="TrackerHits2",
+    OutputCollectionTracks="Tracks2",
+    ExampleInt=5,
+)
 
-transformer = ExampleFunctionalTransformerRuntimeCollectionsMultiple("Transformer",
-                                                       InputCollectionFloat=["VectorFloat0", "VectorFloat1", "VectorFloat2"],
-                                                       InputCollectionParticles=["MCParticles0", "MCParticles2", "MCParticles4"],
-                                                       InputCollectionSimTrackerHits=["SimTrackerHits0", "SimTrackerHits1", "SimTrackerHits2"],
-                                                       InputCollectionTrackerHits=["TrackerHits0", "TrackerHits1", "TrackerHits2"],
-                                                       InputCollectionTracks=["Tracks0", "Tracks1", "Tracks2"],
-                                                       OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],
-                                                       OutputCollectionParticles1=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
-                                                       OutputCollectionParticles2=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"],
-                                                       OutputCollectionSimTrackerHits=["NewSimTrackerHits0", "NewSimTrackerHits1", "NewSimTrackerHits2"],
-                                                       OutputCollectionTrackerHits=["NewTrackerHits0", "NewTrackerHits1", "NewTrackerHits2"],
-                                                       OutputCollectionTracks=["NewTracks0", "NewTracks1", "NewTracks2"],
-                                                       Offset=0,
-                                                       )
+transformer = ExampleFunctionalTransformerRuntimeCollectionsMultiple(
+    "Transformer",
+    InputCollectionFloat=["VectorFloat0", "VectorFloat1", "VectorFloat2"],
+    InputCollectionParticles=["MCParticles0", "MCParticles2", "MCParticles4"],
+    InputCollectionSimTrackerHits=[
+        "SimTrackerHits0",
+        "SimTrackerHits1",
+        "SimTrackerHits2",
+    ],
+    InputCollectionTrackerHits=["TrackerHits0", "TrackerHits1", "TrackerHits2"],
+    InputCollectionTracks=["Tracks0", "Tracks1", "Tracks2"],
+    OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],
+    OutputCollectionParticles1=[
+        "NewMCParticles0",
+        "NewMCParticles1",
+        "NewMCParticles2",
+    ],
+    OutputCollectionParticles2=[
+        "NewMCParticles0",
+        "NewMCParticles1",
+        "NewMCParticles2",
+    ],
+    OutputCollectionSimTrackerHits=[
+        "NewSimTrackerHits0",
+        "NewSimTrackerHits1",
+        "NewSimTrackerHits2",
+    ],
+    OutputCollectionTrackerHits=[
+        "NewTrackerHits0",
+        "NewTrackerHits1",
+        "NewTrackerHits2",
+    ],
+    OutputCollectionTracks=["NewTracks0", "NewTracks1", "NewTracks2"],
+    Offset=0,
+)
 
-consumer0 = ExampleFunctionalConsumerMultiple("Consumer0",
-                                             InputCollectionFloat="NewVectorFloat0",
-                                             InputCollectionParticles="NewMCParticles0",
-                                             InputCollectionSimTrackerHits="NewSimTrackerHits0",
-                                             InputCollectionTrackerHits="NewTrackerHits0",
-                                             InputCollectionTracks="NewTracks0",
-                                             )
+consumer0 = ExampleFunctionalConsumerMultiple(
+    "Consumer0",
+    InputCollectionFloat="NewVectorFloat0",
+    InputCollectionParticles="NewMCParticles0",
+    InputCollectionSimTrackerHits="NewSimTrackerHits0",
+    InputCollectionTrackerHits="NewTrackerHits0",
+    InputCollectionTracks="NewTracks0",
+)
 
-consumer1 = ExampleFunctionalConsumerMultiple("Consumer1",
-                                                InputCollectionFloat="NewVectorFloat1",
-                                                InputCollectionParticles="NewMCParticles2",
-                                                InputCollectionSimTrackerHits="NewSimTrackerHits1",
-                                                InputCollectionTrackerHits="NewTrackerHits1",
-                                                InputCollectionTracks="NewTracks1",
-                                                )
+consumer1 = ExampleFunctionalConsumerMultiple(
+    "Consumer1",
+    InputCollectionFloat="NewVectorFloat1",
+    InputCollectionParticles="NewMCParticles2",
+    InputCollectionSimTrackerHits="NewSimTrackerHits1",
+    InputCollectionTrackerHits="NewTrackerHits1",
+    InputCollectionTracks="NewTracks1",
+)
 
-consumer2 = ExampleFunctionalConsumerMultiple("Consumer2",
-                                                InputCollectionFloat="NewVectorFloat2",
-                                                InputCollectionParticles="NewMCParticles4",
-                                                InputCollectionSimTrackerHits="NewSimTrackerHits2",
-                                                InputCollectionTrackerHits="NewTrackerHits2",
-                                                InputCollectionTracks="NewTracks2",
-                                                )
+consumer2 = ExampleFunctionalConsumerMultiple(
+    "Consumer2",
+    InputCollectionFloat="NewVectorFloat2",
+    InputCollectionParticles="NewMCParticles4",
+    InputCollectionSimTrackerHits="NewSimTrackerHits2",
+    InputCollectionTrackerHits="NewTrackerHits2",
+    InputCollectionTracks="NewTracks2",
+)
 
 
-ApplicationMgr(TopAlg=[producer0, producer1, producer2, transformer,
-                       consumer0, consumer1, consumer2],
-               EvtSel="NONE",
-               EvtMax=10,
-               ExtSvc=[EventDataSvc("EventDataSvc")],
-               OutputLevel=INFO,
-               )
+ApplicationMgr(
+    TopAlg=[
+        producer0,
+        producer1,
+        producer2,
+        transformer,
+        consumer0,
+        consumer1,
+        consumer2,
+    ],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py
index b0b40d88..d0fa63ea 100644
--- a/test/k4FWCoreTest/options/runEventHeaderCheck.py
+++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py
@@ -29,7 +29,9 @@
 
 reader = Reader("Reader")
 
-consumer = ExampleEventHeaderConsumer("EventHeaderCheck", runNumber=42, eventNumberOffset=42)
+consumer = ExampleEventHeaderConsumer(
+    "EventHeaderCheck", runNumber=42, eventNumberOffset=42
+)
 
 ApplicationMgr(
     TopAlg=[reader, consumer],
diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py
index 2752e028..42a909ab 100644
--- a/test/k4FWCoreTest/options/runFunctionalMix.py
+++ b/test/k4FWCoreTest/options/runFunctionalMix.py
@@ -21,8 +21,14 @@
 #
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalConsumerMultiple, ExampleFunctionalTransformerMultiple
-from Configurables import ExampleFunctionalProducerMultiple, k4FWCoreTest_CreateExampleEventData
+from Configurables import (
+    ExampleFunctionalConsumerMultiple,
+    ExampleFunctionalTransformerMultiple,
+)
+from Configurables import (
+    ExampleFunctionalProducerMultiple,
+    k4FWCoreTest_CreateExampleEventData,
+)
 from Configurables import k4FWCoreTest_CheckExampleEventData
 from Configurables import ApplicationMgr
 from Configurables import k4DataSvc
@@ -41,7 +47,9 @@
     "Tracks",
 ]
 
-consumer_input_functional = ExampleFunctionalConsumerMultiple("ExampleFunctionalConsumerMultiple")
+consumer_input_functional = ExampleFunctionalConsumerMultiple(
+    "ExampleFunctionalConsumerMultiple"
+)
 consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData")
 consumer_input_algorithm.mcparticles = "MCParticles1"
 consumer_input_algorithm.keepEventNumberZero = True
@@ -74,13 +82,17 @@
 producer_algorithm.tracks = "Tracks__"
 producer_algorithm.vectorfloat = "VectorFloat__"
 
-consumer_produceralg_functional = ExampleFunctionalConsumerMultiple("FunctionalConsumerAlgorithm")
+consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
+    "FunctionalConsumerAlgorithm"
+)
 consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
 consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
 consumer_produceralg_algorithm.keepEventNumberZero = True
 
 # Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
+transformer_functional = ExampleFunctionalTransformerMultiple(
+    "FunctionalTransformerMultiple"
+)
 
 out = PodioOutput("out")
 out.filename = "output_k4test_exampledata_functional_mix.root"
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index fe521b49..fbae03c4 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -25,7 +25,14 @@
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
+#if __has_include("edm4hep/TrackerHit3DCollection.h")
+#include "edm4hep/TrackerHit3DCollection.h"
+#else
 #include "edm4hep/TrackerHitCollection.h"
+namespace edm4hep {
+  using TrackerHit3DCollection = edm4hep::TrackerHitCollection;
+}  // namespace edm4hep
+#endif
 #include "podio/UserDataCollection.h"
 
 #include <string>
@@ -34,7 +41,7 @@
 
 using retType =
     std::tuple<podio::UserDataCollection<float>, edm4hep::MCParticleCollection, edm4hep::MCParticleCollection,
-               edm4hep::SimTrackerHitCollection, edm4hep::TrackerHitCollection, edm4hep::TrackCollection>;
+               edm4hep::SimTrackerHitCollection, edm4hep::TrackerHit3DCollection, edm4hep::TrackCollection>;
 
 struct ExampleFunctionalProducerMultiple final : k4FWCore::Producer<retType()> {
   // The pairs in KeyValue can be changed from python and they correspond
@@ -68,7 +75,7 @@ struct ExampleFunctionalProducerMultiple final : k4FWCore::Producer<retType()> {
     auto hit            = simTrackerHits.create();
     hit.setPosition({3, 4, 5});
 
-    auto trackerHits = edm4hep::TrackerHitCollection();
+    auto trackerHits = edm4hep::TrackerHit3DCollection();
     auto trackerHit  = trackerHits.create();
     trackerHit.setPosition({3, 4, 5});
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
index 60790743..d460b9d4 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
@@ -22,7 +22,14 @@
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
+#if __has_include("edm4hep/TrackerHit3DCollection.h")
+#include "edm4hep/TrackerHit3DCollection.h"
+#else
 #include "edm4hep/TrackerHitCollection.h"
+namespace edm4hep {
+  using TrackerHit3DCollection = edm4hep::TrackerHitCollection;
+}  // namespace edm4hep
+#endif
 #include "podio/UserDataCollection.h"
 
 #include "k4FWCore/Transformer.h"
@@ -35,14 +42,14 @@
 using FloatColl         = std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>;
 using ParticleColl      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
 using SimTrackerHitColl = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>;
-using TrackerHitColl    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>;
+using TrackerHitColl    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>;
 using TrackColl         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>;
 
 using retType = std::tuple<std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>,
                            std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
                            std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
                            std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>,
+                           std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>,
                            std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>>;
 
 struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
@@ -78,7 +85,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     auto particleMapOut      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
     auto particle2MapOut     = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
     auto simTrackerHitMapOut = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>();
-    auto trackerHitMapOut    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHitCollection>>();
+    auto trackerHitMapOut    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>();
     auto trackMapOut         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>();
 
     if (floatMap.size() != 3) {
@@ -154,7 +161,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     }
 
     for (auto& [key, trackerHits] : trackerHitMap) {
-      auto ptr = std::make_shared<edm4hep::TrackerHitCollection>();
+      auto ptr = std::make_shared<edm4hep::TrackerHit3DCollection>();
       if ((trackerHits->at(0).getPosition()[0] != 3) || (trackerHits->at(0).getPosition()[1] != 4) ||
           (trackerHits->at(0).getPosition()[2] != 5)) {
         std::stringstream error;

From 0b15c1a5c6c1eea05cb172cae99fdf0e96fd765c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:27:52 +0100
Subject: [PATCH 056/127] Delete old files and remove debugging warnings

---
 test/k4FWCoreTest/CMakeLists.txt              |  3 -
 .../options/runExampleFunctionalConsumer.py   | 48 -------------
 .../runExampleFunctionalConsumerMultiple.py   | 52 --------------
 ...runExampleFunctionalTransformerMultiple.py | 67 -------------------
 4 files changed, 170 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/runExampleFunctionalConsumer.py
 delete mode 100644 test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py
 delete mode 100644 test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 34b5f407..bf456202 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -33,9 +33,6 @@ function(set_test_env testname)
   set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
 endfunction()
 
-message(WARNING "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$<TARGET_EXISTS:ROOT::Core>:$<TARGET_FILE_DIR:ROOT::Core>>:$<$<TARGET_EXISTS:EDM4HEP::edm4hep>:$<TARGET_FILE_DIR:EDM4HEP::edm4hep>>:$<$<TARGET_EXISTS:podio::podio>:$<TARGET_FILE_DIR:podio::podio>>:$ENV{LD_LIBRARY_PATH}")
-message(WARNING "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}")
-
 function(add_test_with_env testname)
   foreach(arg ${ARGN})
     if(arg STREQUAL "PROPERTIES")
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalConsumer.py b/test/k4FWCoreTest/options/runExampleFunctionalConsumer.py
deleted file mode 100644
index e8637492..00000000
--- a/test/k4FWCoreTest/options/runExampleFunctionalConsumer.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example reading from a file and using a consumer with a single input
-# to check that the contents of the file are the expected ones
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalConsumer
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioInput
-
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer.root"
-
-inp = PodioInput()
-inp.collections = [
-    "MCParticles",
-]
-
-consumer = ExampleFunctionalConsumer(
-    "ExampleFunctionalConsumer",
-    InputCollection="MCParticles",
-)
-
-ApplicationMgr(
-    TopAlg=[inp, consumer],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[podioevent],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py b/test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py
deleted file mode 100644
index d103921b..00000000
--- a/test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalConsumerMultiple
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioInput
-
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer_multiple.root"
-
-inp = PodioInput()
-inp.collections = [
-    "VectorFloat",
-    "MCParticles1",
-    "MCParticles2",
-    "SimTrackerHits",
-    "TrackerHits",
-    "Tracks",
-]
-
-consumer = ExampleFunctionalConsumerMultiple(
-    "ExampleFunctionalConsumerMultiple",
-)
-
-ApplicationMgr(
-    TopAlg=[inp, consumer],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[podioevent],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py b/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py
deleted file mode 100644
index 44356fe1..00000000
--- a/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalTransformerMultiple
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
-from Configurables import PodioInput
-
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer_multiple.root"
-
-inp = PodioInput()
-inp.collections = [
-    "VectorFloat",
-    "MCParticles1",
-    "SimTrackerHits",
-    "TrackerHits",
-    "Tracks",
-]
-
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_transformer_multiple.root"
-# The collections that we don't drop will also be present in the output file
-out.outputCommands = [
-    "drop VectorFloat",
-    "drop MCParticles1",
-    "drop SimTrackerHits",
-    "drop TrackerHits",
-    "drop Tracks",
-]
-
-transformer = ExampleFunctionalTransformerMultiple(
-    "ExampleFunctionalTransformerMultiple",
-    InputCollectionFloat="VectorFloat",
-    InputCollectionParticles="MCParticles1",
-    InputCollectionSimTrackerHits="SimTrackerHits",
-    InputCollectionTrackerHits="TrackerHits",
-    InputCollectionTracks="Tracks",
-    OutputCollectionCounter="Counter",
-    OutputCollectionParticles="NewMCParticles",
-)
-
-ApplicationMgr(
-    TopAlg=[inp, transformer, out],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[k4DataSvc("EventDataSvc")],
-    OutputLevel=INFO,
-)

From dfef5ae7afb1d3c80b1a01632053081ae71b5353 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:58:36 +0100
Subject: [PATCH 057/127] Fix a few tests and add some comments

---
 .../options/ExampleFunctionalMemory.py        | 25 +++-----
 .../options/ExampleFunctionalProducer.py      | 14 ++---
 .../ExampleFunctionalProducerMultiple.py      | 14 ++---
 .../options/runFunctionalChain.py             | 60 -------------------
 ...unctionalTransformerRuntimeCollections.cpp | 13 +++-
 5 files changed, 30 insertions(+), 96 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/runFunctionalChain.py

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
index 2047c939..5a61a26b 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
@@ -22,23 +22,14 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformer
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
-from Configurables import PodioInput
+from k4FWCore import ApplicationMgr, IOSvc
+from Configurables import EventDataSvc
 
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer.root"
-
-inp = PodioInput()
-inp.collections = [
-    "MCParticles",
-]
-
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_transformer.root"
+iosvc = IOSvc()
+iosvc.input = "output_k4test_exampledata_producer.root"
+iosvc.output = "output_k4test_exampledata_transformer.root"
 # The collections that we don't drop will also be present in the output file
-out.outputCommands = ["drop MCParticles"]
+# out.outputCommands = ["drop MCParticles"]
 
 transformer = ExampleFunctionalTransformer(
     "ExampleFunctionalTransformer",
@@ -47,9 +38,9 @@
 )
 
 ApplicationMgr(
-    TopAlg=[inp, transformer, out],
+    TopAlg=[transformer],
     EvtSel="NONE",
     EvtMax=10,
-    ExtSvc=[k4DataSvc("EventDataSvc")],
+    ExtSvc=[EventDataSvc("EventDataSvc")],
     OutputLevel=INFO,
 )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
index e66ff707..ae0a2584 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
@@ -21,14 +21,12 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducer
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
 
-podioevent = k4DataSvc("EventDataSvc")
 
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_producer.root"
+iosvc = IOSvc("IOSvc")
+iosvc.output = "output_k4test_exampledata_producer.root"
 # Collections can be dropped
 # out.outputCommands = ["drop *"]
 
@@ -36,9 +34,9 @@
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 
 ApplicationMgr(
-    TopAlg=[producer, out],
+    TopAlg=[producer],
     EvtSel="NONE",
     EvtMax=10,
-    ExtSvc=[k4DataSvc("EventDataSvc")],
+    ExtSvc=[EventDataSvc("EventDataSvc")],
     OutputLevel=INFO,
 )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
index b3920470..4a00715f 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
@@ -21,14 +21,12 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducerMultiple
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
+from k4FWCore import ApplicationMgr, IOSvc
+from Configurables import EventDataSvc
 
-podioevent = k4DataSvc("EventDataSvc")
 
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_producer_multiple.root"
+iosvc = IOSvc("IOSvc")
+iosvc.output = "output_k4test_exampledata_producer_multiple.root"
 # Collections can be dropped
 # out.outputCommands = ["drop *"]
 
@@ -44,9 +42,9 @@
 )
 
 ApplicationMgr(
-    TopAlg=[producer, out],
+    TopAlg=[producer],
     EvtSel="NONE",
     EvtMax=10,
-    ExtSvc=[k4DataSvc("EventDataSvc")],
+    ExtSvc=[EventDataSvc("EventDataSvc")],
     OutputLevel=INFO,
 )
diff --git a/test/k4FWCoreTest/options/runFunctionalChain.py b/test/k4FWCoreTest/options/runFunctionalChain.py
deleted file mode 100644
index ec2affd0..00000000
--- a/test/k4FWCoreTest/options/runFunctionalChain.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example of how to run several functional algorithms in a chain
-# The producer produces a collection, the first consumer checks that the collection
-# is the expected one, the transformer transforms the collection and creates a new
-# and the second consumer checks that the new collection is the expected one
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducer
-from Configurables import ExampleFunctionalConsumer
-from Configurables import ExampleFunctionalTransformer
-from Configurables import ApplicationMgr
-from Configurables import k4DataSvc
-from Configurables import PodioOutput
-
-event_data_svc = k4DataSvc("EventDataSvc")
-
-out = PodioOutput("out")
-out.filename = "functional_chain.root"
-
-producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
-consumer = ExampleFunctionalConsumer(
-    "ExampleFunctionalConsumer",
-    InputCollection="MCParticles",
-)
-transformer = ExampleFunctionalTransformer(
-    "ExampleFunctionalTransformer",
-    InputCollection="MCParticles",
-    OutputCollection="NewMCParticles",
-)
-new_consumer = ExampleFunctionalConsumer(
-    "ExampleFunctionalConsumer2",
-    InputCollection="NewMCParticles",
-)
-new_consumer.PossibleOffset = 10
-
-ApplicationMgr(
-    TopAlg=[producer, consumer, transformer, new_consumer, out],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[event_data_svc],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
index c3ffccd1..78c0f27b 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -25,6 +25,13 @@
 
 #include <string>
 
+/* ExampleFunctionalTransformerRuntimeCollections
+ *
+ * This is an example of a functional transformer that takes an arbitrary number
+ * of input collections and produces an arbitrary number of output collections
+ * (in this example the same number of input collections but it can be different)
+ */
+
 using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
 
 struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const mapType& input)> {
@@ -36,16 +43,16 @@ struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transfor
 
   // This is the function that will be called to produce the data
   mapType operator()(const mapType& input) const override {
-    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
+    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> outputCollections;
     for (int i = 0; i < input.size(); ++i) {
       std::string name     = "NewMCParticles" + std::to_string(i);
       auto        old_coll = input.at("MCParticles" + std::to_string(i));
       auto        coll     = std::make_shared<edm4hep::MCParticleCollection>();
       coll->push_back(old_coll->at(0).clone());
       coll->push_back(old_coll->at(1).clone());
-      m_outputCollections[name] = coll;
+      outputCollections[name] = coll;
     }
-    return m_outputCollections;
+    return outputCollections;
   }
 
 private:

From ccf4ff299ecd620cb18d5e9df65518d259c84680 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 5 Mar 2024 11:10:24 +0100
Subject: [PATCH 058/127] Change a few imports

---
 .../options/ExampleFunctionalProducerRuntimeCollections.py      | 2 +-
 .../options/ExampleFunctionalTransformerRuntimeCollections.py   | 2 +-
 .../ExampleFunctionalTransformerRuntimeCollectionsMultiple.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
index 6e55dfdf..90831219 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -25,7 +25,7 @@
     ExampleFunctionalProducerRuntimeCollections,
     ExampleFunctionalConsumer,
 )
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
 producer = ExampleFunctionalProducerRuntimeCollections(
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
index b05afeda..e2e5a9d4 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
@@ -26,7 +26,7 @@
     ExampleFunctionalConsumer,
     ExampleFunctionalProducer,
 )
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducer(
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
index c84f9883..1814fc1d 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
@@ -26,7 +26,7 @@
     ExampleFunctionalConsumerMultiple,
     ExampleFunctionalProducerMultiple,
 )
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
 producer0 = ExampleFunctionalProducerMultiple(

From 8dbcaa23deaeeb824b5b9c3906249bf411529d67 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 5 Mar 2024 14:25:58 +0100
Subject: [PATCH 059/127] Add a test for writing 0 collections

---
 test/k4FWCoreTest/CMakeLists.txt              |  1 +
 test/k4FWCoreTest/options/CheckOutputFiles.py |  7 +-
 ...xampleFunctionalTransformerRuntimeEmpty.py | 64 +++++++++++++++++++
 ...ampleFunctionalTransformerRuntimeEmpty.cpp | 58 +++++++++++++++++
 4 files changed, 129 insertions(+), 1 deletion(-)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
 create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index bf456202..cb1fada5 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -125,6 +125,7 @@ add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctional
 add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
 add_test_with_env(FunctionalProducerRuntimeCollections options/ExampleFunctionalProducerRuntimeCollections.py)
 add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctionalTransformerRuntimeCollections.py)
+add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTransformerRuntimeEmpty.py)
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index 0219aaf0..cc4ffe44 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -27,7 +27,11 @@
 
 def check_collections(filename, names):
     podio_reader = podio.root_io.Reader(filename)
-    for frame in podio_reader.get("events"):
+    frames = podio_reader.get("events")
+    if not len(frames) and len(names):
+        print(f"File {filename} is empty but {names} are expected")
+        raise RuntimeError("File is empty but should not be")
+    for frame in frames:
         available = set(frame.getAvailableCollections())
         if available != set(names):
             print(
@@ -58,3 +62,4 @@ def check_collections(filename, names):
     ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"],
 )
 check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
+check_collections("functional_transformer_runtime_empty.root", [])
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
new file mode 100644
index 00000000..b57fa2a4
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
@@ -0,0 +1,64 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import (
+    ExampleFunctionalTransformerRuntimeEmpty,
+    ExampleFunctionalProducer,
+)
+from k4FWCore import ApplicationMgr, IOSvc
+from Configurables import EventDataSvc
+
+iosvc = IOSvc("IOSvc")
+iosvc.output = "functional_transformer_runtime_empty.root"
+
+producer0 = ExampleFunctionalProducer(
+    "Producer0",
+    OutputCollection="MCParticles0",
+)
+producer1 = ExampleFunctionalProducer(
+    "Producer1",
+    OutputCollection="MCParticles1",
+)
+producer2 = ExampleFunctionalProducer(
+    "Producer2",
+    OutputCollection="MCParticles2",
+)
+
+transformer = ExampleFunctionalTransformerRuntimeEmpty(
+    "Transformer",
+    InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
+    OutputCollections=[""],
+)
+
+ApplicationMgr(
+    TopAlg=[
+        producer0,
+        producer1,
+        producer2,
+        transformer,
+    ],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
new file mode 100644
index 00000000..2665ae53
--- /dev/null
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2014-2024 Key4hep-Project.
+ *
+ * This file is part of Key4hep.
+ * See https://key4hep.github.io/key4hep-doc/ for further info.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Gaudi/Property.h"
+
+#include "edm4hep/MCParticleCollection.h"
+
+#include "k4FWCore/Transformer.h"
+
+#include <string>
+
+/* ExampleFunctionalTransformerRuntimeEmpty
+ *
+ * This is an example of a functional transformer that takes an arbitrary number
+ * of input collections and produces an arbitrary number of output collections
+ * but it's returning an empty map (no output collections)
+ */
+
+using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
+
+struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<mapType(const mapType& input)> {
+  // The pair in KeyValue can be changed from python and it corresponds
+  // to the name of the output collection
+  ExampleFunctionalTransformerRuntimeEmpty(const std::string& name, ISvcLocator* svcLoc)
+      : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})},
+                    {KeyValues("OutputCollections", {"MCParticles"})}) {}
+
+  // This is the function that will be called to produce the data
+  mapType operator()(const mapType& input) const override {
+    // We just return an empty map to make sure it works fine
+    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> outputCollections;
+    return outputCollections;
+  }
+
+private:
+  // We can define any property we want that can be set from python
+  // and use it inside operator()
+  Gaudi::Property<int> m_numberOfCollections{this, "NumberOfCollections", 3,
+                                             "Example int that can be used in the algorithm"};
+};
+
+DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeEmpty)

From b01fa5d92d141da450f9c978f44c9135a39e68b3 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 5 Mar 2024 14:26:40 +0100
Subject: [PATCH 060/127] Fix warning

---
 .../src/components/ExampleFunctionalProducerMultiple.cpp       | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index fbae03c4..3c8520f9 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -67,8 +67,7 @@ struct ExampleFunctionalProducerMultiple final : k4FWCore::Producer<retType()> {
 
     auto              particles = edm4hep::MCParticleCollection();
     edm4hep::Vector3d v{0, 0, 0};
-    edm4hep::Vector3f vv{0, 0, 0};
-    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, vv);
+    particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, v);
     particles.create(2, 3, 4, 5.f, 6.f, 7.f);
 
     auto simTrackerHits = edm4hep::SimTrackerHitCollection();

From 6ac240dcb3bd6c506b4b26b918475c15ebd8a32d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 6 Mar 2024 13:33:07 +0100
Subject: [PATCH 061/127] Fix test

---
 test/k4FWCoreTest/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index cb1fada5..ad799ad4 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -108,7 +108,7 @@ add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIE
 add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit")
 
 
-add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py)
+add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop")
 add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py)
 add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py)
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)

From 446ccb65f38e82b099ee89ae9a4a368ed5e668d5 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 6 Mar 2024 13:37:16 +0100
Subject: [PATCH 062/127] Remove old file

---
 k4FWCore/include/k4FWCore/OldConsumer.h | 105 ------------------------
 1 file changed, 105 deletions(-)
 delete mode 100644 k4FWCore/include/k4FWCore/OldConsumer.h

diff --git a/k4FWCore/include/k4FWCore/OldConsumer.h b/k4FWCore/include/k4FWCore/OldConsumer.h
deleted file mode 100644
index 860fe89b..00000000
--- a/k4FWCore/include/k4FWCore/OldConsumer.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright (c) 2014-2024 Key4hep-Project.
- *
- * This file is part of Key4hep.
- * See https://key4hep.github.io/key4hep-doc/ for further info.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef FWCORE_CONSUMER_H
-#define FWCORE_CONSUMER_H
-
-#include <GaudiKernel/FunctionalFilterDecision.h>
-#include "Gaudi/Functional/details.h"
-#include "Gaudi/Functional/utilities.h"
-
-#include "podio/CollectionBase.h"
-
-#include "k4FWCore/FunctionalUtils.h"
-
-#include "GaudiKernel/CommonMessaging.h"
-
-#include <type_traits>
-#include <utility>
-
-namespace k4FWCore {
-
-  namespace details {
-
-    template <typename Signature, typename Traits_> struct Consumer;
-
-    template <typename... In, typename Traits_>
-    struct Consumer<void(const In&...), Traits_>
-        : Gaudi::Functional::details::DataHandleMixin<
-              std::tuple<>,
-              Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>, Traits_> {
-      using Gaudi::Functional::details::DataHandleMixin<
-          std::tuple<>, Gaudi::Functional::details::filter_evtcontext<decltype(transformType(std::declval<In>()))...>,
-          Traits_>::DataHandleMixin;
-
-      // When reading multiple collections we assume that the variable used is a
-      // std::map<std::string, std::shared_ptr<podio::CollectionBase>>
-      // and read the collections in a space separated string
-      template <size_t Index, typename... Handles>
-      void transformAndApplyAlgoAtIndex(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          using HandleType = std::decay_t<decltype(std::get<Index>(handles))>;
-
-          if constexpr (std::is_same_v<
-                            HandleType,
-                            DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>>) {
-            // Transformation logic for the specific type
-            auto               map = std::map<std::string, std::shared_ptr<podio::CollectionBase>>();
-            std::istringstream ss(std::get<Index>(handles).objKey());
-            std::string        token;
-            while (ss >> token) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(token, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + token, "Consumer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[token]            = collection->getData();
-            }
-            std::get<Index>(handles).put(std::move(map));
-          }
-
-          // Recursive call for the next index
-          transformAndApplyAlgoAtIndex<Index + 1>(handles);
-        }
-      }
-
-      // derived classes are NOT allowed to implement execute ...
-      StatusCode execute(const EventContext& ctx) const override final {
-        try {
-          transformAndApplyAlgoAtIndex<0>(this->m_inputs);
-          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-          return Gaudi::Functional::FilterDecision::PASSED;
-        } catch (GaudiException& e) {
-          (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
-          return e.code();
-        }
-      }
-
-      // ... instead, they must implement the following operator
-      virtual void operator()(const In&...) const = 0;
-    };
-
-  }  // namespace details
-
-  template <typename Signature, typename Traits_ = Gaudi::Functional::Traits::useDefaults>
-  using Consumer = details::Consumer<Signature, Traits_>;
-
-}  // namespace k4FWCore
-
-#endif  // FWCORE_CONSUMER_H

From f9b0bad38c909f964b09e83f642005420d3428c1 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:46:08 +0100
Subject: [PATCH 063/127] Update the transformers not to use std::shared_ptr in
 the output

---
 k4FWCore/include/k4FWCore/Transformer.h       | 17 ++---
 ...leFunctionalProducerRuntimeCollections.cpp | 10 +--
 ...unctionalTransformerRuntimeCollections.cpp | 14 ++--
 ...lTransformerRuntimeCollectionsMultiple.cpp | 68 ++++++++-----------
 ...ampleFunctionalTransformerRuntimeEmpty.cpp |  6 +-
 5 files changed, 52 insertions(+), 63 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 2d9b6b95..cca6e7e1 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -190,11 +190,11 @@ namespace k4FWCore {
         try {
           readMapInputs<0>(this->m_inputs);
           if constexpr (is_map_like<Out>::value) {
-            for (auto [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
-              auto        shared = std::dynamic_pointer_cast<podio::CollectionBase>(val);
-              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));
-              DataObject* p      = w;
-              auto        sc     = this->evtSvc()->registerObject(key, p);
+            for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
+              auto        shared = std::make_shared<decltype(val)>(std::move(val));
+              auto        w  = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(shared);
+              DataObject* p  = w;
+              auto        sc = this->evtSvc()->registerObject(key, p);
             }
           } else {
             Gaudi::Functional::details::put(
@@ -328,12 +328,9 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles> void putMapOutputs(std::tuple<Handles...>&& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
-            using EDM4hepType =
-                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
-            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
 
-            for (auto [key, val] : std::get<Index>(handles)) {
-              auto        shared = std::dynamic_pointer_cast<podio::CollectionBase>(val);
+            for (auto& [key, val] : std::get<Index>(handles)) {
+              auto        shared = std::make_shared<decltype(val)>(std::move(val));
               auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));
               DataObject* p      = w;
               auto        sc     = this->evtSvc()->registerObject(key, p);
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
index 2f512f73..7579844e 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp
@@ -26,21 +26,21 @@
 #include <string>
 
 struct ExampleFunctionalProducerRuntimeCollections final
-    : k4FWCore::Producer<std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>()> {
+    : k4FWCore::Producer<std::map<std::string, edm4hep::MCParticleCollection>()> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalProducerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
       : Producer(name, svcLoc, {}, {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
-  std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> operator()() const override {
-    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> m_outputCollections;
+  std::map<std::string, edm4hep::MCParticleCollection> operator()() const override {
+    std::map<std::string, edm4hep::MCParticleCollection> m_outputCollections;
     for (int i = 0; i < m_numberOfCollections; ++i) {
       std::string name = "MCParticles" + std::to_string(i);
-      auto        coll = std::make_shared<edm4hep::MCParticleCollection>();
+      auto        coll = edm4hep::MCParticleCollection();
       coll->create(1, 2, 3, 4.f, 5.f, 6.f);
       coll->create(2, 3, 4, 5.f, 6.f, 7.f);
-      m_outputCollections[name] = coll;
+      m_outputCollections[name] = std::move(coll);
     }
     return m_outputCollections;
   }
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
index 78c0f27b..9b07071c 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -32,9 +32,9 @@
  * (in this example the same number of input collections but it can be different)
  */
 
-using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
+using mapType = std::map<std::string, edm4hep::MCParticleCollection>;
 
-struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const mapType& input)> {
+struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
@@ -42,15 +42,15 @@ struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transfor
                     {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
-  mapType operator()(const mapType& input) const override {
-    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> outputCollections;
+  mapType operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input) const override {
+    std::map<std::string, edm4hep::MCParticleCollection> outputCollections;
     for (int i = 0; i < input.size(); ++i) {
       std::string name     = "NewMCParticles" + std::to_string(i);
-      auto        old_coll = input.at("MCParticles" + std::to_string(i));
-      auto        coll     = std::make_shared<edm4hep::MCParticleCollection>();
+      auto&       old_coll = input.at("MCParticles" + std::to_string(i));
+      auto        coll     = edm4hep::MCParticleCollection();
       coll->push_back(old_coll->at(0).clone());
       coll->push_back(old_coll->at(1).clone());
-      outputCollections[name] = coll;
+      outputCollections[name] = std::move(coll);
     }
     return outputCollections;
   }
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
index d460b9d4..a3d14e95 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
@@ -21,20 +21,12 @@
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
-#include "edm4hep/TrackCollection.h"
-#if __has_include("edm4hep/TrackerHit3DCollection.h")
 #include "edm4hep/TrackerHit3DCollection.h"
-#else
-#include "edm4hep/TrackerHitCollection.h"
-namespace edm4hep {
-  using TrackerHit3DCollection = edm4hep::TrackerHitCollection;
-}  // namespace edm4hep
-#endif
+#include "edm4hep/TrackCollection.h"
 #include "podio/UserDataCollection.h"
 
 #include "k4FWCore/Transformer.h"
 
-#include <sstream>
 #include <stdexcept>
 #include <string>
 
@@ -45,12 +37,12 @@ using SimTrackerHitColl = std::map<std::string, std::shared_ptr<edm4hep::SimTrac
 using TrackerHitColl    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>;
 using TrackColl         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>;
 
-using retType = std::tuple<std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>,
-                           std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>>;
+using retType = std::tuple<std::map<std::string, podio::UserDataCollection<float>>,
+                           std::map<std::string, edm4hep::MCParticleCollection>,
+                           std::map<std::string, edm4hep::MCParticleCollection>,
+                           std::map<std::string, edm4hep::SimTrackerHitCollection>,
+                           std::map<std::string, edm4hep::TrackerHit3DCollection>,
+                           std::map<std::string, edm4hep::TrackCollection>>;
 
 struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     : k4FWCore::MultiTransformer<retType(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
@@ -81,12 +73,12 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
   retType operator()(const FloatColl& floatMap, const ParticleColl& particlesMap,
                      const SimTrackerHitColl& simTrackerHitMap, const TrackerHitColl& trackerHitMap,
                      const TrackColl& trackMap) const override {
-    auto floatMapOut         = std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>();
-    auto particleMapOut      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
-    auto particle2MapOut     = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>();
-    auto simTrackerHitMapOut = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>();
-    auto trackerHitMapOut    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>();
-    auto trackMapOut         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>();
+    auto floatMapOut         = std::map<std::string, podio::UserDataCollection<float>>();
+    auto particleMapOut      = std::map<std::string, edm4hep::MCParticleCollection>();
+    auto particle2MapOut     = std::map<std::string, edm4hep::MCParticleCollection>();
+    auto simTrackerHitMapOut = std::map<std::string, edm4hep::SimTrackerHitCollection>();
+    auto trackerHitMapOut    = std::map<std::string, edm4hep::TrackerHit3DCollection>();
+    auto trackMapOut         = std::map<std::string, edm4hep::TrackCollection>();
 
     if (floatMap.size() != 3) {
       throw std::runtime_error("Wrong size of the floatVector collection map, expected 3, got " +
@@ -103,11 +95,11 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
               << ", " << floatVector->vec()[1] << ", " << floatVector->vec()[2] << "";
         throw std::runtime_error(error.str());
       }
-      auto ptr = std::make_shared<podio::UserDataCollection<float>>();
-      ptr->push_back(floatVector->vec()[0]);
-      ptr->push_back(floatVector->vec()[1]);
-      ptr->push_back(floatVector->vec()[2]);
-      floatMapOut["New" + key] = floatVector;
+      auto coll = podio::UserDataCollection<float>();
+      coll.push_back(floatVector->vec()[0]);
+      coll.push_back(floatVector->vec()[1]);
+      coll.push_back(floatVector->vec()[2]);
+      floatMapOut["New" + key] = std::move(coll);
     }
 
     if (particlesMap.size() != 3) {
@@ -116,7 +108,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     }
 
     for (auto& [key, particles] : particlesMap) {
-      auto ptr = std::make_shared<edm4hep::MCParticleCollection>();
+      auto coll = edm4hep::MCParticleCollection();
       int  i   = 0;
       for (const auto& particle : *particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
@@ -129,10 +121,10 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
                 << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", "
                 << particle.getMass() << "";
           throw std::runtime_error(error.str());
-          ptr->push_back(particle.clone());
+          coll.push_back(particle.clone());
         }
         i++;
-        particleMapOut["New" + key] = ptr;
+        particleMapOut["New" + key] = std::move(coll);
       }
     }
 
@@ -142,7 +134,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     }
 
     for (auto& [key, simTrackerHits] : simTrackerHitMap) {
-      auto ptr = std::make_shared<edm4hep::SimTrackerHitCollection>();
+      auto coll = edm4hep::SimTrackerHitCollection();
       if ((simTrackerHits->at(0).getPosition()[0] != 3) || (simTrackerHits->at(0).getPosition()[1] != 4) ||
           (simTrackerHits->at(0).getPosition()[2] != 5)) {
         std::stringstream error;
@@ -151,8 +143,8 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
               << simTrackerHits->at(0).getPosition()[2] << "";
         throw std::runtime_error(error.str());
       }
-      ptr->push_back(simTrackerHits->at(0).clone());
-      simTrackerHitMapOut["New" + key] = ptr;
+      coll.push_back(simTrackerHits->at(0).clone());
+      simTrackerHitMapOut["New" + key] = std::move(coll);
     }
 
     if (trackerHitMap.size() != 3) {
@@ -161,7 +153,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     }
 
     for (auto& [key, trackerHits] : trackerHitMap) {
-      auto ptr = std::make_shared<edm4hep::TrackerHit3DCollection>();
+      auto coll = edm4hep::TrackerHit3DCollection();
       if ((trackerHits->at(0).getPosition()[0] != 3) || (trackerHits->at(0).getPosition()[1] != 4) ||
           (trackerHits->at(0).getPosition()[2] != 5)) {
         std::stringstream error;
@@ -169,8 +161,8 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
               << ", " << trackerHits->at(0).getPosition()[1] << ", " << trackerHits->at(0).getPosition()[2] << "";
         throw std::runtime_error(error.str());
       }
-      ptr->push_back(trackerHits->at(0).clone());
-      trackerHitMapOut["New" + key] = ptr;
+      coll.push_back(trackerHits->at(0).clone());
+      trackerHitMapOut["New" + key] = std::move(coll);
     }
 
     if (trackMap.size() != 3) {
@@ -179,7 +171,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     }
 
     for (auto& [key, tracks] : trackMap) {
-      auto ptr = std::make_shared<edm4hep::TrackCollection>();
+      auto coll = edm4hep::TrackCollection();
       if ((tracks->at(0).getType() != 1) || (std::abs(tracks->at(0).getChi2() - 2.1) > 1e-6) ||
           (tracks->at(0).getNdf() != 3) || (std::abs(tracks->at(0).getDEdx() - 4.1) > 1e-6) ||
           (std::abs(tracks->at(0).getDEdxError() - 5.1) > 1e-6) ||
@@ -190,8 +182,8 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
               << ", " << tracks->at(0).getDEdxError() << ", " << tracks->at(0).getRadiusOfInnermostHit() << "";
         throw std::runtime_error(error.str());
       }
-      ptr->push_back(tracks->at(0).clone());
-      trackMapOut["New" + key] = ptr;
+      coll->push_back(tracks->at(0).clone());
+      trackMapOut["New" + key] = std::move(coll);
     }
 
     return std::make_tuple(std::move(floatMapOut), std::move(particleMapOut), std::move(particle2MapOut),
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
index 2665ae53..8bace2e5 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
@@ -34,7 +34,7 @@
 
 using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
 
-struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<mapType(const mapType& input)> {
+struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<std::map<std::string, edm4hep::MCParticleCollection>(const mapType& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeEmpty(const std::string& name, ISvcLocator* svcLoc)
@@ -42,9 +42,9 @@ struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<ma
                     {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
-  mapType operator()(const mapType& input) const override {
+  std::map<std::string, edm4hep::MCParticleCollection> operator()(const mapType& input) const override {
     // We just return an empty map to make sure it works fine
-    std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>> outputCollections;
+    std::map<std::string, edm4hep::MCParticleCollection> outputCollections;
     return outputCollections;
   }
 

From 2062d98f8e14c815fe856e95974cac60d9829451 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:54:32 +0100
Subject: [PATCH 064/127] Update the consumer not to use shared_ptr and use
 references instead

---
 k4FWCore/include/k4FWCore/Consumer.h              | 10 ++++++----
 .../ExampleFunctionalConsumerMultiple.cpp         |  7 -------
 ...xampleFunctionalConsumerRuntimeCollections.cpp |  7 +++----
 ...nctionalConsumerRuntimeCollectionsMultiple.cpp | 15 ++++++---------
 4 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index cca927e6..42f18911 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -92,9 +92,10 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            using EDM4hepType =
-                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
-            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+            // we have to remove the reference to get the actual type
+            using EDM4hepType = std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+            auto map = std::map<std::string, EDM4hepType&>();
 
             // To be locked
             if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
@@ -113,7 +114,8 @@ namespace k4FWCore {
                 throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              auto ptr = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              map.emplace(value, *ptr);
             }
             std::get<Index>(handles).put(std::move(map));
           }
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
index bb546f06..d21efb94 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp
@@ -22,14 +22,7 @@
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
 #include "edm4hep/TrackCollection.h"
-#if __has_include("edm4hep/TrackerHit3DCollection.h")
 #include "edm4hep/TrackerHit3DCollection.h"
-#else
-#include "edm4hep/TrackerHitCollection.h"
-namespace edm4hep {
-  using TrackerHit3DCollection = edm4hep::TrackerHitCollection;
-}  // namespace edm4hep
-#endif
 #include "podio/UserDataCollection.h"
 
 #include "k4FWCore/Consumer.h"
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index fc0683d5..64272be5 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -23,24 +23,23 @@
 
 #include "k4FWCore/Consumer.h"
 
-#include <memory>
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollections final
-    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input)> {
+    : k4FWCore::Consumer<void(const std::map<std::string, edm4hep::MCParticleCollection&>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
       : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input) const override {
+  void operator()(const std::map<std::string, edm4hep::MCParticleCollection&>& input) const override {
     if (input.size() != 3) {
       fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
     }
     for (auto& [key, val] : input) {
       int i = 0;
-      for (const auto& particle : *val) {
+      for (const auto& particle : val) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
             (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index 8ec11409..a0812301 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -25,12 +25,11 @@
 
 #include "k4FWCore/Consumer.h"
 
-#include <memory>
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
-    : k4FWCore::Consumer<void(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
-                              const std::map<std::string, std::shared_ptr<podio::CollectionBase>>&         trackMap,
+    : k4FWCore::Consumer<void(const std::map<std::string, edm4hep::MCParticleCollection&>& particleMap,
+                              const std::map<std::string, edm4hep::TrackCollection&>&         trackMap,
                               const edm4hep::SimTrackerHitCollection& simTrackerHits)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
@@ -41,16 +40,15 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
   // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& particleMap,
-                  const std::map<std::string, std::shared_ptr<podio::CollectionBase>>&         trackMap,
+  void operator()(const std::map<std::string, edm4hep::MCParticleCollection&>& particleMap,
+                  const std::map<std::string, edm4hep::TrackCollection&>&         trackMap,
                   const edm4hep::SimTrackerHitCollection& simTrackerHits) const override {
     info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size()
            << " track collections" << endmsg;
     if (particleMap.size() != 5) {
       fatal() << "Wrong size of the particleMap map, expected 5, got " << particleMap.size() << endmsg;
     }
-    for (auto& [key, val] : particleMap) {
-      const auto& particles = *std::static_pointer_cast<edm4hep::MCParticleCollection>(val);
+    for (auto& [key, particles] : particleMap) {
       int         i         = 0;
       for (const auto& particle : particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
@@ -70,8 +68,7 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
     if (trackMap.size() != 3) {
       fatal() << "Wrong size of the tracks map, expected 3, got " << trackMap.size() << endmsg;
     }
-    for (auto& [key, val] : trackMap) {
-      const auto& tracks = *std::static_pointer_cast<edm4hep::TrackCollection>(val);
+    for (auto& [key, tracks] : trackMap) {
       if ((tracks[0].getType() != 1) || (std::abs(tracks[0].getChi2() - 2.1) > 1e-6) || (tracks[0].getNdf() != 3) ||
           (std::abs(tracks[0].getDEdx() - 4.1) > 1e-6) || (std::abs(tracks[0].getDEdxError() - 5.1) > 1e-6) ||
           (std::abs(tracks[0].getRadiusOfInnermostHit() - 6.1) > 1e-6)) {

From 6154232435496ea14c90d84a4206ab63441bad3c Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:55:49 +0100
Subject: [PATCH 065/127] Format

---
 k4FWCore/include/k4FWCore/Consumer.h    | 5 +++--
 k4FWCore/include/k4FWCore/Transformer.h | 7 +++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 42f18911..e714352d 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -94,7 +94,8 @@ namespace k4FWCore {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
             // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
             // we have to remove the reference to get the actual type
-            using EDM4hepType = std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+            using EDM4hepType =
+                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
             auto map = std::map<std::string, EDM4hepType&>();
 
             // To be locked
@@ -114,7 +115,7 @@ namespace k4FWCore {
                 throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              auto ptr = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
               map.emplace(value, *ptr);
             }
             std::get<Index>(handles).put(std::move(map));
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index cca6e7e1..433d6406 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -192,9 +192,9 @@ namespace k4FWCore {
           if constexpr (is_map_like<Out>::value) {
             for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
               auto        shared = std::make_shared<decltype(val)>(std::move(val));
-              auto        w  = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(shared);
-              DataObject* p  = w;
-              auto        sc = this->evtSvc()->registerObject(key, p);
+              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(shared);
+              DataObject* p      = w;
+              auto        sc     = this->evtSvc()->registerObject(key, p);
             }
           } else {
             Gaudi::Functional::details::put(
@@ -328,7 +328,6 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles> void putMapOutputs(std::tuple<Handles...>&& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
-
             for (auto& [key, val] : std::get<Index>(handles)) {
               auto        shared = std::make_shared<decltype(val)>(std::move(val));
               auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));

From b7ab80cbf97bf58ef5a6c811874f2e5e05c7de93 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:59:54 +0100
Subject: [PATCH 066/127] References should be const

---
 .../ExampleFunctionalConsumerRuntimeCollections.cpp       | 4 ++--
 ...xampleFunctionalConsumerRuntimeCollectionsMultiple.cpp | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index 64272be5..e98b338a 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -26,14 +26,14 @@
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollections final
-    : k4FWCore::Consumer<void(const std::map<std::string, edm4hep::MCParticleCollection&>& input)> {
+    : k4FWCore::Consumer<void(const std::map<std::string, const edm4hep::MCParticleCollection&>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
       : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, edm4hep::MCParticleCollection&>& input) const override {
+  void operator()(const std::map<std::string, const edm4hep::MCParticleCollection&>& input) const override {
     if (input.size() != 3) {
       fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
     }
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index a0812301..04428d32 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -28,8 +28,8 @@
 #include <string>
 
 struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
-    : k4FWCore::Consumer<void(const std::map<std::string, edm4hep::MCParticleCollection&>& particleMap,
-                              const std::map<std::string, edm4hep::TrackCollection&>&         trackMap,
+    : k4FWCore::Consumer<void(const std::map<std::string, const edm4hep::MCParticleCollection&>& particleMap,
+                              const std::map<std::string, const edm4hep::TrackCollection&>&         trackMap,
                               const edm4hep::SimTrackerHitCollection& simTrackerHits)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
@@ -40,8 +40,8 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
   // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {}
 
   // This is the function that will be called to produce the data
-  void operator()(const std::map<std::string, edm4hep::MCParticleCollection&>& particleMap,
-                  const std::map<std::string, edm4hep::TrackCollection&>&         trackMap,
+  void operator()(const std::map<std::string, const edm4hep::MCParticleCollection&>& particleMap,
+                  const std::map<std::string, const edm4hep::TrackCollection&>&         trackMap,
                   const edm4hep::SimTrackerHitCollection& simTrackerHits) const override {
     info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size()
            << " track collections" << endmsg;

From 624158e833ad15fd11efb8e4e199601a282e093d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:01:01 +0100
Subject: [PATCH 067/127] Enforce references being const

---
 k4FWCore/include/k4FWCore/Consumer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index e714352d..3eb2efeb 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -96,7 +96,7 @@ namespace k4FWCore {
             // we have to remove the reference to get the actual type
             using EDM4hepType =
                 std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-            auto map = std::map<std::string, EDM4hepType&>();
+            auto map = std::map<std::string, const EDM4hepType&>();
 
             // To be locked
             if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {

From a9f5cecd9fe6c33093ff80e7c2ce8bb536d33a04 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:08:22 +0100
Subject: [PATCH 068/127] Modify the transformer not to use std::shared_ptr in
 the inputs

---
 k4FWCore/include/k4FWCore/Consumer.h          |  6 +-
 k4FWCore/include/k4FWCore/Transformer.h       | 22 ++++---
 ...unctionalTransformerRuntimeCollections.cpp |  8 +--
 ...lTransformerRuntimeCollectionsMultiple.cpp | 64 +++++++++----------
 ...ampleFunctionalTransformerRuntimeEmpty.cpp |  2 +-
 5 files changed, 54 insertions(+), 48 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 3eb2efeb..9e3c47ca 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -96,7 +96,7 @@ namespace k4FWCore {
             // we have to remove the reference to get the actual type
             using EDM4hepType =
                 std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-            auto map = std::map<std::string, const EDM4hepType&>();
+            auto inputMap = std::map<std::string, const EDM4hepType&>();
 
             // To be locked
             if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
@@ -116,9 +116,9 @@ namespace k4FWCore {
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
               auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-              map.emplace(value, *ptr);
+              inputMap.emplace(value, *ptr);
             }
-            std::get<Index>(handles).put(std::move(map));
+            std::get<Index>(handles).put(std::move(inputMap));
           }
 
           // Recursive call for the next index
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 433d6406..10d6fefb 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -120,9 +120,11 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+            // we have to remove the reference to get the actual type
             using EDM4hepType =
-                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
-            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+            auto inputMap = std::map<std::string, const EDM4hepType&>();
 
             // To be locked
             if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
@@ -141,9 +143,10 @@ namespace k4FWCore {
                 throw GaudiException("Failed to retrieve object " + value, "Transformer", StatusCode::FAILURE);
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              inputMap.emplace(value, *ptr);
             }
-            std::get<Index>(handles).put(std::move(map));
+            std::get<Index>(handles).put(std::move(inputMap));
           }
 
           // Recursive call for the next index
@@ -294,9 +297,11 @@ namespace k4FWCore {
       template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+            // we have to remove the reference to get the actual type
             using EDM4hepType =
-                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
-            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
+                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+            auto inputMap = std::map<std::string, const EDM4hepType&>();
 
             // To be locked
             if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
@@ -315,9 +320,10 @@ namespace k4FWCore {
                 throw GaudiException("Failed to retrieve object " + value, "MultiTransformer", StatusCode::FAILURE);
               }
               const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+              inputMap.emplace(value, *ptr);
             }
-            std::get<Index>(handles).put(std::move(map));
+            std::get<Index>(handles).put(std::move(inputMap));
           }
 
           // Recursive call for the next index
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
index 9b07071c..7ae662a6 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -34,7 +34,7 @@
 
 using mapType = std::map<std::string, edm4hep::MCParticleCollection>;
 
-struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input)> {
+struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const std::map<std::string, const edm4hep::MCParticleCollection&>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
@@ -42,14 +42,14 @@ struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transfor
                     {KeyValues("OutputCollections", {"MCParticles"})}) {}
 
   // This is the function that will be called to produce the data
-  mapType operator()(const std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>& input) const override {
+  mapType operator()(const std::map<std::string, const edm4hep::MCParticleCollection&>& input) const override {
     std::map<std::string, edm4hep::MCParticleCollection> outputCollections;
     for (int i = 0; i < input.size(); ++i) {
       std::string name     = "NewMCParticles" + std::to_string(i);
       auto&       old_coll = input.at("MCParticles" + std::to_string(i));
       auto        coll     = edm4hep::MCParticleCollection();
-      coll->push_back(old_coll->at(0).clone());
-      coll->push_back(old_coll->at(1).clone());
+      coll->push_back(old_coll.at(0).clone());
+      coll->push_back(old_coll.at(1).clone());
       outputCollections[name] = std::move(coll);
     }
     return outputCollections;
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
index a3d14e95..53255d99 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
@@ -31,11 +31,11 @@
 #include <string>
 
 // Which type of collection we are reading
-using FloatColl         = std::map<std::string, std::shared_ptr<podio::UserDataCollection<float>>>;
-using ParticleColl      = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
-using SimTrackerHitColl = std::map<std::string, std::shared_ptr<edm4hep::SimTrackerHitCollection>>;
-using TrackerHitColl    = std::map<std::string, std::shared_ptr<edm4hep::TrackerHit3DCollection>>;
-using TrackColl         = std::map<std::string, std::shared_ptr<edm4hep::TrackCollection>>;
+using FloatColl         = std::map<std::string, const podio::UserDataCollection<float>&>;
+using ParticleColl      = std::map<std::string, const edm4hep::MCParticleCollection&>;
+using SimTrackerHitColl = std::map<std::string, const edm4hep::SimTrackerHitCollection&>;
+using TrackerHitColl    = std::map<std::string, const edm4hep::TrackerHit3DCollection&>;
+using TrackColl         = std::map<std::string, const edm4hep::TrackCollection&>;
 
 using retType = std::tuple<std::map<std::string, podio::UserDataCollection<float>>,
                            std::map<std::string, edm4hep::MCParticleCollection>,
@@ -85,20 +85,20 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
                                std::to_string(floatMap.size()) + "");
     }
     for (const auto& [key, floatVector] : floatMap) {
-      if (floatVector->size() != 3) {
+      if (floatVector.size() != 3) {
         throw std::runtime_error("Wrong size of floatVector collection, expected 3, got " +
-                                 std::to_string(floatVector->size()) + "");
+                                 std::to_string(floatVector.size()) + "");
       }
-      if ((floatVector->vec()[0] != 125) || (floatVector->vec()[1] != 25) || (floatVector->vec()[2] != 0)) {
+      if ((floatVector.vec()[0] != 125) || (floatVector.vec()[1] != 25) || (floatVector.vec()[2] != 0)) {
         std::stringstream error;
-        error << "Wrong data in floatVector collection, expected 125, 25, " << 0 << " got " << floatVector->vec()[0]
-              << ", " << floatVector->vec()[1] << ", " << floatVector->vec()[2] << "";
+        error << "Wrong data in floatVector collection, expected 125, 25, " << 0 << " got " << floatVector.vec()[0]
+              << ", " << floatVector.vec()[1] << ", " << floatVector.vec()[2] << "";
         throw std::runtime_error(error.str());
       }
       auto coll = podio::UserDataCollection<float>();
-      coll.push_back(floatVector->vec()[0]);
-      coll.push_back(floatVector->vec()[1]);
-      coll.push_back(floatVector->vec()[2]);
+      coll.push_back(floatVector.vec()[0]);
+      coll.push_back(floatVector.vec()[1]);
+      coll.push_back(floatVector.vec()[2]);
       floatMapOut["New" + key] = std::move(coll);
     }
 
@@ -110,7 +110,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     for (auto& [key, particles] : particlesMap) {
       auto coll = edm4hep::MCParticleCollection();
       int  i   = 0;
-      for (const auto& particle : *particles) {
+      for (const auto& particle : particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
             (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) {
@@ -135,15 +135,15 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
 
     for (auto& [key, simTrackerHits] : simTrackerHitMap) {
       auto coll = edm4hep::SimTrackerHitCollection();
-      if ((simTrackerHits->at(0).getPosition()[0] != 3) || (simTrackerHits->at(0).getPosition()[1] != 4) ||
-          (simTrackerHits->at(0).getPosition()[2] != 5)) {
+      if ((simTrackerHits.at(0).getPosition()[0] != 3) || (simTrackerHits.at(0).getPosition()[1] != 4) ||
+          (simTrackerHits.at(0).getPosition()[2] != 5)) {
         std::stringstream error;
         error << "Wrong data in simTrackerHits collection, expected 3, 4, 5 got "
-              << simTrackerHits->at(0).getPosition()[0] << ", " << simTrackerHits->at(0).getPosition()[1] << ", "
-              << simTrackerHits->at(0).getPosition()[2] << "";
+              << simTrackerHits.at(0).getPosition()[0] << ", " << simTrackerHits.at(0).getPosition()[1] << ", "
+              << simTrackerHits.at(0).getPosition()[2] << "";
         throw std::runtime_error(error.str());
       }
-      coll.push_back(simTrackerHits->at(0).clone());
+      coll.push_back(simTrackerHits.at(0).clone());
       simTrackerHitMapOut["New" + key] = std::move(coll);
     }
 
@@ -154,14 +154,14 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
 
     for (auto& [key, trackerHits] : trackerHitMap) {
       auto coll = edm4hep::TrackerHit3DCollection();
-      if ((trackerHits->at(0).getPosition()[0] != 3) || (trackerHits->at(0).getPosition()[1] != 4) ||
-          (trackerHits->at(0).getPosition()[2] != 5)) {
+      if ((trackerHits.at(0).getPosition()[0] != 3) || (trackerHits.at(0).getPosition()[1] != 4) ||
+          (trackerHits.at(0).getPosition()[2] != 5)) {
         std::stringstream error;
-        error << "Wrong data in trackerHits collection, expected 3, 4, 5 got " << trackerHits->at(0).getPosition()[0]
-              << ", " << trackerHits->at(0).getPosition()[1] << ", " << trackerHits->at(0).getPosition()[2] << "";
+        error << "Wrong data in trackerHits collection, expected 3, 4, 5 got " << trackerHits.at(0).getPosition()[0]
+              << ", " << trackerHits.at(0).getPosition()[1] << ", " << trackerHits.at(0).getPosition()[2] << "";
         throw std::runtime_error(error.str());
       }
-      coll.push_back(trackerHits->at(0).clone());
+      coll.push_back(trackerHits.at(0).clone());
       trackerHitMapOut["New" + key] = std::move(coll);
     }
 
@@ -172,17 +172,17 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
 
     for (auto& [key, tracks] : trackMap) {
       auto coll = edm4hep::TrackCollection();
-      if ((tracks->at(0).getType() != 1) || (std::abs(tracks->at(0).getChi2() - 2.1) > 1e-6) ||
-          (tracks->at(0).getNdf() != 3) || (std::abs(tracks->at(0).getDEdx() - 4.1) > 1e-6) ||
-          (std::abs(tracks->at(0).getDEdxError() - 5.1) > 1e-6) ||
-          (std::abs(tracks->at(0).getRadiusOfInnermostHit() - 6.1) > 1e-6)) {
+      if ((tracks.at(0).getType() != 1) || (std::abs(tracks.at(0).getChi2() - 2.1) > 1e-6) ||
+          (tracks.at(0).getNdf() != 3) || (std::abs(tracks.at(0).getDEdx() - 4.1) > 1e-6) ||
+          (std::abs(tracks.at(0).getDEdxError() - 5.1) > 1e-6) ||
+          (std::abs(tracks.at(0).getRadiusOfInnermostHit() - 6.1) > 1e-6)) {
         std::stringstream error;
-        error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks->at(0).getType()
-              << ", " << tracks->at(0).getChi2() << ", " << tracks->at(0).getNdf() << ", " << tracks->at(0).getDEdx()
-              << ", " << tracks->at(0).getDEdxError() << ", " << tracks->at(0).getRadiusOfInnermostHit() << "";
+        error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks.at(0).getType()
+              << ", " << tracks.at(0).getChi2() << ", " << tracks.at(0).getNdf() << ", " << tracks.at(0).getDEdx()
+              << ", " << tracks.at(0).getDEdxError() << ", " << tracks.at(0).getRadiusOfInnermostHit() << "";
         throw std::runtime_error(error.str());
       }
-      coll->push_back(tracks->at(0).clone());
+      coll->push_back(tracks.at(0).clone());
       trackMapOut["New" + key] = std::move(coll);
     }
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
index 8bace2e5..c3d8dc96 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
@@ -32,7 +32,7 @@
  * but it's returning an empty map (no output collections)
  */
 
-using mapType = std::map<std::string, std::shared_ptr<edm4hep::MCParticleCollection>>;
+using mapType = std::map<std::string, const edm4hep::MCParticleCollection&>;
 
 struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<std::map<std::string, edm4hep::MCParticleCollection>(const mapType& input)> {
   // The pair in KeyValue can be changed from python and it corresponds

From f0d5dd6da852f59d68babc26f8f36f30d3dff254 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:36:24 +0100
Subject: [PATCH 069/127] Fix a few includes

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 9652127c..c9c31175 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -18,16 +18,16 @@
  */
 #pragma once
 
-#include "Gaudi/Functional/details.h"
-#include "Gaudi/Functional/utilities.h"
-
 #include "podio/CollectionBase.h"
 
+#include "GaudiKernel/DataObjectHandle.h"
+#include "GaudiKernel/EventContext.h"
+#include "GaudiKernel/ThreadLocalContext.h"
+
+#include <map>
 #include <memory>
-#include <stdexcept>
 #include <tuple>
 #include <type_traits>
-#include <unordered_map>
 
 namespace k4FWCore {
 
@@ -39,8 +39,10 @@ namespace k4FWCore {
     template <typename T, typename P> const auto& maybeTransformToEDM4hep(const P& arg) { return arg; }
 
     template <typename T, typename P>
-    requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
-    const auto& maybeTransformToEDM4hep(const P& arg) { return static_cast<const T&>(*arg); }
+      requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
+    const auto& maybeTransformToEDM4hep(const P& arg) {
+      return static_cast<const T&>(*arg);
+    }
 
     template <typename T> struct is_map_like : std::false_type {};
 

From f23ad30ce1671def3a42ae342a04fb3eac1cf301 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:26:15 +0100
Subject: [PATCH 070/127] Move readMapInputs to FunctionalUtils.h

---
 k4FWCore/include/k4FWCore/Consumer.h        | 39 +---------
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 39 ++++++++++
 k4FWCore/include/k4FWCore/Transformer.h     | 80 +--------------------
 3 files changed, 43 insertions(+), 115 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 9e3c47ca..bf2e0c29 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -89,47 +89,10 @@ namespace k4FWCore {
                Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
           : Consumer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}) {}
 
-      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
-            // we have to remove the reference to get the actual type
-            using EDM4hepType =
-                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-            auto inputMap = std::map<std::string, const EDM4hepType&>();
-
-            // To be locked
-            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-              auto vec = std::vector<std::string>();
-              vec.reserve(m_inputLocations[Index].value().size());
-              for (auto& val : m_inputLocations[Index].value()) {
-                vec.push_back(val.key());
-              }
-              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
-            }
-
-            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(value, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-              inputMap.emplace(value, *ptr);
-            }
-            std::get<Index>(handles).put(std::move(inputMap));
-          }
-
-          // Recursive call for the next index
-          readMapInputs<Index + 1>(handles);
-        }
-      }
-
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0>(this->m_inputs);
+          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
           filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index c9c31175..5f9217da 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -127,5 +127,44 @@ namespace k4FWCore {
       }
     };
 
+    template <size_t Index, typename... In, typename... Handles>
+    void readMapInputs(const std::tuple<Handles...>& handles, const auto& m_inputLocations,
+                       auto& m_inputLocationsMap, auto thisClass) {
+      if constexpr (Index < sizeof...(Handles)) {
+        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+          // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+          // we have to remove the reference to get the actual type
+          using EDM4hepType =
+              std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+          auto inputMap = std::map<std::string, const EDM4hepType&>();
+
+          // To be locked
+          if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
+            auto vec = std::vector<std::string>();
+            vec.reserve(m_inputLocations[Index].value().size());
+            for (auto& val : m_inputLocations[Index].value()) {
+              vec.push_back(val.key());
+            }
+            m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+          }
+
+          for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
+            DataObject* p;
+            auto        sc = thisClass->evtSvc()->retrieveObject(value, p);
+            if (!sc.isSuccess()) {
+              throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
+            }
+            const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
+            auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
+            inputMap.emplace(value, *ptr);
+          }
+          std::get<Index>(handles).put(std::move(inputMap));
+        }
+
+        // Recursive call for the next index
+        readMapInputs<Index + 1, In...>(handles, m_inputLocations, m_inputLocationsMap, thisClass);
+      }
+    }
+
   }  // namespace details
 }  // namespace k4FWCore
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 10d6fefb..0e7ac56b 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -117,43 +117,6 @@ namespace k4FWCore {
           : Transformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                         std::index_sequence_for<Out>{}) {}
 
-      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
-            // we have to remove the reference to get the actual type
-            using EDM4hepType =
-                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-            auto inputMap = std::map<std::string, const EDM4hepType&>();
-
-            // To be locked
-            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-              auto vec = std::vector<std::string>();
-              vec.reserve(m_inputLocations[Index].value().size());
-              for (auto& val : m_inputLocations[Index].value()) {
-                vec.push_back(val.key());
-              }
-              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
-            }
-
-            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(value, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + value, "Transformer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-              inputMap.emplace(value, *ptr);
-            }
-            std::get<Index>(handles).put(std::move(inputMap));
-          }
-
-          // Recursive call for the next index
-          readMapInputs<Index + 1>(handles);
-        }
-      }
-
       template <size_t Index, typename... Handles> void putMapInputs(const std::tuple<Handles...>& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
@@ -184,14 +147,14 @@ namespace k4FWCore {
           }
 
           // Recursive call for the next index
-          readMapInputs<Index + 1>(handles);
+          putMapInputs<Index + 1>(handles);
         }
       }
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0>(this->m_inputs);
+          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
           if constexpr (is_map_like<Out>::value) {
             for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
               auto        shared = std::make_shared<decltype(val)>(std::move(val));
@@ -294,43 +257,6 @@ namespace k4FWCore {
           : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                              std::index_sequence_for<Out...>{}) {}
 
-      template <size_t Index, typename... Handles> void readMapInputs(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
-            // we have to remove the reference to get the actual type
-            using EDM4hepType =
-                std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-            auto inputMap = std::map<std::string, const EDM4hepType&>();
-
-            // To be locked
-            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-              auto vec = std::vector<std::string>();
-              vec.reserve(m_inputLocations[Index].value().size());
-              for (auto& val : m_inputLocations[Index].value()) {
-                vec.push_back(val.key());
-              }
-              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
-            }
-
-            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(value, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + value, "MultiTransformer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-              inputMap.emplace(value, *ptr);
-            }
-            std::get<Index>(handles).put(std::move(inputMap));
-          }
-
-          // Recursive call for the next index
-          readMapInputs<Index + 1>(handles);
-        }
-      }
-
       template <size_t Index, typename... Handles> void putMapOutputs(std::tuple<Handles...>&& handles) const {
         if constexpr (Index < sizeof...(Handles)) {
           if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
@@ -353,7 +279,7 @@ namespace k4FWCore {
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0>(this->m_inputs);
+          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
 
           auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           putMapOutputs<0>(std::move(tmp));

From 7f68c31791f9d278f3e08a9fa688a42a9769b9fc Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:40:24 +0100
Subject: [PATCH 071/127] Clean up and remove ExtractInnerType

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 11 -------
 k4FWCore/include/k4FWCore/Transformer.h     | 34 ---------------------
 2 files changed, 45 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 5f9217da..1760c02e 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -69,17 +69,6 @@ namespace k4FWCore {
       return hash_string(first.first);
     }
 
-    template <typename T> struct ExtractInnerType;
-
-    template <typename Value>
-    struct ExtractInnerType<DataObjectReadHandle<std::map<std::string, std::shared_ptr<Value>>>> {
-      using type = Value;
-    };
-
-    template <typename Value> struct ExtractInnerType<std::map<std::string, std::shared_ptr<Value>>> {
-      using type = Value;
-    };
-
     template <typename T>
     std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(
         const T& arg) {
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 0e7ac56b..7d002787 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -117,40 +117,6 @@ namespace k4FWCore {
           : Transformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                         std::index_sequence_for<Out>{}) {}
 
-      template <size_t Index, typename... Handles> void putMapInputs(const std::tuple<Handles...>& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-            using EDM4hepType =
-                typename ExtractInnerType<typename std::decay_t<decltype(std::get<Index>(handles))>>::type;
-            auto map = std::map<std::string, std::shared_ptr<EDM4hepType>>();
-
-            // To be locked
-            if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-              auto vec = std::vector<std::string>();
-              vec.reserve(m_inputLocations[Index].value().size());
-              for (auto& val : m_inputLocations[Index].value()) {
-                vec.push_back(val.key());
-              }
-              m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
-            }
-
-            for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-              DataObject* p;
-              auto        sc = this->evtSvc()->retrieveObject(value, p);
-              if (!sc.isSuccess()) {
-                throw GaudiException("Failed to retrieve object " + value, "Transformer", StatusCode::FAILURE);
-              }
-              const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-              map[value]            = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-            }
-            std::get<Index>(handles).put(std::move(map));
-          }
-
-          // Recursive call for the next index
-          putMapInputs<Index + 1>(handles);
-        }
-      }
-
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {

From 6f4d0bf86060681e26e82068616d5496994bbf99 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:56:34 +0100
Subject: [PATCH 072/127] Formatting and clean up

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 12 ++++++++----
 k4FWCore/include/k4FWCore/Transformer.h     |  5 +----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 1760c02e..1c05de5d 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -16,11 +16,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#pragma once
+#ifndef FWCORE_FUNCTIONALUTILS_H
+#define FWCORE_FUNCTIONALUTILS_H
 
+#include "GaudiKernel/AnyDataWrapper.h"
+#include "GaudiKernel/DataObject.h"
 #include "podio/CollectionBase.h"
 
-#include "GaudiKernel/DataObjectHandle.h"
 #include "GaudiKernel/EventContext.h"
 #include "GaudiKernel/ThreadLocalContext.h"
 
@@ -117,8 +119,8 @@ namespace k4FWCore {
     };
 
     template <size_t Index, typename... In, typename... Handles>
-    void readMapInputs(const std::tuple<Handles...>& handles, const auto& m_inputLocations,
-                       auto& m_inputLocationsMap, auto thisClass) {
+    void readMapInputs(const std::tuple<Handles...>& handles, const auto& m_inputLocations, auto& m_inputLocationsMap,
+                       auto thisClass) {
       if constexpr (Index < sizeof...(Handles)) {
         if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
           // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
@@ -157,3 +159,5 @@ namespace k4FWCore {
 
   }  // namespace details
 }  // namespace k4FWCore
+
+#endif  // CORE_FUNCTIONALUTILS_H
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 7d002787..a2fbe2c3 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -51,8 +51,7 @@ namespace k4FWCore {
 
       std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
       std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>>                   m_outputs;
-      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
-      std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
+      // std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
       std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
@@ -159,7 +158,6 @@ namespace k4FWCore {
       std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
       std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>...>                m_outputs;
       std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
-      std::map<std::string, std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraOutputs;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
       std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
       mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
@@ -246,7 +244,6 @@ namespace k4FWCore {
       StatusCode execute(const EventContext& ctx) const override final {
         try {
           readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
-
           auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           putMapOutputs<0>(std::move(tmp));
           return Gaudi::Functional::FilterDecision::PASSED;

From 70ef4a85c034d9a5b782d2da5d1591a6974aa9fd Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:56:52 +0100
Subject: [PATCH 073/127] Use the new reader and writer

---
 k4FWCore/components/IOSvc.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index f869e96b..77140734 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -28,8 +28,6 @@
 
 #include "podio/ROOTReader.h"
 #include "podio/ROOTWriter.h"
-// #include "podio/RNTupleReader.h"
-// #include "podio/RNTupleWriter.h"
 
 #include "k4FWCore/KeepDropSwitch.h"
 

From f009f3a11586a04b252d91b743b39234d6d909d4 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 14:51:20 +0100
Subject: [PATCH 074/127] Fix the FunctionalMemory test as it was intended to
 be (no IO)

---
 test/k4FWCoreTest/CMakeLists.txt              |  2 +-
 .../options/ExampleFunctionalMemory.py        | 24 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index ad799ad4..cb1fada5 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -108,7 +108,7 @@ add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIE
 add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit")
 
 
-add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop")
+add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py)
 add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py)
 add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py)
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
index 5a61a26b..207109e5 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
@@ -21,24 +21,24 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalTransformer
-from k4FWCore import ApplicationMgr, IOSvc
+from Configurables import ExampleFunctionalTransformer, ExampleFunctionalProducer, ExampleFunctionalConsumer
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
-iosvc = IOSvc()
-iosvc.input = "output_k4test_exampledata_producer.root"
-iosvc.output = "output_k4test_exampledata_transformer.root"
-# The collections that we don't drop will also be present in the output file
-# out.outputCommands = ["drop MCParticles"]
-
 transformer = ExampleFunctionalTransformer(
-    "ExampleFunctionalTransformer",
-    InputCollection="MCParticles",
-    OutputCollection="NewMCParticles",
+    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+)
+
+producer = ExampleFunctionalProducer("Producer", OutputCollection="MCParticles")
+
+consumer = ExampleFunctionalConsumer(
+    "Consumer",
+    InputCollection="NewMCParticles",
+    Offset=10,
 )
 
 ApplicationMgr(
-    TopAlg=[transformer],
+    TopAlg=[producer, transformer, consumer],
     EvtSel="NONE",
     EvtMax=10,
     ExtSvc=[EventDataSvc("EventDataSvc")],

From 507452694a668a55ec72072cc301af315a5d4352 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 20:27:03 +0100
Subject: [PATCH 075/127] Clean up includes and use k4FWCore::frameLocation

---
 k4FWCore/components/IOSvc.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index d24dd016..e1a60b18 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -19,17 +19,16 @@
 
 #include "IOSvc.h"
 
-#include "edm4hep/MCParticleCollection.h"
 #include "podio/Frame.h"
 #include "podio/FrameCategories.h"
 
+#include "k4FWCore/FunctionalUtils.h"
 #include "k4FWCore/KeepDropSwitch.h"
 
 #include "GaudiKernel/AnyDataWrapper.h"
 #include "GaudiKernel/IEventProcessor.h"
 
 #include <mutex>
-#include <thread>
 #include <tuple>
 
 StatusCode IOSvc::initialize() {
@@ -114,12 +113,16 @@ void IOSvc::handle(const Incident& incident) {
     }
   }
   DataObject* p;
-  code = m_dataSvc->retrieveObject("/Event/_Frame", p);
+  code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
   if (code.isFailure()) {
     return;
   }
 
   auto frame = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+  if (!frame) {
+    error() << "Expected Frame in " << k4FWCore::frameLocation << " but there was something else" << endmsg;
+    return;
+  }
   for (const auto& coll : frame->getData().getAvailableCollections()) {
     DataObject* collPtr;
     code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);

From 40b1b3ff56ad7600f72e872a66111a15df2d6393 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 8 Mar 2024 20:37:47 +0100
Subject: [PATCH 076/127] Move the destructor comment, use .reset() and
 std::make_shared

---
 k4FWCore/components/IOSvc.h | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 77140734..d823bf4b 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -34,15 +34,12 @@
 #include "IIOSvc.h"
 
 #include <string>
-#include <type_traits>
 #include <vector>
 
 class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   using extends::extends;
 
 public:
-  // Gaudi doesn't run the destructor of the Services so we have to
-  // manually ask for the writer to be deleted so it will call finish()
   ~IOSvc() override = default;
 
   StatusCode initialize() override;
@@ -80,21 +77,15 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
 
   std::shared_ptr<podio::ROOTWriter> getWriter() override {
     if (!m_writer) {
-      m_writer = std::shared_ptr<podio::ROOTWriter>(new podio::ROOTWriter(m_writingFileName.value()));
+      m_writer = std::make_shared<podio::ROOTWriter>(m_writingFileName.value());
     }
     return m_writer;
   }
 
-  void deleteWriter() override {
-    if (m_writer) {
-      m_writer = nullptr;
-    }
-  }
-  void deleteReader() override {
-    if (m_reader) {
-      m_reader = nullptr;
-    }
-  }
+  // Gaudi doesn't always run the destructor of the Services so we have to
+  // manually ask for the writer to be deleted so it will call finish()
+  void deleteWriter() override { m_writer.reset(); }
+  void deleteReader() override { m_reader.reset(); }
 
   SmartIF<IDataProviderSvc> m_dataSvc;
   SmartIF<IIncidentSvc>     m_incidentSvc;

From 576a257f5bf2888bb929d648efe32a253ea747c2 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:53:29 +0200
Subject: [PATCH 077/127] Clean up and fix a test

---
 k4FWCore/CMakeLists.txt                           | 6 ++++++
 k4FWCore/components/IIOSvc.h                      | 1 -
 k4FWCore/components/Reader.cpp                    | 6 +++---
 k4FWCore/components/Writer.cpp                    | 2 --
 test/k4FWCoreTest/scripts/check_KeepDropSwitch.py | 5 +++--
 5 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/k4FWCore/CMakeLists.txt b/k4FWCore/CMakeLists.txt
index d28602bd..6a33b8f6 100644
--- a/k4FWCore/CMakeLists.txt
+++ b/k4FWCore/CMakeLists.txt
@@ -57,3 +57,9 @@ add_custom_command(
         COMMAND ${CMAKE_COMMAND} -E copy_directory
                 ${PROJECT_SOURCE_DIR}/k4FWCore/python/k4FWCore
                 ${CMAKE_CURRENT_BINARY_DIR}/genConfDir/k4FWCore)
+
+# This is needed to overwrite the __init__.py, see a long comment in the
+# CMakeLists.txt in the test folder
+add_custom_command(TARGET k4FWCorePlugins POST_BUILD
+                   COMMAND ${CMAKE_COMMAND} -E copy
+                   ${PROJECT_SOURCE_DIR}/python/k4FWCore/__init__.py ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/__init__.py)
diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index be3a353a..6f56c9da 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -24,7 +24,6 @@
 #include "podio/CollectionBase.h"
 #include "podio/ROOTWriter.h"
 
-#include <exception>
 #include <memory>
 #include <vector>
 
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index cbbd0582..66849bc4 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -48,8 +48,8 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
     try {
       auto out = (*this)();
 
-      auto outColls        = std::get<0>(out);
-      auto outputLocations = std::get<1>(out);
+      auto outColls        = std::get<std::vector<std::shared_ptr<podio::CollectionBase>>>(out);
+      auto outputLocations = std::get<std::vector<std::string>>(out);
 
       // if (out.size() != m_outputs.size()) {
       //   throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) +
@@ -117,7 +117,7 @@ class Reader final : public CollectionPusher {
     auto val = iosvc->next();
 
     auto eds   = eventSvc().as<IDataProviderSvc>();
-    auto frame = std::move(std::get<2>(val));
+    auto frame = std::move(std::get<podio::Frame>(val));
 
     auto tmp  = new AnyDataWrapper<podio::Frame>(std::move(frame));
     auto code = eds->registerObject("/Event" + k4FWCore::frameLocation, tmp);
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 7b1fb334..2000f499 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -24,7 +24,6 @@
 #include "GaudiKernel/SmartDataPtr.h"
 #include "GaudiKernel/StatusCode.h"
 
-#include "edm4hep/MCParticleCollection.h"
 #include "podio/Frame.h"
 
 #include "IIOSvc.h"
@@ -34,7 +33,6 @@
 
 #include <GaudiKernel/IHiveWhiteBoard.h>
 #include <memory>
-#include <thread>
 
 class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&)> {
 public:
diff --git a/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py b/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py
index 82c3e576..09e24533 100644
--- a/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py
+++ b/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py
@@ -17,10 +17,11 @@
 # limitations under the License.
 #
 import ROOT
+import sys
 
 ROOT.gSystem.Load("libedm4hepDict")
 
-file = ROOT.TFile.Open("output_k4test_exampledata_2.root")
+file = ROOT.TFile.Open(sys.argv[1])
 tree = file.Get("events")
 tree.GetEntry(0)
 
@@ -29,5 +30,5 @@
     raise Exception("podio::CollectionBase read from file did not saved properly")
 
 status = tree.GetBranchStatus("MCParticles")
-if status == True:
+if status:
     raise Exception("KeepDropSwitch did not drop the collection")

From 729991662b5d150057cf4cbf4c19d9c35c2a9fcc Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Sun, 10 Mar 2024 17:47:46 +0100
Subject: [PATCH 078/127] Fix writing an arbitrary number of collections

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 18 ++++++++++++++++++
 k4FWCore/include/k4FWCore/Transformer.h       |  8 ++++++++
 test/k4FWCoreTest/options/CheckOutputFiles.py |  3 ++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 1c05de5d..b7084bf6 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -157,6 +157,24 @@ namespace k4FWCore {
       }
     }
 
+    template <size_t Index, typename... In, typename... Handles>
+    void deleteMapInputs(const std::tuple<Handles...>& handles, auto thisClass) {
+      if constexpr (Index < sizeof...(Handles)) {
+        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+          // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+          // we have to remove the reference to get the actual type
+          auto sc = thisClass->evtSvc()->unregisterObject(std::get<Index>(handles).objKey());
+          if (!sc.isSuccess()) {
+            throw GaudiException("Failed to retrieve object " + std::get<Index>(handles).objKey(), "Consumer",
+                                 StatusCode::FAILURE);
+          }
+        }
+      // Recursive call for the next index
+      deleteMapInputs<Index + 1, In...>(handles, thisClass);
+      }
+
+    }
+
   }  // namespace details
 }  // namespace k4FWCore
 
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index a2fbe2c3..29ad8b3b 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -132,6 +132,10 @@ namespace k4FWCore {
                 std::get<0>(this->m_outputs),
                 ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
           }
+          // If any input has map type, we remove it from the store since it has been pushed
+          // but we don't want the map to be available as a map for other algorithms and
+          // all the individual collections have alreaody been pushed
+          deleteMapInputs<0, In...>(this->m_inputs, this);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
@@ -246,6 +250,10 @@ namespace k4FWCore {
           readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
           auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
           putMapOutputs<0>(std::move(tmp));
+          // If any input has map type, we remove it from the store since it has been pushed
+          // but we don't want the map to be available as a map for other algorithms and
+          // all the individual collections have alreaody been pushed
+          deleteMapInputs<0, In...>(this->m_inputs, this);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index cc4ffe44..1c9dcf5b 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -26,6 +26,7 @@
 
 
 def check_collections(filename, names):
+    print(f'Checking file "{filename}" for collections {names}')
     podio_reader = podio.root_io.Reader(filename)
     frames = podio_reader.get("events")
     if not len(frames) and len(names):
@@ -62,4 +63,4 @@ def check_collections(filename, names):
     ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"],
 )
 check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
-check_collections("functional_transformer_runtime_empty.root", [])
+check_collections("functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"])

From a8335d1f46608752eed256890785e8a4f4513f97 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:16:13 +0100
Subject: [PATCH 079/127] Create all the needed data handles for the consumer

---
 k4FWCore/include/k4FWCore/Consumer.h          | 101 ++++++++++++------
 k4FWCore/include/k4FWCore/FunctionalUtils.h   |  67 ++++++++----
 ...ionalConsumerRuntimeCollectionsMultiple.py |   2 +-
 .../components/ExampleFunctionalConsumer.cpp  |   1 -
 ...leFunctionalConsumerRuntimeCollections.cpp |   2 +-
 ...onalConsumerRuntimeCollectionsMultiple.cpp |   2 +-
 6 files changed, 122 insertions(+), 53 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index bf2e0c29..bba1e4b9 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -23,11 +23,10 @@
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
 
-#include "podio/CollectionBase.h"
+// #include "GaudiKernel/CommonMessaging.h"
 
 #include "k4FWCore/FunctionalUtils.h"
 
-#include <memory>
 #include <type_traits>
 #include <utility>
 
@@ -35,6 +34,61 @@ namespace k4FWCore {
 
   namespace details {
 
+    template <size_t Index, typename... In, typename... Handles, typename InputTuple>
+    void readMapInputss(const std::tuple<Handles...>& handles, auto thisClass, InputTuple& inputTuple) {
+      if constexpr (Index < sizeof...(Handles)) {
+        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+          // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
+          // we have to remove the reference to get the actual type
+          using EDM4hepType =
+              std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
+          auto inputMap = std::map<std::string, const EDM4hepType&>();
+          for (auto& handle : std::get<Index>(handles)) {
+            auto in = get(handle, thisClass, Gaudi::Hive::currentContext());
+            inputMap.emplace(handle.objKey(), *static_cast<EDM4hepType*>(in.get()));
+          }
+          std::get<Index>(inputTuple) = std::move(inputMap);
+
+        } else {
+          auto in                     = get(std::get<Index>(handles)[0], thisClass, Gaudi::Hive::currentContext());
+          std::get<Index>(inputTuple) = static_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(in.get());
+        }
+
+        // Recursive call for the next index
+        readMapInputss<Index + 1, In...>(handles, thisClass, inputTuple);
+      }
+    }
+
+    inline std::vector<DataObjID> to_DataObjID(const std::vector<std::string>& in) {
+      std::vector<DataObjID> out;
+      out.reserve(in.size());
+      std::transform(in.begin(), in.end(), std::back_inserter(out), [](const std::string& i) { return DataObjID{i}; });
+      return out;
+    }
+
+    template <typename... In> struct filter_evtcontext_ttt {
+      static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
+                    "EventContext can only appear as first argument");
+
+      template <typename Algorithm, typename Handles> static auto apply(const Algorithm& algo, Handles& handles) {
+        return std::apply(
+            [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
+      }
+
+      template <typename Algorithm, typename Handles>
+      static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+        auto inputTuple = std::tuple<SelectType<In>...>();
+
+        // Build the input tuple by picking up either std::map with an arbitrary
+        // number of collections or single collections
+        readMapInputss<0, In...>(handles, &algo, inputTuple);
+
+        return std::apply(
+            [&](const auto&... input) { return algo(maybeTransformToEDM4hep2<decltype(input)>(input)...); },
+            inputTuple);
+      }
+    };
+
     template <typename Signature, typename Traits_> struct Consumer;
 
     template <typename... In, typename Traits_>
@@ -45,11 +99,8 @@ namespace k4FWCore {
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                 m_inputs;
-      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>> m_extraInputs;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                        m_inputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                     m_inputLocationsPair{};
-      mutable std::map<std::string, std::vector<std::string>>                                   m_inputLocationsMap;
+      std::tuple<std::vector<InputHandle_t<typename transformType<In>::type>>...> m_inputs;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>          m_inputLocations{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
@@ -59,31 +110,19 @@ namespace k4FWCore {
       Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
           : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
-            m_inputLocationsPair{Gaudi::Property<DataObjID>{
-                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
+                this, std::get<I>(inputs).first, to_DataObjID(std::get<I>(inputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<In>::value) {
-                    auto& handle = std::get<I>(m_inputs);
-                    auto& ins    = m_inputLocationsPair[I];
-                    handle       = {ins.value(), this};
+                  std::vector<InputHandle_t<typename transformType<In>::type>> h;
+                  for (auto& value : this->m_inputLocations[I].value()) {
+                    auto handle = InputHandle_t<typename transformType<In>::type>(value, this);
+                    h.push_back(std::move(handle));
                   }
+                  std::get<I>(m_inputs) = std::move(h);
                 },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
-            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...}
-
-      {
-        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
-        //   // for (auto& value : std::get<I...>(inputs).second) {
-        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
-        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
-        //   // }
-        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
-        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
-        //   // auto& handles = std::get<0>(m_inputs);
-        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
-        // }
-      }
+
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...}
+
+      {}
 
       Consumer(std::string name, ISvcLocator* locator,
                Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
@@ -92,8 +131,8 @@ namespace k4FWCore {
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
-          filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+          Gaudi::Algorithm::info() << "Executing " << this->name() << endmsg;
+          filter_evtcontext_ttt<In...>::apply(*this, ctx, m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index b7084bf6..f8df9764 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -46,6 +46,41 @@ namespace k4FWCore {
       return static_cast<const T&>(*arg);
     }
 
+    // This function will be used to modify std::shared_ptr<podio::CollectionBase> to the actual collection type
+    template <typename T, typename P> const auto& maybeTransformToEDM4hep2(const P& arg) { return arg; }
+    template <typename T, typename P>
+      requires std::is_base_of_v<podio::CollectionBase, P>
+    const auto& maybeTransformToEDM4hep2(P* arg) {
+      return *arg;
+    }
+
+    template <typename T, typename P>
+      requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
+    const auto& maybeTransformToEDM4hep2(P arg) {
+      return static_cast<const T&>(*arg);
+    }
+
+    template <typename T, bool addPtr = std::is_base_of_v<podio::CollectionBase, T>>
+    using SelectType = std::conditional_t<addPtr, std::add_pointer_t<T>, T>;
+
+    template <typename Tuple, typename F, std::size_t... I>
+    decltype(auto) apply_helper(F&& f, Tuple&& t, std::index_sequence<I...>) {
+      return (std::forward<F>(f)(std::get<I...>(std::forward<Tuple>(t))), 0);
+    }
+
+    template <typename Tuple, typename F> decltype(auto) apply_index_sequence(F&& f, Tuple&& t) {
+      constexpr std::size_t tuple_size = std::tuple_size<std::remove_reference_t<Tuple>>::value;
+      return apply_helper(std::forward<F>(f), std::forward<Tuple>(t), std::make_index_sequence<tuple_size>{});
+    }
+
+    template <typename Tuple, typename F> decltype(auto) apply(F&& f, auto ctx, Tuple&& t) {
+      return apply_index_sequence(std::forward<F>(f), ctx, std::forward<Tuple>(t));
+    }
+
+    template <typename T> const auto& transformtoEDM4hep(const std::shared_ptr<podio::CollectionBase>& arg) {
+      return static_cast<const T>(*arg);
+    }
+
     template <typename T> struct is_map_like : std::false_type {};
 
     template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
@@ -71,23 +106,20 @@ namespace k4FWCore {
       return hash_string(first.first);
     }
 
-    template <typename T>
-    std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::shared_ptr<podio::CollectionBase>> transformType(
-        const T& arg) {
-      return std::shared_ptr<podio::CollectionBase>(arg);
-    }
+    // transformType function to transform the types from the ones that the user wants
+    // like edm4hep::MCParticleCollection, to the ones that are actually stored in the
+    // event store, like std::shared_ptr<podio::CollectionBase>
+    // For std::map<std::string, Coll>, th
 
-    // template <typename T>
-    // std::enable_if_t<std::is_base_of_v<podio::CollectionBase, T>, std::map<std::string, std::shared_ptr<T>>>
-    // transformType(const std::map<std::string, std::shared_ptr<T>>& arg) {
-    //   return std::map<std::string, std::shared_ptr<T>>();
-    // }
+    template <typename T> struct transformType {
+      using type = T;
+    };
 
     template <typename T>
-    std::enable_if_t<!std::is_base_of_v<podio::CollectionBase, T>, T> transformType(const T& arg) {
-      // Default: no transformation
-      return arg;
-    }
+      requires std::is_base_of_v<podio::CollectionBase, T> || is_map_like<T>::value
+    struct transformType<T> {
+      using type = std::shared_ptr<podio::CollectionBase>;
+    };
 
     template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
     auto ptrOrCast(T&& arg) {
@@ -149,7 +181,7 @@ namespace k4FWCore {
             auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
             inputMap.emplace(value, *ptr);
           }
-          std::get<Index>(handles).put(std::move(inputMap));
+          // std::get<Index>(handles).put(std::move(inputMap));
         }
 
         // Recursive call for the next index
@@ -169,10 +201,9 @@ namespace k4FWCore {
                                  StatusCode::FAILURE);
           }
         }
-      // Recursive call for the next index
-      deleteMapInputs<Index + 1, In...>(handles, thisClass);
+        // Recursive call for the next index
+        deleteMapInputs<Index + 1, In...>(handles, thisClass);
       }
-
     }
 
   }  // namespace details
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index b7706925..18d6d329 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -69,7 +69,7 @@
         "MCParticles4",
     ],
     Tracks=["Tracks0", "Tracks1", "Tracks2"],
-    SimTrackerHits="SimTrackerHits0",
+    SimTrackerHits=["SimTrackerHits0"],
     Offset=0,
 )
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index 3bc56cd7..903c3bcc 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -25,7 +25,6 @@
 
 #include <stdexcept>
 #include <string>
-#include <thread>
 
 struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::MCParticleCollection& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
index e98b338a..dd016eab 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp
@@ -35,7 +35,7 @@ struct ExampleFunctionalConsumerRuntimeCollections final
   // This is the function that will be called to produce the data
   void operator()(const std::map<std::string, const edm4hep::MCParticleCollection&>& input) const override {
     if (input.size() != 3) {
-      fatal() << "Wrong size of the input map, expected 3, got " << input.size() << endmsg;
+      throw std::runtime_error("Wrong size of the input map, expected 3, got " + std::to_string(input.size()));
     }
     for (auto& [key, val] : input) {
       int i = 0;
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index 04428d32..233812d8 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -46,7 +46,7 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
     info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size()
            << " track collections" << endmsg;
     if (particleMap.size() != 5) {
-      fatal() << "Wrong size of the particleMap map, expected 5, got " << particleMap.size() << endmsg;
+      throw std::runtime_error("Wrong size of the particleMap map, expected 5, got " + std::to_string(particleMap.size()));
     }
     for (auto& [key, particles] : particleMap) {
       int         i         = 0;

From 4ade0f32cbda54baf556f9ff5aaf8f638e26dc97 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 13 Mar 2024 22:00:34 +0100
Subject: [PATCH 080/127] Create the right handles when using the transformer

---
 k4FWCore/include/k4FWCore/Consumer.h          |  59 +-----
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 147 ++++++---------
 k4FWCore/include/k4FWCore/Transformer.h       | 175 ++++++------------
 ...pleFunctionalConsumerRuntimeCollections.py |   6 +-
 ...ionalConsumerRuntimeCollectionsMultiple.py |  36 ++--
 .../options/ExampleFunctionalFile.py          |   2 +-
 .../options/ExampleFunctionalMTFile.py        |   2 +-
 .../options/ExampleFunctionalMTMemory.py      |   9 +-
 .../options/ExampleFunctionalMemory.py        |   6 +-
 .../ExampleFunctionalMultipleMemory.py        |  24 +--
 .../ExampleFunctionalProducerMultiple.py      |  12 +-
 ...pleFunctionalProducerRuntimeCollections.py |   6 +-
 ...alTransformerRuntimeCollectionsMultiple.py |  72 ++++---
 ...xampleFunctionalTransformerRuntimeEmpty.py |   7 +-
 14 files changed, 212 insertions(+), 351 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index bba1e4b9..c9a82b98 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -34,61 +34,6 @@ namespace k4FWCore {
 
   namespace details {
 
-    template <size_t Index, typename... In, typename... Handles, typename InputTuple>
-    void readMapInputss(const std::tuple<Handles...>& handles, auto thisClass, InputTuple& inputTuple) {
-      if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-          // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
-          // we have to remove the reference to get the actual type
-          using EDM4hepType =
-              std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
-          auto inputMap = std::map<std::string, const EDM4hepType&>();
-          for (auto& handle : std::get<Index>(handles)) {
-            auto in = get(handle, thisClass, Gaudi::Hive::currentContext());
-            inputMap.emplace(handle.objKey(), *static_cast<EDM4hepType*>(in.get()));
-          }
-          std::get<Index>(inputTuple) = std::move(inputMap);
-
-        } else {
-          auto in                     = get(std::get<Index>(handles)[0], thisClass, Gaudi::Hive::currentContext());
-          std::get<Index>(inputTuple) = static_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(in.get());
-        }
-
-        // Recursive call for the next index
-        readMapInputss<Index + 1, In...>(handles, thisClass, inputTuple);
-      }
-    }
-
-    inline std::vector<DataObjID> to_DataObjID(const std::vector<std::string>& in) {
-      std::vector<DataObjID> out;
-      out.reserve(in.size());
-      std::transform(in.begin(), in.end(), std::back_inserter(out), [](const std::string& i) { return DataObjID{i}; });
-      return out;
-    }
-
-    template <typename... In> struct filter_evtcontext_ttt {
-      static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
-                    "EventContext can only appear as first argument");
-
-      template <typename Algorithm, typename Handles> static auto apply(const Algorithm& algo, Handles& handles) {
-        return std::apply(
-            [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles);
-      }
-
-      template <typename Algorithm, typename Handles>
-      static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
-        auto inputTuple = std::tuple<SelectType<In>...>();
-
-        // Build the input tuple by picking up either std::map with an arbitrary
-        // number of collections or single collections
-        readMapInputss<0, In...>(handles, &algo, inputTuple);
-
-        return std::apply(
-            [&](const auto&... input) { return algo(maybeTransformToEDM4hep2<decltype(input)>(input)...); },
-            inputTuple);
-      }
-    };
-
     template <typename Signature, typename Traits_> struct Consumer;
 
     template <typename... In, typename Traits_>
@@ -119,9 +64,7 @@ namespace k4FWCore {
                   }
                   std::get<I>(m_inputs) = std::move(h);
                 },
-
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...}
-
       {}
 
       Consumer(std::string name, ISvcLocator* locator,
@@ -132,7 +75,7 @@ namespace k4FWCore {
       StatusCode execute(const EventContext& ctx) const override final {
         try {
           Gaudi::Algorithm::info() << "Executing " << this->name() << endmsg;
-          filter_evtcontext_ttt<In...>::apply(*this, ctx, m_inputs);
+          filter_evtcontext_tt<In...>::apply(*this, ctx, m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index f8df9764..1542d1c4 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -19,8 +19,8 @@
 #ifndef FWCORE_FUNCTIONALUTILS_H
 #define FWCORE_FUNCTIONALUTILS_H
 
-#include "GaudiKernel/AnyDataWrapper.h"
-#include "GaudiKernel/DataObject.h"
+#include "Gaudi/Functional/details.h"
+#include "GaudiKernel/DataObjID.h"
 #include "podio/CollectionBase.h"
 
 #include "GaudiKernel/EventContext.h"
@@ -39,43 +39,20 @@ namespace k4FWCore {
 
     // This function will be used to modify std::shared_ptr<podio::CollectionBase> to the actual collection type
     template <typename T, typename P> const auto& maybeTransformToEDM4hep(const P& arg) { return arg; }
-
-    template <typename T, typename P>
-      requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
-    const auto& maybeTransformToEDM4hep(const P& arg) {
-      return static_cast<const T&>(*arg);
-    }
-
-    // This function will be used to modify std::shared_ptr<podio::CollectionBase> to the actual collection type
-    template <typename T, typename P> const auto& maybeTransformToEDM4hep2(const P& arg) { return arg; }
     template <typename T, typename P>
       requires std::is_base_of_v<podio::CollectionBase, P>
-    const auto& maybeTransformToEDM4hep2(P* arg) {
+    const auto& maybeTransformToEDM4hep(P* arg) {
       return *arg;
     }
 
     template <typename T, typename P>
       requires std::same_as<P, std::shared_ptr<podio::CollectionBase>>
-    const auto& maybeTransformToEDM4hep2(P arg) {
+    const auto& maybeTransformToEDM4hep(P arg) {
       return static_cast<const T&>(*arg);
     }
 
     template <typename T, bool addPtr = std::is_base_of_v<podio::CollectionBase, T>>
-    using SelectType = std::conditional_t<addPtr, std::add_pointer_t<T>, T>;
-
-    template <typename Tuple, typename F, std::size_t... I>
-    decltype(auto) apply_helper(F&& f, Tuple&& t, std::index_sequence<I...>) {
-      return (std::forward<F>(f)(std::get<I...>(std::forward<Tuple>(t))), 0);
-    }
-
-    template <typename Tuple, typename F> decltype(auto) apply_index_sequence(F&& f, Tuple&& t) {
-      constexpr std::size_t tuple_size = std::tuple_size<std::remove_reference_t<Tuple>>::value;
-      return apply_helper(std::forward<F>(f), std::forward<Tuple>(t), std::make_index_sequence<tuple_size>{});
-    }
-
-    template <typename Tuple, typename F> decltype(auto) apply(F&& f, auto ctx, Tuple&& t) {
-      return apply_index_sequence(std::forward<F>(f), ctx, std::forward<Tuple>(t));
-    }
+    using addPtrIfColl = std::conditional_t<addPtr, std::add_pointer_t<T>, T>;
 
     template <typename T> const auto& transformtoEDM4hep(const std::shared_ptr<podio::CollectionBase>& arg) {
       return static_cast<const T>(*arg);
@@ -85,32 +62,10 @@ namespace k4FWCore {
 
     template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
 
-    inline std::string hash_string(const std::string& str) {
-      std::string result = "_";
-      for (auto& c : std::to_string(std::hash<std::string>{}(str))) {
-        result += 'a' + (c - '0');
-      }
-      return result;
-    }
-
-    template <typename T, typename K> std::string getName(const K& first, bool pair = false) {
-      if constexpr (is_map_like<T>::value) {
-        if (pair) {
-          return hash_string(first.first);
-        }
-        return first.first;
-      }
-      if (pair) {
-        return first.first;
-      }
-      return hash_string(first.first);
-    }
-
     // transformType function to transform the types from the ones that the user wants
     // like edm4hep::MCParticleCollection, to the ones that are actually stored in the
     // event store, like std::shared_ptr<podio::CollectionBase>
     // For std::map<std::string, Coll>, th
-
     template <typename T> struct transformType {
       using type = T;
     };
@@ -144,15 +99,19 @@ namespace k4FWCore {
 
       template <typename Algorithm, typename Handles>
       static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+        auto inputTuple = std::tuple<addPtrIfColl<In>...>();
+
+        // Build the input tuple by picking up either std::map with an arbitrary
+        // number of collections or single collections
+        readMapInputs<0, In...>(handles, &algo, inputTuple);
+
         return std::apply(
-            [&](const auto&... handle) { return algo(maybeTransformToEDM4hep<In>(get(handle, algo, ctx))...); },
-            handles);
+            [&](const auto&... input) { return algo(maybeTransformToEDM4hep<decltype(input)>(input)...); }, inputTuple);
       }
     };
 
-    template <size_t Index, typename... In, typename... Handles>
-    void readMapInputs(const std::tuple<Handles...>& handles, const auto& m_inputLocations, auto& m_inputLocationsMap,
-                       auto thisClass) {
+    template <size_t Index, typename... In, typename... Handles, typename InputTuple>
+    void readMapInputs(const std::tuple<Handles...>& handles, auto thisClass, InputTuple& inputTuple) {
       if constexpr (Index < sizeof...(Handles)) {
         if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
           // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
@@ -160,52 +119,68 @@ namespace k4FWCore {
           using EDM4hepType =
               std::remove_reference_t<typename std::tuple_element_t<Index, std::tuple<In...>>::mapped_type>;
           auto inputMap = std::map<std::string, const EDM4hepType&>();
-
-          // To be locked
-          if (!m_inputLocationsMap.contains(std::get<Index>(handles).objKey())) {
-            auto vec = std::vector<std::string>();
-            vec.reserve(m_inputLocations[Index].value().size());
-            for (auto& val : m_inputLocations[Index].value()) {
-              vec.push_back(val.key());
-            }
-            m_inputLocationsMap[std::get<Index>(handles).objKey()] = vec;
+          for (auto& handle : std::get<Index>(handles)) {
+            auto in = get(handle, thisClass, Gaudi::Hive::currentContext());
+            inputMap.emplace(handle.objKey(), *static_cast<EDM4hepType*>(in.get()));
           }
+          std::get<Index>(inputTuple) = std::move(inputMap);
 
-          for (auto& value : m_inputLocationsMap.at(std::get<Index>(handles).objKey())) {
-            DataObject* p;
-            auto        sc = thisClass->evtSvc()->retrieveObject(value, p);
-            if (!sc.isSuccess()) {
-              throw GaudiException("Failed to retrieve object " + value, "Consumer", StatusCode::FAILURE);
-            }
-            const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(p);
-            auto       ptr        = std::dynamic_pointer_cast<EDM4hepType>(collection->getData());
-            inputMap.emplace(value, *ptr);
-          }
-          // std::get<Index>(handles).put(std::move(inputMap));
+        } else {
+          auto in                     = get(std::get<Index>(handles)[0], thisClass, Gaudi::Hive::currentContext());
+          std::get<Index>(inputTuple) = static_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(in.get());
         }
 
         // Recursive call for the next index
-        readMapInputs<Index + 1, In...>(handles, m_inputLocations, m_inputLocationsMap, thisClass);
+        readMapInputs<Index + 1, In...>(handles, thisClass, inputTuple);
       }
     }
 
-    template <size_t Index, typename... In, typename... Handles>
-    void deleteMapInputs(const std::tuple<Handles...>& handles, auto thisClass) {
+    template <size_t Index, typename... Out, typename... Handles>
+    void putMapOutputs(std::tuple<Handles...>&& handles, const auto& m_outputs, auto thisClass) {
       if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
-          // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
-          // we have to remove the reference to get the actual type
-          auto sc = thisClass->evtSvc()->unregisterObject(std::get<Index>(handles).objKey());
-          if (!sc.isSuccess()) {
-            throw GaudiException("Failed to retrieve object " + std::get<Index>(handles).objKey(), "Consumer",
-                                 StatusCode::FAILURE);
+        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
+          int i = 0;
+          if (std::get<Index>(handles).size() != std::get<Index>(m_outputs).size()) {
+            std::string msg = "Size of the output map " + std::to_string(std::get<Index>(handles).size()) +
+                              " does not match the expected size from the steering file " +
+                              std::to_string(std::get<Index>(m_outputs).size()) + ". Expected the collections: ";
+            for (auto& out : std::get<Index>(m_outputs)) {
+              msg += out.objKey() + " ";
+            }
+            msg += " but got the collections: ";
+            for (auto& out : std::get<Index>(handles)) {
+              msg += out.first + " ";
+            }
+            throw GaudiException(thisClass->name(), msg, StatusCode::FAILURE);
           }
+          for (auto& [key, val] : std::get<Index>(handles)) {
+            if (key != std::get<Index>(m_outputs)[i].objKey()) {
+              throw GaudiException(thisClass->name(),
+                                   "Output key in the map \"" + key +
+                                       "\" does not match the expected key from the steering file \"" +
+                                       std::get<Index>(m_outputs)[i].objKey() + "\"",
+                                   StatusCode::FAILURE);
+            }
+            Gaudi::Functional::details::put(std::get<Index>(m_outputs)[i], ptrOrCast(std::move(val)));
+            i++;
+          }
+        } else {
+          Gaudi::Functional::details::put(std::get<Index>(m_outputs)[0],
+                                          ptrOrCast(std::move(std::get<Index>(handles))));
         }
+
         // Recursive call for the next index
-        deleteMapInputs<Index + 1, In...>(handles, thisClass);
+        putMapOutputs<Index + 1, Out...>(std::move(handles), m_outputs, thisClass);
       }
     }
 
+    inline std::vector<DataObjID> to_DataObjID(const std::vector<std::string>& in) {
+      std::vector<DataObjID> out;
+      out.reserve(in.size());
+      std::transform(in.begin(), in.end(), std::back_inserter(out), [](const std::string& i) { return DataObjID{i}; });
+      return out;
+    }
+
   }  // namespace details
 }  // namespace k4FWCore
 
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 29ad8b3b..f5bd8ced 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -23,13 +23,10 @@
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
 
-#include "podio/CollectionBase.h"
-
 #include "k4FWCore/FunctionalUtils.h"
 
 // #include "GaudiKernel/CommonMessaging.h"
 
-#include <memory>
 #include <type_traits>
 #include <utility>
 
@@ -49,14 +46,10 @@ namespace k4FWCore {
       template <typename T>
       using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
-      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>>                   m_outputs;
-      // std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
-      mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, 1>                                     m_outputLocations{};
-      std::array<Gaudi::Property<DataObjID>, 1> m_outputLocationsPair{};
+      std::tuple<std::vector<InputHandle_t<typename transformType<In>::type>>...> m_inputs;
+      std::tuple<std::vector<OutputHandle_t<typename transformType<Out>::type>>>           m_outputs;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>          m_inputLocations{};
+      std::array<Gaudi::Property<std::vector<DataObjID>>, 1>                      m_outputLocations{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
@@ -67,45 +60,32 @@ namespace k4FWCore {
                   const OArgs& outputs, std::index_sequence<J...>)
           : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
-            m_inputLocationsPair{Gaudi::Property<DataObjID>{
-                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
+                this, std::get<I>(inputs).first, to_DataObjID(std::get<I>(inputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<In>::value) {
-                    auto& handle = std::get<I>(m_inputs);
-                    auto& ins    = m_inputLocationsPair[I];
-                    handle       = {ins.value(), this};
+                  std::vector<InputHandle_t<typename transformType<In>::type>> h;
+                  for (auto& value : this->m_inputLocations[I].value()) {
+                    auto handle = InputHandle_t<typename transformType<In>::type>(value, this);
+                    h.push_back(std::move(handle));
                   }
+                  std::get<I>(m_inputs) = std::move(h);
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
             m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this, getName<Out>(std::get<J>(outputs), false), {DataObjID{std::get<J>(outputs).second[0]}}}...},
-            m_outputLocationsPair{Gaudi::Property<DataObjID>{
-                this, getName<Out>(std::get<J>(outputs), true), DataObjID{std::get<J>(outputs).second[0]},
+                this, std::get<J>(outputs).first, to_DataObjID(std::get<J>(outputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<Out>::value) {
-                    auto& handle = std::get<J>(m_outputs);
-                    auto& ins    = m_outputLocationsPair[J];
-                    handle       = {ins.value(), this};
+                  std::vector<OutputHandle_t<typename transformType<Out>::type>> h;
+                  std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
+                            [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
+                  for (auto& inpID : this->m_outputLocations[J].value()) {
+                    if (inpID.key().empty()) {
+                      continue;
+                    }
+                    auto handle = OutputHandle_t<typename transformType<Out>::type>(inpID, this);
+                    h.push_back(std::move(handle));
                   }
+                  std::get<0>(m_outputs) = std::move(h);
                 },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
-            // m_inputs{std::tuple<InputHandle_t<decltype(transformType<In>...)>>(InputHandle_t<decltype(transformType<In>...)>(std::get<I>(inputs).first, this)...)}
-            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...},
-            m_outputs{OutputHandle_t<decltype(transformType(std::declval<Out>()))>(std::get<J>(outputs).first, this)...}
-
-      {
-        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
-        //   // for (auto& value : std::get<I...>(inputs).second) {
-        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
-        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
-        //   // }
-        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
-        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
-        //   // auto& handles = std::get<0>(m_inputs);
-        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
-        // }
-      }
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {}
 
       constexpr static std::size_t N_in  = sizeof...(In);
       constexpr static std::size_t N_out = 1;
@@ -119,23 +99,25 @@ namespace k4FWCore {
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
           if constexpr (is_map_like<Out>::value) {
-            for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
-              auto        shared = std::make_shared<decltype(val)>(std::move(val));
-              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(shared);
-              DataObject* p      = w;
-              auto        sc     = this->evtSvc()->registerObject(key, p);
-            }
+            std::tuple<Out> tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+            // auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
+            putMapOutputs<0, Out>(std::move(tmp), m_outputs, this);
+            // int i = 0;
+            // for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
+            //   if (key != m_outputs[i].objKey()) {
+            //     throw GaudiException("Transformer",
+            //                          "Output key does not match the expected key " + m_outputs[i].objKey(),
+            //                          StatusCode::FAILURE);
+            //   }
+            //   Gaudi::Functional::details::put(m_outputs[i], ptrOrCast(std::move(val)));
+            //   i++;
+            // }
           } else {
             Gaudi::Functional::details::put(
-                std::get<0>(this->m_outputs),
+                                            std::get<0>(this->m_outputs)[0],
                 ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
           }
-          // If any input has map type, we remove it from the store since it has been pushed
-          // but we don't want the map to be available as a map for other algorithms and
-          // all the individual collections have alreaody been pushed
-          deleteMapInputs<0, In...>(this->m_inputs, this);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
@@ -159,14 +141,10 @@ namespace k4FWCore {
       template <typename T>
       using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
-      std::tuple<InputHandle_t<decltype(transformType(std::declval<In>()))>...>                  m_inputs;
-      std::tuple<OutputHandle_t<decltype(transformType(std::declval<Out>()))>...>                m_outputs;
-      std::map<std::string, std::vector<InputHandle_t<std::shared_ptr<podio::CollectionBase>>>>  m_extraInputs;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>                         m_inputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(In)>                                      m_inputLocationsPair{};
-      mutable std::map<std::string, std::vector<std::string>>                                    m_inputLocationsMap;
-      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)>                        m_outputLocations{};
-      std::array<Gaudi::Property<DataObjID>, sizeof...(Out)> m_outputLocationsPair{};
+      std::tuple<std::vector<InputHandle_t<typename transformType<In>::type>>...>   m_inputs;
+      std::tuple<std::vector<OutputHandle_t<typename transformType<Out>::type>>...> m_outputs;
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>            m_inputLocations{};
+      std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(Out)>           m_outputLocations{};
 
       using base_class = Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>;
 
@@ -177,44 +155,32 @@ namespace k4FWCore {
                        const OArgs& outputs, std::index_sequence<J...>)
           : base_class(std::move(name), locator),
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this, getName<In>(std::get<I>(inputs), false), {DataObjID{std::get<I>(inputs).second[0]}}}...},
-            m_inputLocationsPair{Gaudi::Property<DataObjID>{
-                this, getName<In>(std::get<I>(inputs), true), DataObjID{std::get<I>(inputs).second[0]},
+                this, std::get<I>(inputs).first, to_DataObjID(std::get<I>(inputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<In>::value) {
-                    auto& handle = std::get<I>(m_inputs);
-                    auto& ins    = m_inputLocationsPair[I];
-                    handle       = {ins.value(), this};
+                  std::vector<InputHandle_t<typename transformType<In>::type>> h;
+                  for (auto& value : this->m_inputLocations[I].value()) {
+                    auto handle = InputHandle_t<typename transformType<In>::type>(value, this);
+                    h.push_back(std::move(handle));
                   }
+                  std::get<I>(m_inputs) = std::move(h);
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
             m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
-                this, getName<Out>(std::get<J>(outputs), false), {DataObjID{std::get<J>(outputs).second[0]}}}...},
-            m_outputLocationsPair{Gaudi::Property<DataObjID>{
-                this, getName<Out>(std::get<J>(outputs), true), DataObjID{std::get<J>(outputs).second[0]},
+                this, std::get<J>(outputs).first, to_DataObjID(std::get<J>(outputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  if constexpr (!is_map_like<Out>::value) {
-                    auto& handle = std::get<J>(m_outputs);
-                    auto& ins    = m_outputLocationsPair[J];
-                    handle       = {ins.value(), this};
+                  std::vector<OutputHandle_t<typename transformType<Out>::type>> h;
+                  std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
+                            [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
+                  for (auto& inpID : this->m_outputLocations[J].value()) {
+                    if (inpID.key().empty()) {
+                      continue;
+                    }
+                    auto handle = OutputHandle_t<typename transformType<Out>::type>(inpID, this);
+                    h.push_back(std::move(handle));
                   }
+                  std::get<J>(m_outputs) = std::move(h);
                 },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
-            m_inputs{InputHandle_t<decltype(transformType(std::declval<In>()))>(std::get<I>(inputs).first, this)...},
-            m_outputs{OutputHandle_t<decltype(transformType(std::declval<Out>()))>(std::get<J>(outputs).first, this)...}
-
-      {
-        // if constexpr (std::is_same_v<In, std::map<std::string, std::shared_ptr<podio::CollectionBase>>>) {
-        //   // for (auto& value : std::get<I...>(inputs).second) {
-        //   //   Gaudi::Algorithm::info() << "Adding extra input " << value << endmsg;
-        //   //   m_extraInputs["InputCollection"].emplace_back(InputHandle_t<std::shared_ptr<podio::CollectionBase>>(value, this));
-        //   // }
-        //   // m_inputs = std::make_tuple( typename maybeVector<In>::type()... );
-        //   // m_inputs = std::make_tuple( DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>() );
-        //   // auto& handles = std::get<0>(m_inputs);
-        //   // handles.push_back(DataObjectReadHandle<std::map<std::string, std::shared_ptr<podio::CollectionBase>>>(DataObjID{"InputCollection"}, this));
-        // }
-      }
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {}
 
       constexpr static std::size_t N_in  = sizeof...(In);
       constexpr static std::size_t N_out = sizeof...(Out);
@@ -225,35 +191,12 @@ namespace k4FWCore {
           : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                              std::index_sequence_for<Out...>{}) {}
 
-      template <size_t Index, typename... Handles> void putMapOutputs(std::tuple<Handles...>&& handles) const {
-        if constexpr (Index < sizeof...(Handles)) {
-          if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
-            for (auto& [key, val] : std::get<Index>(handles)) {
-              auto        shared = std::make_shared<decltype(val)>(std::move(val));
-              auto        w      = new AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>(std::move(shared));
-              DataObject* p      = w;
-              auto        sc     = this->evtSvc()->registerObject(key, p);
-            }
-
-          } else {
-            Gaudi::Functional::details::put(std::get<Index>(m_outputs), ptrOrCast(std::move(std::get<Index>(handles))));
-          }
-
-          // Recursive call for the next index
-          putMapOutputs<Index + 1>(std::move(handles));
-        }
-      }
 
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          readMapInputs<0, In...>(this->m_inputs, m_inputLocations, m_inputLocationsMap, this);
           auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-          putMapOutputs<0>(std::move(tmp));
-          // If any input has map type, we remove it from the store since it has been pushed
-          // but we don't want the map to be available as a map for other algorithms and
-          // all the individual collections have alreaody been pushed
-          deleteMapInputs<0, In...>(this->m_inputs, this);
+          putMapOutputs<0, Out...>(std::move(tmp), m_outputs, this);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
           (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg;
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
index 0109faaf..68b38c28 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py
@@ -30,15 +30,15 @@
 
 producer0 = ExampleFunctionalProducer(
     "Producer0",
-    OutputCollection="MCParticles0",
+    OutputCollection=["MCParticles0"],
 )
 producer1 = ExampleFunctionalProducer(
     "Producer1",
-    OutputCollection="MCParticles1",
+    OutputCollection=["MCParticles1"],
 )
 producer2 = ExampleFunctionalProducer(
     "Producer2",
-    OutputCollection="MCParticles2",
+    OutputCollection=["MCParticles2"],
 )
 consumer = ExampleFunctionalConsumerRuntimeCollections(
     "Consumer",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
index 18d6d329..dbeb62da 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py
@@ -30,32 +30,32 @@
 
 producer0 = ExampleFunctionalProducerMultiple(
     "Producer0",
-    OutputCollectionFloat="VectorFloat0",
-    OutputCollectionParticles1="MCParticles0",
-    OutputCollectionParticles2="MCParticles1",
-    OutputCollectionSimTrackerHits="SimTrackerHits0",
-    OutputCollectionTrackerHits="TrackerHits0",
-    OutputCollectionTracks="Tracks0",
+    OutputCollectionFloat=["VectorFloat0"],
+    OutputCollectionParticles1=["MCParticles0"],
+    OutputCollectionParticles2=["MCParticles1"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits0"],
+    OutputCollectionTrackerHits=["TrackerHits0"],
+    OutputCollectionTracks=["Tracks0"],
     ExampleInt=5,
 )
 producer1 = ExampleFunctionalProducerMultiple(
     "Producer1",
-    OutputCollectionFloat="VectorFloat1",
-    OutputCollectionParticles1="MCParticles2",
-    OutputCollectionParticles2="MCParticles3",
-    OutputCollectionSimTrackerHits="SimTrackerHits1",
-    OutputCollectionTrackerHits="TrackerHits1",
-    OutputCollectionTracks="Tracks1",
+    OutputCollectionFloat=["VectorFloat1"],
+    OutputCollectionParticles1=["MCParticles2"],
+    OutputCollectionParticles2=["MCParticles3"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits1"],
+    OutputCollectionTrackerHits=["TrackerHits1"],
+    OutputCollectionTracks=["Tracks1"],
     ExampleInt=5,
 )
 producer2 = ExampleFunctionalProducerMultiple(
     "Producer2",
-    OutputCollectionFloat="VectorFloat2",
-    OutputCollectionParticles1="MCParticles4",
-    OutputCollectionParticles2="MCParticles5",
-    OutputCollectionSimTrackerHits="SimTrackerHits2",
-    OutputCollectionTrackerHits="TrackerHits2",
-    OutputCollectionTracks="Tracks2",
+    OutputCollectionFloat=["VectorFloat2"],
+    OutputCollectionParticles1=["MCParticles4"],
+    OutputCollectionParticles2=["MCParticles5"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits2"],
+    OutputCollectionTrackerHits=["TrackerHits2"],
+    OutputCollectionTracks=["Tracks2"],
     ExampleInt=5,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
index bcf60a42..18f7475d 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py
@@ -30,7 +30,7 @@
 svc.output = "functional_transformer.root"
 
 transformer = ExampleFunctionalTransformer(
-    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
 )
 
 mgr = ApplicationMgr(
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 218ef7fe..4a5ad644 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -46,7 +46,7 @@
 
 
 transformer = ExampleFunctionalTransformer(
-    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
 )
 
 mgr = ApplicationMgr(
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index e7044b77..42f47c35 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -43,16 +43,19 @@
 )
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+scheduler.ShowDataDependencies = True
+scheduler.ShowDataFlow = True
+scheduler.ShowControlFlow = True
 
 transformer = ExampleFunctionalTransformer(
-    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
 )
 
-producer = ExampleFunctionalProducer("Producer", OutputCollection="MCParticles")
+producer = ExampleFunctionalProducer("Producer", OutputCollection=["MCParticles"])
 
 consumer = ExampleFunctionalConsumer(
     "Consumer",
-    InputCollection="NewMCParticles",
+    InputCollection=["NewMCParticles"],
     Offset=10,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
index 207109e5..c687ae3c 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
@@ -26,14 +26,14 @@
 from Configurables import EventDataSvc
 
 transformer = ExampleFunctionalTransformer(
-    "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles"
+    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
 )
 
-producer = ExampleFunctionalProducer("Producer", OutputCollection="MCParticles")
+producer = ExampleFunctionalProducer("Producer", OutputCollection=["MCParticles"])
 
 consumer = ExampleFunctionalConsumer(
     "Consumer",
-    InputCollection="NewMCParticles",
+    InputCollection=["NewMCParticles"],
     Offset=10,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
index 2773dbbc..2808ecea 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py
@@ -31,13 +31,13 @@
 
 transformer = ExampleFunctionalTransformerMultiple(
     "Transformer",
-    InputCollectionFloat="VectorFloat",
-    InputCollectionParticles="MCParticles1",
-    InputCollectionSimTrackerHits="SimTrackerHits",
-    InputCollectionTrackerHits="TrackerHits",
-    InputCollectionTracks="Tracks",
-    OutputCollectionCounter="Counter",
-    OutputCollectionParticles="NewMCParticles",
+    InputCollectionFloat=["VectorFloat"],
+    InputCollectionParticles=["MCParticles1"],
+    InputCollectionSimTrackerHits=["SimTrackerHits"],
+    InputCollectionTrackerHits=["TrackerHits"],
+    InputCollectionTracks=["Tracks"],
+    OutputCollectionCounter=["Counter"],
+    OutputCollectionParticles=["NewMCParticles"],
     Offset=10,
 )
 
@@ -45,11 +45,11 @@
 
 consumer = ExampleFunctionalConsumerMultiple(
     "Consumer",
-    InputCollectionFloat="VectorFloat",
-    InputCollectionParticles="NewMCParticles",
-    InputCollectionSimTrackerHits="SimTrackerHits",
-    InputCollectionTrackerHits="TrackerHits",
-    InputCollectionTracks="Tracks",
+    InputCollectionFloat=["VectorFloat"],
+    InputCollectionParticles=["NewMCParticles"],
+    InputCollectionSimTrackerHits=["SimTrackerHits"],
+    InputCollectionTrackerHits=["TrackerHits"],
+    InputCollectionTracks=["Tracks"],
     Offset=10,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
index 4a00715f..5ceae857 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py
@@ -32,12 +32,12 @@
 
 producer = ExampleFunctionalProducerMultiple(
     "ExampleFunctionalProducerMultiple",
-    OutputCollectionFloat="VectorFloat",
-    OutputCollectionParticles1="MCParticles1",
-    OutputCollectionParticles2="MCParticles2",
-    OutputCollectionSimTrackerHits="SimTrackerHits",
-    OutputCollectionTrackerHits="TrackerHits",
-    OutputCollectionTracks="Tracks",
+    OutputCollectionFloat=["VectorFloat"],
+    OutputCollectionParticles1=["MCParticles1"],
+    OutputCollectionParticles2=["MCParticles2"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits"],
+    OutputCollectionTrackerHits=["TrackerHits"],
+    OutputCollectionTracks=["Tracks"],
     ExampleInt=5,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
index 90831219..f0ecc472 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py
@@ -35,17 +35,17 @@
 
 consumer0 = ExampleFunctionalConsumer(
     "Consumer0",
-    InputCollection="MCParticles0",
+    InputCollection=["MCParticles0"],
     Offset=0,
 )
 consumer1 = ExampleFunctionalConsumer(
     "Consumer1",
-    InputCollection="MCParticles1",
+    InputCollection=["MCParticles1"],
     Offset=0,
 )
 consumer2 = ExampleFunctionalConsumer(
     "Consumer2",
-    InputCollection="MCParticles2",
+    InputCollection=["MCParticles2"],
     Offset=0,
 )
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
index 1814fc1d..c7868b44 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py
@@ -31,32 +31,32 @@
 
 producer0 = ExampleFunctionalProducerMultiple(
     "Producer0",
-    OutputCollectionFloat="VectorFloat0",
-    OutputCollectionParticles1="MCParticles0",
-    OutputCollectionParticles2="MCParticles1",
-    OutputCollectionSimTrackerHits="SimTrackerHits0",
-    OutputCollectionTrackerHits="TrackerHits0",
-    OutputCollectionTracks="Tracks0",
+    OutputCollectionFloat=["VectorFloat0"],
+    OutputCollectionParticles1=["MCParticles0"],
+    OutputCollectionParticles2=["MCParticles1"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits0"],
+    OutputCollectionTrackerHits=["TrackerHits0"],
+    OutputCollectionTracks=["Tracks0"],
     ExampleInt=5,
 )
 producer1 = ExampleFunctionalProducerMultiple(
     "Producer1",
-    OutputCollectionFloat="VectorFloat1",
-    OutputCollectionParticles1="MCParticles2",
-    OutputCollectionParticles2="MCParticles3",
-    OutputCollectionSimTrackerHits="SimTrackerHits1",
-    OutputCollectionTrackerHits="TrackerHits1",
-    OutputCollectionTracks="Tracks1",
+    OutputCollectionFloat=["VectorFloat1"],
+    OutputCollectionParticles1=["MCParticles2"],
+    OutputCollectionParticles2=["MCParticles3"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits1"],
+    OutputCollectionTrackerHits=["TrackerHits1"],
+    OutputCollectionTracks=["Tracks1"],
     ExampleInt=5,
 )
 producer2 = ExampleFunctionalProducerMultiple(
     "Producer2",
-    OutputCollectionFloat="VectorFloat2",
-    OutputCollectionParticles1="MCParticles4",
-    OutputCollectionParticles2="MCParticles5",
-    OutputCollectionSimTrackerHits="SimTrackerHits2",
-    OutputCollectionTrackerHits="TrackerHits2",
-    OutputCollectionTracks="Tracks2",
+    OutputCollectionFloat=["VectorFloat2"],
+    OutputCollectionParticles1=["MCParticles4"],
+    OutputCollectionParticles2=["MCParticles5"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits2"],
+    OutputCollectionTrackerHits=["TrackerHits2"],
+    OutputCollectionTracks=["Tracks2"],
     ExampleInt=5,
 )
 
@@ -74,13 +74,11 @@
     OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"],
     OutputCollectionParticles1=[
         "NewMCParticles0",
-        "NewMCParticles1",
         "NewMCParticles2",
+        "NewMCParticles4",
     ],
     OutputCollectionParticles2=[
-        "NewMCParticles0",
-        "NewMCParticles1",
-        "NewMCParticles2",
+        "",
     ],
     OutputCollectionSimTrackerHits=[
         "NewSimTrackerHits0",
@@ -98,29 +96,29 @@
 
 consumer0 = ExampleFunctionalConsumerMultiple(
     "Consumer0",
-    InputCollectionFloat="NewVectorFloat0",
-    InputCollectionParticles="NewMCParticles0",
-    InputCollectionSimTrackerHits="NewSimTrackerHits0",
-    InputCollectionTrackerHits="NewTrackerHits0",
-    InputCollectionTracks="NewTracks0",
+    InputCollectionFloat=["NewVectorFloat0"],
+    InputCollectionParticles=["NewMCParticles0"],
+    InputCollectionSimTrackerHits=["NewSimTrackerHits0"],
+    InputCollectionTrackerHits=["NewTrackerHits0"],
+    InputCollectionTracks=["NewTracks0"],
 )
 
 consumer1 = ExampleFunctionalConsumerMultiple(
     "Consumer1",
-    InputCollectionFloat="NewVectorFloat1",
-    InputCollectionParticles="NewMCParticles2",
-    InputCollectionSimTrackerHits="NewSimTrackerHits1",
-    InputCollectionTrackerHits="NewTrackerHits1",
-    InputCollectionTracks="NewTracks1",
+    InputCollectionFloat=["NewVectorFloat1"],
+    InputCollectionParticles=["NewMCParticles2"],
+    InputCollectionSimTrackerHits=["NewSimTrackerHits1"],
+    InputCollectionTrackerHits=["NewTrackerHits1"],
+    InputCollectionTracks=["NewTracks1"],
 )
 
 consumer2 = ExampleFunctionalConsumerMultiple(
     "Consumer2",
-    InputCollectionFloat="NewVectorFloat2",
-    InputCollectionParticles="NewMCParticles4",
-    InputCollectionSimTrackerHits="NewSimTrackerHits2",
-    InputCollectionTrackerHits="NewTrackerHits2",
-    InputCollectionTracks="NewTracks2",
+    InputCollectionFloat=["NewVectorFloat2"],
+    InputCollectionParticles=["NewMCParticles4"],
+    InputCollectionSimTrackerHits=["NewSimTrackerHits2"],
+    InputCollectionTrackerHits=["NewTrackerHits2"],
+    InputCollectionTracks=["NewTracks2"],
 )
 
 
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
index b57fa2a4..129d79ce 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py
@@ -33,19 +33,18 @@
 
 producer0 = ExampleFunctionalProducer(
     "Producer0",
-    OutputCollection="MCParticles0",
+    OutputCollection=["MCParticles0"],
 )
 producer1 = ExampleFunctionalProducer(
     "Producer1",
-    OutputCollection="MCParticles1",
+    OutputCollection=["MCParticles1"],
 )
 producer2 = ExampleFunctionalProducer(
     "Producer2",
-    OutputCollection="MCParticles2",
+    OutputCollection=["MCParticles2"],
 )
 
 transformer = ExampleFunctionalTransformerRuntimeEmpty(
-    "Transformer",
     InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"],
     OutputCollections=[""],
 )

From 70099c7bbaa24871db7d243fb04edce5a275729d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 25 Mar 2024 11:50:13 +0100
Subject: [PATCH 081/127] Remove remaining GaudiAlg headers

---
 k4FWCore/include/k4FWCore/BaseClass.h                          | 2 +-
 test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/BaseClass.h b/k4FWCore/include/k4FWCore/BaseClass.h
index 7032d574..3d679251 100644
--- a/k4FWCore/include/k4FWCore/BaseClass.h
+++ b/k4FWCore/include/k4FWCore/BaseClass.h
@@ -20,7 +20,7 @@
 #ifndef K4FWCORE_FUNCTIONALUTILS_H
 #define K4FWCORE_FUNCTIONALUTILS_H
 
-#include "GaudiAlg/GaudiAlgorithm.h"
+#include "Gaudi/Algorithm.h"
 #include "GaudiKernel/DataObjectHandle.h"
 #include "k4FWCore/DataWrapper.h"
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
index afa2222b..bda86666 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp
@@ -18,7 +18,6 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Producer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 

From 01d675195ae953c7f89625770b2780b3d6080406 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 25 Mar 2024 17:10:41 +0100
Subject: [PATCH 082/127] Make inputs and outputs lists

---
 ...ExampleFunctionalTransformerRuntimeCollections.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
index e2e5a9d4..0f85a361 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py
@@ -31,15 +31,15 @@
 
 producer0 = ExampleFunctionalProducer(
     "Producer0",
-    OutputCollection="MCParticles0",
+    OutputCollection=["MCParticles0"],
 )
 producer1 = ExampleFunctionalProducer(
     "Producer1",
-    OutputCollection="MCParticles1",
+    OutputCollection=["MCParticles1"],
 )
 producer2 = ExampleFunctionalProducer(
     "Producer2",
-    OutputCollection="MCParticles2",
+    OutputCollection=["MCParticles2"],
 )
 
 transformer = ExampleFunctionalTransformerRuntimeCollections(
@@ -50,17 +50,17 @@
 
 consumer0 = ExampleFunctionalConsumer(
     "Consumer0",
-    InputCollection="NewMCParticles0",
+    InputCollection=["NewMCParticles0"],
     Offset=0,
 )
 consumer1 = ExampleFunctionalConsumer(
     "Consumer1",
-    InputCollection="NewMCParticles1",
+    InputCollection=["NewMCParticles1"],
     Offset=0,
 )
 consumer2 = ExampleFunctionalConsumer(
     "Consumer2",
-    InputCollection="NewMCParticles2",
+    InputCollection=["NewMCParticles2"],
     Offset=0,
 )
 

From 9de8b81591dbcdd8646d5a66182519584dfb98aa Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 27 Mar 2024 11:09:54 +0100
Subject: [PATCH 083/127] Add a few comments and a requirement on is_map_like

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 1542d1c4..c453b6e5 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -58,9 +58,14 @@ namespace k4FWCore {
       return static_cast<const T>(*arg);
     }
 
+    // Check if the type is a map like type, where map type is the special map
+    // type to have an arbitrary number of collections as input or output:
+    // std::map<std::string, Coll> where Coll is the collection type
     template <typename T> struct is_map_like : std::false_type {};
 
-    template <typename Value> struct is_map_like<std::map<std::string, Value>> : std::true_type {};
+    template <typename Value>
+    requires std::is_base_of_v<podio::CollectionBase, std::remove_cvref_t<Value>>
+    struct is_map_like<std::map<std::string, Value>> : std::true_type {};
 
     // transformType function to transform the types from the ones that the user wants
     // like edm4hep::MCParticleCollection, to the ones that are actually stored in the

From dbc1b35de46a5c6da16ecf0a704f6c2f1062146b Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 27 Mar 2024 11:18:34 +0100
Subject: [PATCH 084/127] Add asserts to have error messages when using a
 disallowed type

---
 k4FWCore/include/k4FWCore/Consumer.h    |  6 ++++--
 k4FWCore/include/k4FWCore/Transformer.h | 15 ++++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index c9a82b98..e177e1a2 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -41,6 +41,9 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+                    "Consumer input types must be EDM4hep collections or maps to collections");
+
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
@@ -64,8 +67,7 @@ namespace k4FWCore {
                   }
                   std::get<I>(m_inputs) = std::move(h);
                 },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...}
-      {}
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {}
 
       Consumer(std::string name, ISvcLocator* locator,
                Gaudi::Functional::details::RepeatValues_<KeyValues, sizeof...(In)> const& inputs)
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index f5bd8ced..05d2d117 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -41,13 +41,18 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+                    "Transformer and Producer input types must be EDM4hep collections or maps to collections");
+      static_assert((std::is_base_of_v<podio::CollectionBase, Out> || is_map_like<Out>::value),
+                    "Transformer and Producer output types must be EDM4hep collections or maps to collections");
+
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
       template <typename T>
       using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
 
       std::tuple<std::vector<InputHandle_t<typename transformType<In>::type>>...> m_inputs;
-      std::tuple<std::vector<OutputHandle_t<typename transformType<Out>::type>>>           m_outputs;
+      std::tuple<std::vector<OutputHandle_t<typename transformType<Out>::type>>>  m_outputs;
       std::array<Gaudi::Property<std::vector<DataObjID>>, sizeof...(In)>          m_inputLocations{};
       std::array<Gaudi::Property<std::vector<DataObjID>>, 1>                      m_outputLocations{};
 
@@ -115,7 +120,7 @@ namespace k4FWCore {
             // }
           } else {
             Gaudi::Functional::details::put(
-                                            std::get<0>(this->m_outputs)[0],
+                std::get<0>(this->m_outputs)[0],
                 ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
           }
           return Gaudi::Functional::FilterDecision::PASSED;
@@ -136,6 +141,11 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+                    "Transformer and Producer input types must be EDM4hep collections or maps to collections");
+      static_assert(((std::is_base_of_v<podio::CollectionBase, Out> || is_map_like<Out>::value) && ...),
+                    "Transformer and Producer output types must be EDM4hep collections or maps to collections");
+
       template <typename T>
       using InputHandle_t = Gaudi::Functional::details::InputHandle_t<Traits_, std::remove_pointer_t<T>>;
       template <typename T>
@@ -191,7 +201,6 @@ namespace k4FWCore {
           : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for<In...>{}, outputs,
                              std::index_sequence_for<Out...>{}) {}
 
-
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {

From 821b8f31c8c3eddd3f63d5587e7b5afe67af10ab Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 2 Apr 2024 13:42:13 +0200
Subject: [PATCH 085/127] Check the services interfaces in IOSvc.cpp

---
 k4FWCore/components/IOSvc.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index e1a60b18..82e19009 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -46,9 +46,17 @@ StatusCode IOSvc::initialize() {
   m_switch = KeepDropSwitch(m_outputCommands);
 
   m_incidentSvc = service("IncidentSvc");
+  if (!m_incidentSvc) {
+    error() << "Unable to locate IIncidentSvc interface" << endmsg;
+    return StatusCode::FAILURE;
+  }
   m_incidentSvc->addListener(this, IncidentType::EndEvent);
 
   m_dataSvc = service("EventDataSvc");
+  if (!m_dataSvc) {
+    error() << "Unable to locate IDataSvc interface" << endmsg;
+    return StatusCode::FAILURE;
+  }
 
   m_hiveWhiteBoard = service("EventDataSvc");
 

From 42551fd649db33e6f07245638179d02b048931b0 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 2 Apr 2024 13:45:52 +0200
Subject: [PATCH 086/127] Add Service::initialize in IOSvc.cpp

---
 k4FWCore/components/IOSvc.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 82e19009..e4e53fdf 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -32,6 +32,11 @@
 #include <tuple>
 
 StatusCode IOSvc::initialize() {
+  StatusCode sc = Service::initialize();
+  if ( !sc.isSuccess() ) {
+    error() << "Unable to initialize base class Service." << endmsg;
+    return sc;
+  }
   if (!m_readingFileNames.empty()) {
     m_reader = std::make_unique<podio::ROOTReader>();
     try {

From 8b7a162bacdf105313d92ab0a47fa6f111b29974 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 2 Apr 2024 15:39:33 +0200
Subject: [PATCH 087/127] Fix Service::initialize and clean up template
 functions

---
 k4FWCore/components/IOSvc.cpp               |  2 +-
 k4FWCore/include/k4FWCore/Consumer.h        |  2 +-
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 26 +++++++--------------
 k4FWCore/include/k4FWCore/Transformer.h     | 14 +++++------
 4 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index e4e53fdf..52db37a8 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -65,7 +65,7 @@ StatusCode IOSvc::initialize() {
 
   m_hiveWhiteBoard = service("EventDataSvc");
 
-  return Service::initialize();
+  return StatusCode::SUCCESS;
 }
 
 StatusCode IOSvc::finalize() { return Service::finalize(); }
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index e177e1a2..f1d73515 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -41,7 +41,7 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
-      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || isMapToCollLike<In>::value) && ...),
                     "Consumer input types must be EDM4hep collections or maps to collections");
 
       template <typename T>
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index c453b6e5..22c91352 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -61,11 +61,11 @@ namespace k4FWCore {
     // Check if the type is a map like type, where map type is the special map
     // type to have an arbitrary number of collections as input or output:
     // std::map<std::string, Coll> where Coll is the collection type
-    template <typename T> struct is_map_like : std::false_type {};
+    template <typename T> struct isMapToCollLike : std::false_type {};
 
     template <typename Value>
-    requires std::is_base_of_v<podio::CollectionBase, std::remove_cvref_t<Value>>
-    struct is_map_like<std::map<std::string, Value>> : std::true_type {};
+      requires std::is_base_of_v<podio::CollectionBase, std::remove_cvref_t<Value>>
+    struct isMapToCollLike<std::map<std::string, Value>> : std::true_type {};
 
     // transformType function to transform the types from the ones that the user wants
     // like edm4hep::MCParticleCollection, to the ones that are actually stored in the
@@ -76,22 +76,14 @@ namespace k4FWCore {
     };
 
     template <typename T>
-      requires std::is_base_of_v<podio::CollectionBase, T> || is_map_like<T>::value
+      requires std::is_base_of_v<podio::CollectionBase, T> || isMapToCollLike<T>::value
     struct transformType<T> {
       using type = std::shared_ptr<podio::CollectionBase>;
     };
 
-    template <typename T, std::enable_if_t<!std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-    auto ptrOrCast(T&& arg) {
-      // return arg;
+    template <typename T> auto convertToSharedPtr(T&& arg) {
       return std::shared_ptr<podio::CollectionBase>(std::make_shared<T>(std::move(arg)));
     }
-    template <typename T, std::enable_if_t<std::is_same_v<std::shared_ptr<podio::CollectionBase>, T>, int> = 0>
-    auto ptrOrCast(T&& arg) {
-      // return arg;
-      std::cout << "Calling static_cast<const T&>(*arg) (ptrOrCast)" << std::endl;
-      return static_cast<const T&>(*arg);
-    }
 
     template <typename... In> struct filter_evtcontext_tt {
       static_assert(!std::disjunction_v<std::is_same<EventContext, In>...>,
@@ -118,7 +110,7 @@ namespace k4FWCore {
     template <size_t Index, typename... In, typename... Handles, typename InputTuple>
     void readMapInputs(const std::tuple<Handles...>& handles, auto thisClass, InputTuple& inputTuple) {
       if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+        if constexpr (isMapToCollLike<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
           // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
           // we have to remove the reference to get the actual type
           using EDM4hepType =
@@ -143,7 +135,7 @@ namespace k4FWCore {
     template <size_t Index, typename... Out, typename... Handles>
     void putMapOutputs(std::tuple<Handles...>&& handles, const auto& m_outputs, auto thisClass) {
       if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (is_map_like<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
+        if constexpr (isMapToCollLike<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
           int i = 0;
           if (std::get<Index>(handles).size() != std::get<Index>(m_outputs).size()) {
             std::string msg = "Size of the output map " + std::to_string(std::get<Index>(handles).size()) +
@@ -166,12 +158,12 @@ namespace k4FWCore {
                                        std::get<Index>(m_outputs)[i].objKey() + "\"",
                                    StatusCode::FAILURE);
             }
-            Gaudi::Functional::details::put(std::get<Index>(m_outputs)[i], ptrOrCast(std::move(val)));
+            Gaudi::Functional::details::put(std::get<Index>(m_outputs)[i], convertToSharedPtr(std::move(val)));
             i++;
           }
         } else {
           Gaudi::Functional::details::put(std::get<Index>(m_outputs)[0],
-                                          ptrOrCast(std::move(std::get<Index>(handles))));
+                                          convertToSharedPtr(std::move(std::get<Index>(handles))));
         }
 
         // Recursive call for the next index
diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 05d2d117..318ced46 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -41,9 +41,9 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
-      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || isMapToCollLike<In>::value) && ...),
                     "Transformer and Producer input types must be EDM4hep collections or maps to collections");
-      static_assert((std::is_base_of_v<podio::CollectionBase, Out> || is_map_like<Out>::value),
+      static_assert((std::is_base_of_v<podio::CollectionBase, Out> || isMapToCollLike<Out>::value),
                     "Transformer and Producer output types must be EDM4hep collections or maps to collections");
 
       template <typename T>
@@ -104,7 +104,7 @@ namespace k4FWCore {
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          if constexpr (is_map_like<Out>::value) {
+          if constexpr (isMapToCollLike<Out>::value) {
             std::tuple<Out> tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
             // auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
             putMapOutputs<0, Out>(std::move(tmp), m_outputs, this);
@@ -115,13 +115,13 @@ namespace k4FWCore {
             //                          "Output key does not match the expected key " + m_outputs[i].objKey(),
             //                          StatusCode::FAILURE);
             //   }
-            //   Gaudi::Functional::details::put(m_outputs[i], ptrOrCast(std::move(val)));
+            //   Gaudi::Functional::details::put(m_outputs[i], convertToSharedPtr(std::move(val)));
             //   i++;
             // }
           } else {
             Gaudi::Functional::details::put(
                 std::get<0>(this->m_outputs)[0],
-                ptrOrCast(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
+                convertToSharedPtr(std::move(filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs))));
           }
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
@@ -141,9 +141,9 @@ namespace k4FWCore {
         : Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_> {
       using Gaudi::Functional::details::DataHandleMixin<std::tuple<>, std::tuple<>, Traits_>::DataHandleMixin;
 
-      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || is_map_like<In>::value) && ...),
+      static_assert(((std::is_base_of_v<podio::CollectionBase, In> || isMapToCollLike<In>::value) && ...),
                     "Transformer and Producer input types must be EDM4hep collections or maps to collections");
-      static_assert(((std::is_base_of_v<podio::CollectionBase, Out> || is_map_like<Out>::value) && ...),
+      static_assert(((std::is_base_of_v<podio::CollectionBase, Out> || isMapToCollLike<Out>::value) && ...),
                     "Transformer and Producer output types must be EDM4hep collections or maps to collections");
 
       template <typename T>

From e56f68ff164df28956e1b15ab6b0b10d7949fa7a Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 09:06:40 +0200
Subject: [PATCH 088/127] Apply suggestions from code review

Co-authored-by: Andre Sailer <andre.philippe.sailer@cern.ch>
Co-authored-by: Mateusz Jakub Fila <37295697+m-fila@users.noreply.github.com>
---
 k4FWCore/components/IOSvc.cpp          | 4 ++--
 k4FWCore/components/Reader.cpp         | 2 +-
 k4FWCore/components/Writer.cpp         | 4 ++--
 k4FWCore/include/k4FWCore/DataHandle.h | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 52db37a8..3949cce8 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -33,7 +33,7 @@
 
 StatusCode IOSvc::initialize() {
   StatusCode sc = Service::initialize();
-  if ( !sc.isSuccess() ) {
+  if (sc.isFailure()) {
     error() << "Unable to initialize base class Service." << endmsg;
     return sc;
   }
@@ -59,7 +59,7 @@ StatusCode IOSvc::initialize() {
 
   m_dataSvc = service("EventDataSvc");
   if (!m_dataSvc) {
-    error() << "Unable to locate IDataSvc interface" << endmsg;
+    error() << "Unable to locate the EvtDataSvc" << endmsg;
     return StatusCode::FAILURE;
   }
 
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 66849bc4..3f550769 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -61,7 +61,7 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
         if (auto sc = m_dataSvc->registerObject(outputLocations[i], objectp.get()); sc.isFailure()) {
         }
         // The store has the ownership so we shouldn't delete the object
-        auto ptr = objectp.release();
+        objectp.release();
       }
       return Gaudi::Functional::FilterDecision::PASSED;
     } catch (GaudiException& e) {
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 2000f499..e478d82d 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -37,7 +37,7 @@
 class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&)> {
 public:
   Writer(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) {
-    // Non-reeentrant algorithms have a cardinality of 1
+    // Non-reentrant algorithms have a cardinality of 1
     setProperty("Cardinality", 1).ignore();
   }
 
@@ -62,7 +62,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
     m_dataSvc = service("EventDataSvc", true);
     if (!m_dataSvc) {
-      error() << "Unable to locate IDataSvc interface" << endmsg;
+      error() << "Unable to locate EventDataSvc service" << endmsg;
       return StatusCode::FAILURE;
     }
 
diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h
index 53fc7c67..37ada6f1 100644
--- a/k4FWCore/include/k4FWCore/DataHandle.h
+++ b/k4FWCore/include/k4FWCore/DataHandle.h
@@ -132,7 +132,7 @@ template <typename T> const T* DataHandle<T>::get() {
       DataWrapper<podio::CollectionBase>* tmp = static_cast<DataWrapper<podio::CollectionBase>*>(dataObjectp);
       return reinterpret_cast<const T*>(tmp->collectionBase());
     } else {
-      // When a functional has pushed an std::shared_ptr<podio::CollectionBase> into the store
+      // When a functional has pushed a std::shared_ptr<podio::CollectionBase> into the store
       auto ptr = static_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(dataObjectp)->getData();
       if (ptr) {
         return reinterpret_cast<const T*>(ptr.get());

From 0256dad308df0f028cbb203699f3e29444adb6cd Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 09:11:53 +0200
Subject: [PATCH 089/127] Apply suggestions for Writer.cpp

Co-authored-by: Andre Sailer <andre.philippe.sailer@cern.ch>
---
 k4FWCore/components/Writer.cpp | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index e478d82d..439b8b32 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -162,7 +162,6 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     // more than one instance is created
     std::scoped_lock<std::mutex> lock(m_mutex);
 
-    StatusCode code;
 
     if (m_hiveWhiteBoard) {
       // It's never set to valid but it has the slot information
@@ -171,19 +170,18 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       //   return;
       // }
       info() << "Setting store to " << ctx.slot() << endmsg;
-      code = m_hiveWhiteBoard->selectStore(ctx.slot());
-      if (code.isFailure()) {
+      if (m_hiveWhiteBoard->selectStore(ctx.slot()).isFailure()) {
         error() << "Error when setting store" << endmsg;
         throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
       }
     }
 
     DataObject* p;
-    code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
+    SatusCode code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
     AnyDataWrapper<podio::Frame>* ptr;
     // This is the case when we are reading from a file
     if (code.isSuccess()) {
-      code = m_dataSvc->unregisterObject(p);
+      m_dataSvc->unregisterObject(p).ignore();
       ptr  = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
     }
     // This is the case when no reading is being done
@@ -213,14 +211,12 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
     for (auto& coll : ptr->getData().getAvailableCollections()) {
       DataObject* storeCollection;
-      code = m_dataSvc->retrieveObject("/Event/" + coll, storeCollection);
-      if (code.isFailure()) {
+      if (m_dataSvc->retrieveObject("/Event/" + coll, storeCollection).isFailure()) {
         error() << "Failed to retrieve collection " << coll << endmsg;
         return;
       }
       // We take ownership back from the store
-      code = m_dataSvc->unregisterObject(storeCollection);
-      if (code.isFailure()) {
+      if (m_dataSvc->unregisterObject(storeCollection).isFailure()) {
         error() << "Failed to unregister collection " << coll << endmsg;
         return;
       }
@@ -228,14 +224,12 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
     for (auto& coll : m_collectionsToAdd) {
       DataObject* storeCollection;
-      code = m_dataSvc->retrieveObject("/Event/" + coll, storeCollection);
-      if (code.isFailure()) {
+      if (m_dataSvc->retrieveObject("/Event/" + coll, storeCollection).isFailure()) {
         error() << "Failed to retrieve collection " << coll << endmsg;
         return;
       }
       // We take ownership back from the store
-      code = m_dataSvc->unregisterObject(storeCollection);
-      if (code.isFailure()) {
+      if (m_dataSvc->unregisterObject(storeCollection).isFailure()) {
         error() << "Failed to unregister collection " << coll << endmsg;
         return;
       }

From 197db779318317ea439a55c2660719b4920e5fc1 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 5 Apr 2024 15:21:30 +0200
Subject: [PATCH 090/127] Allow combining functionals together with old
 algorithms

---
 k4FWCore/components/Writer.cpp                |  4 +--
 k4FWCore/include/k4FWCore/FunctionalUtils.h   | 31 +++++++++++++++++--
 test/k4FWCoreTest/CMakeLists.txt              |  1 +
 test/k4FWCoreTest/options/runFunctionalMix.py | 21 +++++++------
 .../k4FWCoreTest_CheckExampleEventData.cpp    | 13 ++++----
 .../k4FWCoreTest_CreateExampleEventData.cpp   |  4 +--
 6 files changed, 53 insertions(+), 21 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 439b8b32..9e2f6846 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -236,12 +236,12 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       const auto collection = dynamic_cast<AnyDataWrapper<std::shared_ptr<podio::CollectionBase>>*>(storeCollection);
       if (!collection) {
         // Check the case when the data has been produced using the old DataHandle
-        const auto old_collection = dynamic_cast<DataWrapper<podio::CollectionBase>*>(storeCollection);
+        const auto old_collection = dynamic_cast<DataWrapperBase*>(storeCollection);
         if (!old_collection) {
           error() << "Failed to cast collection " << coll << endmsg;
           return;
         } else {
-          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->getData()));
+          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->collectionBase()));
           ptr->getData().put(std::move(uptr), coll);
         }
 
diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 22c91352..117ad013 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -19,13 +19,18 @@
 #ifndef FWCORE_FUNCTIONALUTILS_H
 #define FWCORE_FUNCTIONALUTILS_H
 
+#include <GaudiKernel/AnyDataWrapper.h>
+#include <GaudiKernel/IDataProviderSvc.h>
 #include "Gaudi/Functional/details.h"
 #include "GaudiKernel/DataObjID.h"
+#include "k4FWCore/DataWrapper.h"
 #include "podio/CollectionBase.h"
 
 #include "GaudiKernel/EventContext.h"
 #include "GaudiKernel/ThreadLocalContext.h"
 
+// #include "GaudiKernel/CommonMessaging.h"
+
 #include <map>
 #include <memory>
 #include <tuple>
@@ -123,8 +128,30 @@ namespace k4FWCore {
           std::get<Index>(inputTuple) = std::move(inputMap);
 
         } else {
-          auto in                     = get(std::get<Index>(handles)[0], thisClass, Gaudi::Hive::currentContext());
-          std::get<Index>(inputTuple) = static_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(in.get());
+          try {
+            auto in                     = get(std::get<Index>(handles)[0], thisClass, Gaudi::Hive::currentContext());
+            std::get<Index>(inputTuple) = static_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(in.get());
+          } catch (GaudiException& e) {
+            // When the type of the collection is different from the one requested, this can happen because
+            // 1. a mistake was made in the input types of a functional algorithm
+            // 2. the data was produced using the old DataHandle, which is never going to be in the input type
+            if (e.message().find("different from") != std::string::npos) {
+              thisClass->error() << "Trying to cast the collection " << std::get<Index>(handles)[0].objKey()
+                                 << " to the requested type " << thisClass->name() << endmsg;
+              DataObject*       p;
+              IDataProviderSvc* svc = thisClass->serviceLocator()->template service<IDataProviderSvc>("EventDataSvc");
+              svc->retrieveObject("/Event/" + std::get<Index>(handles)[0].objKey(), p).ignore();
+              const auto wrp = dynamic_cast<const DataWrapper<std::tuple_element_t<Index, std::tuple<In...>>>*>(p);
+              if (!wrp) {
+                throw GaudiException(thisClass->name(),
+                                     "Failed to cast collection " + std::get<Index>(handles)[0].objKey(),
+                                     StatusCode::FAILURE);
+              }
+              std::get<Index>(inputTuple) = const_cast<std::tuple_element_t<Index, std::tuple<In...>>*>(wrp->getData());
+            } else {
+              throw e;
+            }
+          }
         }
 
         // Recursive call for the next index
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index cb1fada5..435c04e3 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -120,6 +120,7 @@ add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalMixIOSvc options/runFunctionalMixIOSvc.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
 add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py
index 42a909ab..daedfb7f 100644
--- a/test/k4FWCoreTest/options/runFunctionalMix.py
+++ b/test/k4FWCoreTest/options/runFunctionalMix.py
@@ -48,7 +48,8 @@
 ]
 
 consumer_input_functional = ExampleFunctionalConsumerMultiple(
-    "ExampleFunctionalConsumerMultiple"
+    "ExampleFunctionalConsumerMultiple",
+    Offset=0,
 )
 consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData")
 consumer_input_algorithm.mcparticles = "MCParticles1"
@@ -57,18 +58,19 @@
 # We only care about the new FunctionalMCParticles collection in this example
 producer_functional = ExampleFunctionalProducerMultiple(
     "ProducerFunctional",
-    OutputCollectionFloat="VectorFloat_",
-    OutputCollectionParticles1="FunctionalMCParticles",
-    OutputCollectionParticles2="MCParticles2_",
-    OutputCollectionSimTrackerHits="SimTrackerHits_",
-    OutputCollectionTrackerHits="TrackerHits_",
-    OutputCollectionTracks="Tracks_",
+    OutputCollectionFloat=["VectorFloat_"],
+    OutputCollectionParticles1=["FunctionalMCParticles"],
+    OutputCollectionParticles2=["MCParticles2_"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits_"],
+    OutputCollectionTrackerHits=["TrackerHits_"],
+    OutputCollectionTracks=["Tracks_"],
     ExampleInt=5,
 )
 
 consumer_producerfun_functional = ExampleFunctionalConsumerMultiple(
     "FunctionalConsumerFunctional",
-    InputCollectionParticles="FunctionalMCParticles",
+    InputCollectionParticles=["FunctionalMCParticles"],
+    Offset=0,
 )
 consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional")
 consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles"
@@ -83,7 +85,8 @@
 producer_algorithm.vectorfloat = "VectorFloat__"
 
 consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerAlgorithm"
+    "FunctionalConsumerAlgorithm",
+    Offset=0,
 )
 consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
 consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
index f53615cb..fda76d93 100644
--- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
+++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
@@ -55,12 +55,13 @@ StatusCode k4FWCoreTest_CheckExampleEventData::execute(const EventContext&) cons
 
   auto particles = m_mcParticleHandle.get();
   auto particle  = (*particles)[0];
-  if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 5) ||
-      (particle.getMass() != m_magicNumberOffset + m_event + 8)) {
-    fatal() << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x
-            << " (expected " << m_magicNumberOffset + m_event + 5 << "), mass = " << particle.getMass() << " (expected "
-            << m_magicNumberOffset + m_event + 8 << ")" << endmsg;
-    // return StatusCode::FAILURE;
+  if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 0) ||
+      (particle.getMass() != m_magicNumberOffset + m_event + 6)) {
+    std::stringstream error;
+    error << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x
+          << " (expected " << m_magicNumberOffset + m_event + 0 << "), mass = " << particle.getMass() << " (expected "
+          << m_magicNumberOffset + m_event + 6 << ")";
+    throw std::runtime_error(error.str());
   }
 
   if (!m_keepEventNumberZero) {
diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
index 36e0aa84..a11140db 100644
--- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
+++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
@@ -64,8 +64,8 @@ StatusCode k4FWCoreTest_CreateExampleEventData::execute(const EventContext&) con
   edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut();
 
   auto particle = particles->create();
-  particle.setMomentum({m_magicNumberOffset + m_event + 5.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0});
-  particle.setMass(m_magicNumberOffset + m_event + 8);
+  particle.setMomentum({m_magicNumberOffset + m_event + 0.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0});
+  particle.setMass(m_magicNumberOffset + m_event + 6);
 
   edm4hep::SimTrackerHitCollection* simTrackerHits = m_simTrackerHitHandle.createAndPut();
   auto                              hit            = simTrackerHits->create();

From 7029ffd54cfde6d833a40f76eb91f5817fc1ec2b Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 15 Apr 2024 17:40:55 +0200
Subject: [PATCH 091/127] Remove unused ctx

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 117ad013..37234d1f 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -100,7 +100,7 @@ namespace k4FWCore {
       }
 
       template <typename Algorithm, typename Handles>
-      static auto apply(const Algorithm& algo, const EventContext& ctx, Handles& handles) {
+      static auto apply(const Algorithm& algo, const EventContext&, Handles& handles) {
         auto inputTuple = std::tuple<addPtrIfColl<In>...>();
 
         // Build the input tuple by picking up either std::map with an arbitrary

From 97b1c3cb54ffe75197ccf4533374c6dd2ea36d81 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 09:14:03 +0200
Subject: [PATCH 092/127] Fix typo

---
 k4FWCore/components/Writer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 9e2f6846..bafc6d80 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -177,7 +177,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     }
 
     DataObject* p;
-    SatusCode code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
+    StatusCode code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
     AnyDataWrapper<podio::Frame>* ptr;
     // This is the case when we are reading from a file
     if (code.isSuccess()) {

From d9a58e7d21e91d4390d7be9b0805c4abfe053e11 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 11:30:02 +0200
Subject: [PATCH 093/127] Address a few comments in the PR

---
 k4FWCore/components/IIOSvc.h         |  6 +++++-
 k4FWCore/components/IOSvc.cpp        |  3 ++-
 k4FWCore/components/IOSvc.h          |  3 +--
 k4FWCore/components/Writer.cpp       |  6 +++---
 k4FWCore/include/k4FWCore/Consumer.h | 10 +++++++---
 test/k4FWCoreTest/CMakeLists.txt     |  6 +++---
 6 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 6f56c9da..3465a445 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -40,6 +40,10 @@ class IIOSvc : virtual public IInterface {
   /// InterfaceID
   DeclareInterfaceID(IIOSvc, 1, 0);
 
+  /**
+   * @brief Read the next event from the input file
+   * @return A tuple containing the collections read, the collection names and the frame that owns the collections
+   */
   virtual std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std::string>, podio::Frame>
                                                     next()                     = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
@@ -47,7 +51,7 @@ class IIOSvc : virtual public IInterface {
   virtual std::shared_ptr<podio::ROOTWriter> getWriter()                                  = 0;
   virtual void                               deleteWriter()                               = 0;
   virtual void                               deleteReader()                               = 0;
-  virtual bool                               writeCollection(const std::string& collName) = 0;
+  virtual bool                               checkIfWriteCollection(const std::string& collName) = 0;
 };
 
 #endif
diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 3949cce8..dee6a1a4 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -111,6 +111,7 @@ std::tuple<std::vector<std::shared_ptr<podio::CollectionBase>>, std::vector<std:
 // After every event if there is still a frame in the TES
 // that means it hasn't been written so the collections inside the Frame
 // should be removed so that they are deleted when the Frame is deleted
+// and not deleted when clearing the store
 void IOSvc::handle(const Incident& incident) {
   StatusCode code;
   if (m_hiveWhiteBoard) {
@@ -153,6 +154,6 @@ void IOSvc::setReadingCollectionNames(const std::vector<std::string>& names) { m
 
 void IOSvc::setReadingFileNames(const std::vector<std::string>& names) { m_readingFileNames = names; }
 
-bool IOSvc::writeCollection(const std::string& collName) { return m_switch.isOn(collName); }
+bool IOSvc::checkIfWriteCollection(const std::string& collName) { return m_switch.isOn(collName); }
 
 DECLARE_COMPONENT(IOSvc)
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index d823bf4b..4bef027e 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -30,7 +30,6 @@
 #include "podio/ROOTWriter.h"
 
 #include "k4FWCore/KeepDropSwitch.h"
-
 #include "IIOSvc.h"
 
 #include <string>
@@ -95,7 +94,7 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   int m_entries{0};
   int m_nextEntry{0};
 
-  bool writeCollection(const std::string& collName) override;
+  bool checkIfWriteCollection(const std::string& collName) override;
 };
 
 #endif
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index bafc6d80..4f0d6983 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -122,13 +122,13 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
 
     SmartDataPtr<DataObject> root(eventSvc(), "/Event");
     if (!root) {
-      info() << "Failed to retrieve root object /Event" << endmsg;
+      error() << "Failed to retrieve root object /Event" << endmsg;
       return;
     }
 
     auto pObj = root->registry();
     if (!pObj) {
-      error() << "Failed to retrieve root object" << endmsg;
+      error() << "Failed to retrieve the root registry object" << endmsg;
       return;
     }
     auto mgr = eventSvc().as<IDataManagerSvc>();
@@ -195,7 +195,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       // and cache them
       getOutputCollections();
       for (const auto& coll : m_availableCollections) {
-        if (iosvc->writeCollection(coll)) {
+        if (iosvc->checkIfWriteCollection(coll)) {
           m_collectionsToSave.push_back(coll);
           const auto& frameCollections = ptr->getData().getAvailableCollections();
           if (std::find(frameCollections.begin(), frameCollections.end(), coll) == frameCollections.end()) {
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index f1d73515..3a430dd5 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -57,15 +57,19 @@ namespace k4FWCore {
       template <typename IArgs, std::size_t... I>
       Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>)
           : base_class(std::move(name), locator),
+            // The input locations are filled by creating a property with a callback function
+            // that creates the handles because that is when the input locations become available
+            // (from a steering file, for example) and the handles have to be created for
+            // Gaudi to know the data flow
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
                 this, std::get<I>(inputs).first, to_DataObjID(std::get<I>(inputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
-                  std::vector<InputHandle_t<typename transformType<In>::type>> h;
+                  std::vector<InputHandle_t<typename transformType<In>::type>> handles;
                   for (auto& value : this->m_inputLocations[I].value()) {
                     auto handle = InputHandle_t<typename transformType<In>::type>(value, this);
-                    h.push_back(std::move(handle));
+                    handles.push_back(std::move(handle));
                   }
-                  std::get<I>(m_inputs) = std::move(h);
+                  std::get<I>(m_inputs) = std::move(handles);
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {}
 
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 435c04e3..90f5c993 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -14,7 +14,6 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
 ]]
 
 file(GLOB k4fwcoretest_plugin_sources src/components/*.cpp)
@@ -116,8 +115,6 @@ add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMu
 add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py)
 add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS FunctionalProducer)
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
-add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalProducer PASS_REGULAR_EXPRESSION
-                  "Application Manager Terminated successfully with a user requested ScheduledStop")
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMixIOSvc options/runFunctionalMixIOSvc.py PROPERTIES DEPENDS FunctionalProducerMultiple)
@@ -130,6 +127,9 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
+# Do this after checking the files not to overwrite them
+add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
+                  "Application Manager Terminated successfully with a user requested ScheduledStop")
 
 # The following is done to make the tests work without installing the files in
 # the installation directory. The k4FWCore in the build directory is populated by

From 9b1cf054d9c3419edf685345dcf67f32fcac69de Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 15:37:43 +0200
Subject: [PATCH 094/127] Add error when a frame can't be pushed

---
 k4FWCore/components/Reader.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 3f550769..f2c30792 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -119,8 +119,10 @@ class Reader final : public CollectionPusher {
     auto eds   = eventSvc().as<IDataProviderSvc>();
     auto frame = std::move(std::get<podio::Frame>(val));
 
-    auto tmp  = new AnyDataWrapper<podio::Frame>(std::move(frame));
-    auto code = eds->registerObject("/Event" + k4FWCore::frameLocation, tmp);
+    auto tmp = new AnyDataWrapper<podio::Frame>(std::move(frame));
+    if (eds->registerObject("/Event" + k4FWCore::frameLocation, tmp).isFailure()) {
+      error() << "Failed to register Frame object" << endmsg;
+    }
 
     return std::make_tuple(std::get<0>(val), std::get<1>(val));
   }

From 35cbb47f625606c3d61babfd370c61eb43d6e04d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 15:49:58 +0200
Subject: [PATCH 095/127] Fix properties docstrings

---
 k4FWCore/components/IOSvc.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index 4bef027e..b4ea458f 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -60,9 +60,9 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
       "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
   Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
 
-  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of files to read"};
+  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of collections to read"};
   Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
-  Gaudi::Property<std::string>              m_writingFileName{this, "output", {}, "List of files to read"};
+  Gaudi::Property<std::string>              m_writingFileName{this, "output", {}, "List of files to write output to"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
       this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
   Gaudi::Property<std::string> m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"};

From 797dfe30a682f6ac7139f79037c3075e50a67a71 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:05:06 +0200
Subject: [PATCH 096/127] Use isMapToCollLike_v

---
 k4FWCore/include/k4FWCore/FunctionalUtils.h | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h
index 37234d1f..61aab1aa 100644
--- a/k4FWCore/include/k4FWCore/FunctionalUtils.h
+++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h
@@ -72,6 +72,8 @@ namespace k4FWCore {
       requires std::is_base_of_v<podio::CollectionBase, std::remove_cvref_t<Value>>
     struct isMapToCollLike<std::map<std::string, Value>> : std::true_type {};
 
+    template <class T> inline constexpr bool isMapToCollLike_v = isMapToCollLike<T>::value;
+
     // transformType function to transform the types from the ones that the user wants
     // like edm4hep::MCParticleCollection, to the ones that are actually stored in the
     // event store, like std::shared_ptr<podio::CollectionBase>
@@ -81,7 +83,7 @@ namespace k4FWCore {
     };
 
     template <typename T>
-      requires std::is_base_of_v<podio::CollectionBase, T> || isMapToCollLike<T>::value
+      requires std::is_base_of_v<podio::CollectionBase, T> || isMapToCollLike_v<T>
     struct transformType<T> {
       using type = std::shared_ptr<podio::CollectionBase>;
     };
@@ -115,7 +117,7 @@ namespace k4FWCore {
     template <size_t Index, typename... In, typename... Handles, typename InputTuple>
     void readMapInputs(const std::tuple<Handles...>& handles, auto thisClass, InputTuple& inputTuple) {
       if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (isMapToCollLike<std::tuple_element_t<Index, std::tuple<In...>>>::value) {
+        if constexpr (isMapToCollLike_v<std::tuple_element_t<Index, std::tuple<In...>>>) {
           // In case of map types like std::map<std::string, edm4hep::MCParticleCollection&>
           // we have to remove the reference to get the actual type
           using EDM4hepType =
@@ -162,7 +164,7 @@ namespace k4FWCore {
     template <size_t Index, typename... Out, typename... Handles>
     void putMapOutputs(std::tuple<Handles...>&& handles, const auto& m_outputs, auto thisClass) {
       if constexpr (Index < sizeof...(Handles)) {
-        if constexpr (isMapToCollLike<std::tuple_element_t<Index, std::tuple<Out...>>>::value) {
+        if constexpr (isMapToCollLike_v<std::tuple_element_t<Index, std::tuple<Out...>>>) {
           int i = 0;
           if (std::get<Index>(handles).size() != std::get<Index>(m_outputs).size()) {
             std::string msg = "Size of the output map " + std::to_string(std::get<Index>(handles).size()) +

From be69179aa2c40fe34ed958716d0ebda0aabf5123 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:55:21 +0200
Subject: [PATCH 097/127] Return StatusCode::FAILURE instead of throwing

---
 k4FWCore/components/IOSvc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index dee6a1a4..4583020a 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -43,7 +43,7 @@ StatusCode IOSvc::initialize() {
       m_reader->openFiles(m_readingFileNames);
     } catch (std::runtime_error& e) {
       error() << "Error when opening files: " << e.what() << endmsg;
-      throw e;
+      return StatusCode::FAILURE;
     }
     m_entries = m_reader->getEntries(podio::Category::Event);
   }

From 936638edd323356ecdb9dacf44619d9385ec0686 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:12:50 +0200
Subject: [PATCH 098/127] Add an error message

---
 k4FWCore/components/Reader.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index f2c30792..176cf340 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -58,10 +58,11 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
       // }
       for (unsigned i = 0; i != outColls.size(); ++i) {
         auto objectp = std::make_unique<AnyDataWrapper<Out>>(std::move(outColls[i]));
-        if (auto sc = m_dataSvc->registerObject(outputLocations[i], objectp.get()); sc.isFailure()) {
+        if (m_dataSvc->registerObject(outputLocations[i], objectp.get()).isFailure()) {
+          error() << "Failed to register object at " << outputLocations[i] << endmsg;
         }
         // The store has the ownership so we shouldn't delete the object
-        objectp.release();
+        (void) objectp.release();
       }
       return Gaudi::Functional::FilterDecision::PASSED;
     } catch (GaudiException& e) {

From d6de2e241483d687e5800698a5fb553ab9fa1d5b Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:46:20 +0200
Subject: [PATCH 099/127] Format a few files

---
 k4FWCore/components/IIOSvc.h                       |  6 +++---
 k4FWCore/components/IOSvc.h                        |  5 +++--
 k4FWCore/components/Reader.cpp                     |  2 +-
 k4FWCore/components/Writer.cpp                     | 12 ++++++------
 ...unctionalConsumerRuntimeCollectionsMultiple.cpp | 13 +++++++------
 ...mpleFunctionalTransformerRuntimeCollections.cpp |  3 ++-
 ...tionalTransformerRuntimeCollectionsMultiple.cpp | 14 ++++++--------
 .../ExampleFunctionalTransformerRuntimeEmpty.cpp   |  3 ++-
 8 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h
index 3465a445..d5721e0a 100644
--- a/k4FWCore/components/IIOSvc.h
+++ b/k4FWCore/components/IIOSvc.h
@@ -48,9 +48,9 @@ class IIOSvc : virtual public IInterface {
                                                     next()                     = 0;
   virtual std::shared_ptr<std::vector<std::string>> getCollectionNames() const = 0;
 
-  virtual std::shared_ptr<podio::ROOTWriter> getWriter()                                  = 0;
-  virtual void                               deleteWriter()                               = 0;
-  virtual void                               deleteReader()                               = 0;
+  virtual std::shared_ptr<podio::ROOTWriter> getWriter()                                         = 0;
+  virtual void                               deleteWriter()                                      = 0;
+  virtual void                               deleteReader()                                      = 0;
   virtual bool                               checkIfWriteCollection(const std::string& collName) = 0;
 };
 
diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index b4ea458f..fd19861f 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -29,8 +29,8 @@
 #include "podio/ROOTReader.h"
 #include "podio/ROOTWriter.h"
 
-#include "k4FWCore/KeepDropSwitch.h"
 #include "IIOSvc.h"
+#include "k4FWCore/KeepDropSwitch.h"
 
 #include <string>
 #include <vector>
@@ -60,7 +60,8 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
       "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
   Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
 
-  Gaudi::Property<std::vector<std::string>> m_collectionNames{this, "CollectionNames", {}, "List of collections to read"};
+  Gaudi::Property<std::vector<std::string>> m_collectionNames{
+      this, "CollectionNames", {}, "List of collections to read"};
   Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
   Gaudi::Property<std::string>              m_writingFileName{this, "output", {}, "List of files to write output to"};
   Gaudi::Property<std::vector<std::string>> m_outputCommands{
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 176cf340..22ed16b9 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -62,7 +62,7 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
           error() << "Failed to register object at " << outputLocations[i] << endmsg;
         }
         // The store has the ownership so we shouldn't delete the object
-        (void) objectp.release();
+        (void)objectp.release();
       }
       return Gaudi::Functional::FilterDecision::PASSED;
     } catch (GaudiException& e) {
diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 4f0d6983..519ab096 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -66,7 +66,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       return StatusCode::FAILURE;
     }
 
-    m_hiveWhiteBoard = service<IHiveWhiteBoard>("EventDataSvc", true);
+    m_hiveWhiteBoard = service("EventDataSvc", true);
     if (!m_hiveWhiteBoard) {
       debug() << "Unable to locate IHiveWhiteBoard interface. This isn't a problem if we are not running in a "
                  "multi-threaded environment"
@@ -162,7 +162,6 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     // more than one instance is created
     std::scoped_lock<std::mutex> lock(m_mutex);
 
-
     if (m_hiveWhiteBoard) {
       // It's never set to valid but it has the slot information
       // if (ctx.valid()) {
@@ -176,13 +175,13 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       }
     }
 
-    DataObject* p;
-    StatusCode code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
+    DataObject*                   p;
+    StatusCode                    code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
     AnyDataWrapper<podio::Frame>* ptr;
     // This is the case when we are reading from a file
     if (code.isSuccess()) {
       m_dataSvc->unregisterObject(p).ignore();
-      ptr  = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+      ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
     }
     // This is the case when no reading is being done
     // needs to be fixed? (new without delete)
@@ -241,7 +240,8 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
           error() << "Failed to cast collection " << coll << endmsg;
           return;
         } else {
-          std::unique_ptr<podio::CollectionBase> uptr(const_cast<podio::CollectionBase*>(old_collection->collectionBase()));
+          std::unique_ptr<podio::CollectionBase> uptr(
+              const_cast<podio::CollectionBase*>(old_collection->collectionBase()));
           ptr->getData().put(std::move(uptr), coll);
         }
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
index 233812d8..5b2ca067 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp
@@ -29,8 +29,8 @@
 
 struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
     : k4FWCore::Consumer<void(const std::map<std::string, const edm4hep::MCParticleCollection&>& particleMap,
-                              const std::map<std::string, const edm4hep::TrackCollection&>&         trackMap,
-                              const edm4hep::SimTrackerHitCollection& simTrackerHits)> {
+                              const std::map<std::string, const edm4hep::TrackCollection&>&      trackMap,
+                              const edm4hep::SimTrackerHitCollection&                            simTrackerHits)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalConsumerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc)
@@ -41,15 +41,16 @@ struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final
 
   // This is the function that will be called to produce the data
   void operator()(const std::map<std::string, const edm4hep::MCParticleCollection&>& particleMap,
-                  const std::map<std::string, const edm4hep::TrackCollection&>&         trackMap,
-                  const edm4hep::SimTrackerHitCollection& simTrackerHits) const override {
+                  const std::map<std::string, const edm4hep::TrackCollection&>&      trackMap,
+                  const edm4hep::SimTrackerHitCollection&                            simTrackerHits) const override {
     info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size()
            << " track collections" << endmsg;
     if (particleMap.size() != 5) {
-      throw std::runtime_error("Wrong size of the particleMap map, expected 5, got " + std::to_string(particleMap.size()));
+      throw std::runtime_error("Wrong size of the particleMap map, expected 5, got " +
+                               std::to_string(particleMap.size()));
     }
     for (auto& [key, particles] : particleMap) {
-      int         i         = 0;
+      int i = 0;
       for (const auto& particle : particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
index 7ae662a6..23e07962 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp
@@ -34,7 +34,8 @@
 
 using mapType = std::map<std::string, edm4hep::MCParticleCollection>;
 
-struct ExampleFunctionalTransformerRuntimeCollections final : k4FWCore::Transformer<mapType(const std::map<std::string, const edm4hep::MCParticleCollection&>& input)> {
+struct ExampleFunctionalTransformerRuntimeCollections final
+    : k4FWCore::Transformer<mapType(const std::map<std::string, const edm4hep::MCParticleCollection&>& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
index 53255d99..0cf79e30 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp
@@ -21,8 +21,8 @@
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/SimTrackerHitCollection.h"
-#include "edm4hep/TrackerHit3DCollection.h"
 #include "edm4hep/TrackCollection.h"
+#include "edm4hep/TrackerHit3DCollection.h"
 #include "podio/UserDataCollection.h"
 
 #include "k4FWCore/Transformer.h"
@@ -37,12 +37,10 @@ using SimTrackerHitColl = std::map<std::string, const edm4hep::SimTrackerHitColl
 using TrackerHitColl    = std::map<std::string, const edm4hep::TrackerHit3DCollection&>;
 using TrackColl         = std::map<std::string, const edm4hep::TrackCollection&>;
 
-using retType = std::tuple<std::map<std::string, podio::UserDataCollection<float>>,
-                           std::map<std::string, edm4hep::MCParticleCollection>,
-                           std::map<std::string, edm4hep::MCParticleCollection>,
-                           std::map<std::string, edm4hep::SimTrackerHitCollection>,
-                           std::map<std::string, edm4hep::TrackerHit3DCollection>,
-                           std::map<std::string, edm4hep::TrackCollection>>;
+using retType = std::tuple<
+    std::map<std::string, podio::UserDataCollection<float>>, std::map<std::string, edm4hep::MCParticleCollection>,
+    std::map<std::string, edm4hep::MCParticleCollection>, std::map<std::string, edm4hep::SimTrackerHitCollection>,
+    std::map<std::string, edm4hep::TrackerHit3DCollection>, std::map<std::string, edm4hep::TrackCollection>>;
 
 struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
     : k4FWCore::MultiTransformer<retType(const FloatColl&, const ParticleColl&, const SimTrackerHitColl&,
@@ -109,7 +107,7 @@ struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final
 
     for (auto& [key, particles] : particlesMap) {
       auto coll = edm4hep::MCParticleCollection();
-      int  i   = 0;
+      int  i    = 0;
       for (const auto& particle : particles) {
         if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) ||
             (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) ||
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
index c3d8dc96..f5c0ba86 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp
@@ -34,7 +34,8 @@
 
 using mapType = std::map<std::string, const edm4hep::MCParticleCollection&>;
 
-struct ExampleFunctionalTransformerRuntimeEmpty final : k4FWCore::Transformer<std::map<std::string, edm4hep::MCParticleCollection>(const mapType& input)> {
+struct ExampleFunctionalTransformerRuntimeEmpty final
+    : k4FWCore::Transformer<std::map<std::string, edm4hep::MCParticleCollection>(const mapType& input)> {
   // The pair in KeyValue can be changed from python and it corresponds
   // to the name of the output collection
   ExampleFunctionalTransformerRuntimeEmpty(const std::string& name, ISvcLocator* svcLoc)

From 9357347e8408a7e77ccd11a0864bcbb9f2943f69 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:31:13 +0200
Subject: [PATCH 100/127] Do a bit of cleanup and small fixes

---
 k4FWCore/components/IOSvc.h                   |  5 --
 python/k4FWCore/IOSvc.py                      |  2 +-
 .../ExampleFunctionalConsumerMemory.py        | 44 --------------
 ...ExampleFunctionalConsumerMultipleMemory.py | 59 -------------------
 .../options/ExampleFunctionalProducer.py      |  5 +-
 .../options/runEventHeaderCheck.py            |  2 +-
 6 files changed, 6 insertions(+), 111 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
 delete mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py

diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h
index fd19861f..b77025d1 100644
--- a/k4FWCore/components/IOSvc.h
+++ b/k4FWCore/components/IOSvc.h
@@ -55,11 +55,6 @@ class IOSvc : public extends<Service, IIOSvc, IIncidentListener> {
   void setReadingFileNames(const std::vector<std::string>& names);
 
 protected:
-  Gaudi::Property<unsigned int> m_bufferNbEvents{
-      this, "BufferNbEvents", 20000,
-      "approximate size of the buffer used to prefetch rawbanks in terms of number of events. Default is 20000"};
-  Gaudi::Property<unsigned int> m_nbSkippedEvents{this, "NSkip", 0, "First event to process"};
-
   Gaudi::Property<std::vector<std::string>> m_collectionNames{
       this, "CollectionNames", {}, "List of collections to read"};
   Gaudi::Property<std::vector<std::string>> m_readingFileNames{this, "input", {}, "List of files to read"};
diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
index b927713a..65550ea2 100644
--- a/python/k4FWCore/IOSvc.py
+++ b/python/k4FWCore/IOSvc.py
@@ -19,6 +19,7 @@
 from Configurables import IOSvc as IO
 import os
 
+
 class IOSvc:
 
     def __init__(self, *args, **kwargs):
@@ -29,7 +30,6 @@ def __getattr__(self, attr):
         return getattr(self._svc, attr)
 
     def __setattr__(self, attr, value):
-        print(f'IOSvc: {attr} {value}')
         if attr == '_svc':
             super().__setattr__(attr, value)
             return
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
deleted file mode 100644
index 559e1ccc..00000000
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMemory.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example reading from a file and using a consumer with a single input
-# to check that the contents of the file are the expected ones
-
-from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalProducer, ExampleFunctionalConsumer
-from Configurables import EventDataSvc
-from k4FWCore import ApplicationMgr
-
-producer = ExampleFunctionalProducer(
-    "ExampleFunctionalProducer",
-    OutputCollection="MCParticles",
-)
-
-consumer = ExampleFunctionalConsumer(
-    "ExampleFunctionalConsumer",
-    InputCollection="MCParticles",
-)
-
-ApplicationMgr(
-    TopAlg=[producer, consumer],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[EventDataSvc("EventDataSvc")],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
deleted file mode 100644
index 3ea8afe9..00000000
--- a/test/k4FWCoreTest/options/ExampleFunctionalConsumerMultipleMemory.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example reading from a file and using a consumer with a single input
-# to check that the contents of the file are the expected ones
-
-from Gaudi.Configuration import INFO
-from Configurables import (
-    ExampleFunctionalProducerMultiple,
-    ExampleFunctionalConsumerMultiple,
-)
-from Configurables import EventDataSvc
-from k4FWCore import ApplicationMgr
-
-producer = ExampleFunctionalProducerMultiple(
-    "Producer",
-    OutputCollectionFloat="VectorFloat",
-    OutputCollectionParticles1="MCParticles1",
-    OutputCollectionParticles2="MCParticles2",
-    OutputCollectionSimTrackerHits="SimTrackerHits",
-    OutputCollectionTrackerHits="TrackerHits",
-    OutputCollectionTracks="Tracks",
-    ExampleInt=5,
-)
-
-consumer = ExampleFunctionalConsumerMultiple(
-    "Consumer",
-    InputCollectionFloat="VectorFloat",
-    InputCollectionParticles1="MCParticles1",
-    InputCollectionParticles2="MCParticles2",
-    InputCollectionSimTrackerHits="SimTrackerHits",
-    InputCollectionTrackerHits="TrackerHits",
-    InputCollectionTracks="Tracks",
-    ExampleInt=5,
-)
-
-ApplicationMgr(
-    TopAlg=[producer, consumer],
-    EvtSel="NONE",
-    EvtMax=10,
-    ExtSvc=[EventDataSvc("EventDataSvc")],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
index ae0a2584..ec9ab87c 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
@@ -23,6 +23,7 @@
 from Configurables import ExampleFunctionalProducer
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
+from Configurables import Writer
 
 
 iosvc = IOSvc("IOSvc")
@@ -33,8 +34,10 @@
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 
+writer = Writer("Writer")
+
 ApplicationMgr(
-    TopAlg=[producer],
+    TopAlg=[producer, writer],
     EvtSel="NONE",
     EvtMax=10,
     ExtSvc=[EventDataSvc("EventDataSvc")],
diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py
index d0fa63ea..b2c5ef5f 100644
--- a/test/k4FWCoreTest/options/runEventHeaderCheck.py
+++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py
@@ -20,7 +20,7 @@
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleEventHeaderConsumer
-from Configurables import ApplicationMgr
+from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc, IOSvc, Reader
 
 svc = IOSvc("IOSvc")

From ad7b8c91a040559c816dd0ee948858cac99745ff Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:56:14 +0200
Subject: [PATCH 101/127] Add a few comments to the Transformer

---
 k4FWCore/include/k4FWCore/Transformer.h | 27 +++++++++++--------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h
index 318ced46..f5030ddd 100644
--- a/k4FWCore/include/k4FWCore/Transformer.h
+++ b/k4FWCore/include/k4FWCore/Transformer.h
@@ -64,6 +64,11 @@ namespace k4FWCore {
       Transformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence<I...>,
                   const OArgs& outputs, std::index_sequence<J...>)
           : base_class(std::move(name), locator),
+            // The input locations are filled by creating a property with a
+            // callback function that creates the handles because when the
+            // callback runs is when the input locations become available (from
+            // a steering file, for example) and the handles have to be created
+            // for Gaudi to know the data flow
             m_inputLocations{Gaudi::Property<std::vector<DataObjID>>{
                 this, std::get<I>(inputs).first, to_DataObjID(std::get<I>(inputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
@@ -75,12 +80,14 @@ namespace k4FWCore {
                   std::get<I>(m_inputs) = std::move(h);
                 },
                 Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...},
+            // Same as above for the output locations
             m_outputLocations{Gaudi::Property<std::vector<DataObjID>>{
                 this, std::get<J>(outputs).first, to_DataObjID(std::get<J>(outputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
                   std::vector<OutputHandle_t<typename transformType<Out>::type>> h;
-                  std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
-                            [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
+                  // Is this needed?
+                  // std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
+                  //           [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
                   for (auto& inpID : this->m_outputLocations[J].value()) {
                     if (inpID.key().empty()) {
                       continue;
@@ -106,18 +113,7 @@ namespace k4FWCore {
         try {
           if constexpr (isMapToCollLike<Out>::value) {
             std::tuple<Out> tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
-            // auto tmp = filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs);
             putMapOutputs<0, Out>(std::move(tmp), m_outputs, this);
-            // int i = 0;
-            // for (auto& [key, val] : filter_evtcontext_tt<In...>::apply(*this, ctx, this->m_inputs)) {
-            //   if (key != m_outputs[i].objKey()) {
-            //     throw GaudiException("Transformer",
-            //                          "Output key does not match the expected key " + m_outputs[i].objKey(),
-            //                          StatusCode::FAILURE);
-            //   }
-            //   Gaudi::Functional::details::put(m_outputs[i], convertToSharedPtr(std::move(val)));
-            //   i++;
-            // }
           } else {
             Gaudi::Functional::details::put(
                 std::get<0>(this->m_outputs)[0],
@@ -179,8 +175,9 @@ namespace k4FWCore {
                 this, std::get<J>(outputs).first, to_DataObjID(std::get<J>(outputs).second),
                 [this](Gaudi::Details::PropertyBase&) {
                   std::vector<OutputHandle_t<typename transformType<Out>::type>> h;
-                  std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
-                            [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
+                  // Is this needed?
+                  // std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(),
+                  //           [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); });
                   for (auto& inpID : this->m_outputLocations[J].value()) {
                     if (inpID.key().empty()) {
                       continue;

From fc5d25c06080216bc1fb3d43986ab1cd48cae9ba Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:56:48 +0200
Subject: [PATCH 102/127] Open a file to get the collections before running

---
 k4FWCore/components/Reader.cpp    | 38 ++++++++++++++++++++++++-------
 python/k4FWCore/ApplicationMgr.py | 30 ++++++++++++++++++++----
 2 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 22ed16b9..c8831184 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <GaudiKernel/SmartIF.h>
 #include "Gaudi/Functional/details.h"
 #include "Gaudi/Functional/utilities.h"
 #include "GaudiKernel/AnyDataWrapper.h"
@@ -25,6 +26,7 @@
 
 #include "podio/CollectionBase.h"
 #include "podio/Frame.h"
+#include "podio/ROOTReader.h"
 
 #include "IIOSvc.h"
 #include "k4FWCore/FunctionalUtils.h"
@@ -40,8 +42,33 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
   using base_class = Gaudi::Functional::details::BaseClass_t<Traits_>;
   static_assert(std::is_base_of_v<Algorithm, base_class>, "BaseClass must inherit from Algorithm");
 
+  template <typename T>
+  using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
+  std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>> m_outputs;
+  Gaudi::Property<std::string>                                        m_input{this, "Input", "Event", "Input file"};
+
 public:
-  CollectionPusher(std::string name, ISvcLocator* locator) : base_class(std::move(name), locator) {}
+  CollectionPusher(std::string name, ISvcLocator* locator)
+      : base_class(std::move(name), locator),
+        m_input{this, "Input", "Event",
+                [this](Gaudi::Details::PropertyBase& b) {
+                  const std::string cmd = System::cmdLineArgs()[0];
+                  if (cmd.find("genconf") != std::string::npos) {
+                    return;
+                  }
+                  if (m_input.value() == "Event") {
+                    return;
+                  }
+                  auto reader = podio::ROOTReader();
+                  reader.openFile(m_input.value());
+                  auto frame = podio::Frame(reader.readNextEntry(podio::Category::Event));
+                  auto colls = frame.getAvailableCollections();
+
+                  for (auto& c : colls) {
+                      m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
+                  }
+                },
+                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}} {}
 
   // derived classes can NOT implement execute
   StatusCode execute(const EventContext&) const override final {
@@ -56,13 +83,8 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
       //                            " containers, got " + std::to_string(out.size()) + " instead",
       //                        this->name(), StatusCode::FAILURE);
       // }
-      for (unsigned i = 0; i != outColls.size(); ++i) {
-        auto objectp = std::make_unique<AnyDataWrapper<Out>>(std::move(outColls[i]));
-        if (m_dataSvc->registerObject(outputLocations[i], objectp.get()).isFailure()) {
-          error() << "Failed to register object at " << outputLocations[i] << endmsg;
-        }
-        // The store has the ownership so we shouldn't delete the object
-        (void)objectp.release();
+      for (size_t i = 0; i != outColls.size(); ++i) {
+        m_outputs[i].put(std::move(outColls[i]));
       }
       return Gaudi::Functional::FilterDecision::PASSED;
     } catch (GaudiException& e) {
diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 7e4ca9ea..bd36fc5a 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -18,7 +18,8 @@
 #
 from Configurables import ApplicationMgr as AppMgr
 from Configurables import Reader, Writer, IOSvc
-from Configurables import Gaudi__Sequencer
+import os
+
 
 class ApplicationMgr:
 
@@ -28,7 +29,26 @@ def __init__(self, **kwargs):
         for conf in frozenset(self._mgr.allConfigurables.values()):
             if isinstance(conf, IOSvc):
                 props = conf.getPropertiesWithDescription()
-                if 'input' in props:
-                    self._mgr.TopAlg =  [Reader("Reader")] + self._mgr.TopAlg
-                if 'output' in props:
-                    self._mgr.TopAlg = [Gaudi__Sequencer("Sequencer", Members=[*self._mgr.TopAlg, Writer("Writer")], Sequential=True)]
+                reader = writer = None
+                add_reader = add_writer = False
+                for alg in self._mgr.TopAlg:
+                    if isinstance(alg, Reader):
+                        reader = alg
+                    elif isinstance(alg, Writer):
+                        writer = alg
+                if reader is None and props['input'][0]:
+                    reader = Reader("k4FWCore__Reader")
+                    add_reader = True
+                # It seems for a single string the default without a value is '<no value>'
+                # while for a list it's an empty list
+                if writer is None and props['output'][0] and props['output'][0] != '<no value>':
+                    writer = Writer("k4FWCore__Writer")
+                    add_writer = True
+                # Let's tell the Reader one of the input files so it can
+                # know which collections it's going to read
+                if reader is not None:
+                    if os.path.exists(props['input'][0][0]):
+                        reader.Input = props['input'][0][0]
+                    else:
+                        reader.Input = os.getcwd() + '/' + props['input'][0][0]
+                self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg + ([writer] if add_writer else [])

From a40f42ed07e86b5b1e4d3911655c94fb61d9cf8a Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:57:10 +0200
Subject: [PATCH 103/127] Change one test

---
 .../options/ExampleFunctionalMTFile.py         | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 4a5ad644..4da1f385 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -21,12 +21,12 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO, WARNING
-from Configurables import ExampleFunctionalTransformer
+from Configurables import ExampleFunctionalTransformer, ExampleFunctionalConsumer
 from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc
 from k4FWCore import ApplicationMgr, IOSvc
 
-evtslots = 2
-threads = 2
+evtslots = 3
+threads = 3
 
 whiteboard = HiveWhiteBoard(
     "EventDataSvc",
@@ -38,19 +38,25 @@
     "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
 )
 
-scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True,
+                                  OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
-svc.input = "output_k4test_exampledata_producer.root"
+svc.input = "output_k4test_exampledata_producer_multiple.root"
 svc.output = "functional_transformerMT.root"
 
+consumer1 = ExampleFunctionalConsumer(
+    "Consumer1",
+    InputCollection=["MCParticles1"],
+    Offset=0,
+)
 
 transformer = ExampleFunctionalTransformer(
     "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
 )
 
 mgr = ApplicationMgr(
-    TopAlg=[transformer],
+    TopAlg=[consumer1, transformer],
     EvtSel="NONE",
     EvtMax=-1,
     ExtSvc=[whiteboard],

From c14b2307cb37327fdf1444894684ebc0576f076a Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 23 Apr 2024 12:25:48 +0200
Subject: [PATCH 104/127] Remove the remaining GaudiAlgs

---
 .../src/components/ExampleFunctionalProducerMultiple.cpp         | 1 -
 .../k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp | 1 -
 2 files changed, 2 deletions(-)

diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
index 3c8520f9..40ef1331 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp
@@ -18,7 +18,6 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Producer.h"
 
 #include "k4FWCore/Producer.h"
 
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
index c4233151..54946940 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp
@@ -18,7 +18,6 @@
  */
 
 #include "Gaudi/Property.h"
-#include "GaudiAlg/Transformer.h"
 
 #include "edm4hep/MCParticleCollection.h"
 #include "edm4hep/MutableMCParticle.h"

From 4f7433b32b29cd9bf69185c64d2580eed2395081 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 25 Apr 2024 19:24:17 +0200
Subject: [PATCH 105/127] Change EvtDataSvc to EventDataSvc

---
 k4FWCore/components/IOSvc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 4583020a..52b3bdf3 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -59,7 +59,7 @@ StatusCode IOSvc::initialize() {
 
   m_dataSvc = service("EventDataSvc");
   if (!m_dataSvc) {
-    error() << "Unable to locate the EvtDataSvc" << endmsg;
+    error() << "Unable to locate the EventDataSvc" << endmsg;
     return StatusCode::FAILURE;
   }
 

From ade9bbd623396f445a676373fccd5b8d426c0940 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 25 Apr 2024 20:46:13 +0200
Subject: [PATCH 106/127] Add missing test

---
 .../options/runFunctionalMixIOSvc.py          | 118 ++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 test/k4FWCoreTest/options/runFunctionalMixIOSvc.py

diff --git a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
new file mode 100644
index 00000000..97011704
--- /dev/null
+++ b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
@@ -0,0 +1,118 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example mixing functional and non-functional algorithms
+#
+
+from Gaudi.Configuration import INFO
+from Configurables import (
+    ExampleFunctionalConsumerMultiple,
+    ExampleFunctionalTransformerMultiple,
+)
+from Configurables import (
+    ExampleFunctionalProducerMultiple,
+    k4FWCoreTest_CreateExampleEventData,
+)
+from Configurables import k4FWCoreTest_CheckExampleEventData
+from k4FWCore import ApplicationMgr, IOSvc
+
+iosvc = IOSvc("IOSvc")
+iosvc.input = "output_k4test_exampledata_producer_multiple.root"
+
+# inp.collections = [
+#     "VectorFloat",
+#     "MCParticles1",
+#     "MCParticles2",
+#     "SimTrackerHits",
+#     "TrackerHits",
+#     "Tracks",
+# ]
+
+consumer_input_functional = ExampleFunctionalConsumerMultiple(
+    "ExampleFunctionalConsumerMultiple",
+    Offset=0,
+)
+consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData")
+consumer_input_algorithm.mcparticles = "MCParticles1"
+consumer_input_algorithm.keepEventNumberZero = True
+
+# We only care about the new FunctionalMCParticles collection in this example
+producer_functional = ExampleFunctionalProducerMultiple(
+    "ProducerFunctional",
+    OutputCollectionFloat=["VectorFloat_"],
+    OutputCollectionParticles1=["FunctionalMCParticles"],
+    OutputCollectionParticles2=["MCParticles2_"],
+    OutputCollectionSimTrackerHits=["SimTrackerHits_"],
+    OutputCollectionTrackerHits=["TrackerHits_"],
+    OutputCollectionTracks=["Tracks_"],
+    ExampleInt=5,
+)
+
+consumer_producerfun_functional = ExampleFunctionalConsumerMultiple(
+    "FunctionalConsumerFunctional",
+    InputCollectionParticles=["FunctionalMCParticles"],
+    Offset=0,
+)
+consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional")
+consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles"
+consumer_producerfun_algorithm.keepEventNumberZero = True
+
+producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData")
+# We only care about the MCParticles collection
+producer_algorithm.mcparticles = "AlgorithmMCParticles"
+producer_algorithm.simtrackhits = "SimTrackerHits__"
+producer_algorithm.trackhits = "TrackerHits__"
+producer_algorithm.tracks = "Tracks__"
+producer_algorithm.vectorfloat = "VectorFloat__"
+
+consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
+    "FunctionalConsumerAlgorithm",
+    Offset=0,
+)
+consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
+consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
+consumer_produceralg_algorithm.keepEventNumberZero = True
+
+# Let's also run the transformer, why not
+transformer_functional = ExampleFunctionalTransformerMultiple(
+    "FunctionalTransformerMultiple"
+)
+
+iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root"
+
+ApplicationMgr(
+    TopAlg=[
+        # Check we can read input
+        consumer_input_functional,
+        consumer_input_algorithm,
+        producer_functional,
+        # Check we can read what's produced by a functional
+        consumer_producerfun_functional,
+        consumer_producerfun_algorithm,
+        producer_algorithm,
+        # Check we can read what's produced by an algorithm
+        consumer_produceralg_functional,
+        consumer_produceralg_algorithm,
+        transformer_functional,
+    ],
+    EvtSel="NONE",
+    EvtMax=-1,
+    ExtSvc=[iosvc],
+    OutputLevel=INFO,
+)

From 11e68e34fa056eccdf8f68cc4e9184f325d4f465 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Thu, 25 Apr 2024 20:58:02 +0200
Subject: [PATCH 107/127] Fix list of tests

---
 test/k4FWCoreTest/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 90f5c993..4f2764f1 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -126,7 +126,7 @@ add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctio
 add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTransformerRuntimeEmpty.py)
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
-add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS FunctionalFile FunctionalMultipleFile FunctionalOutputCommands FunctionalProducerAbsolutePath)
+add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS "FunctionalFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")

From 969672afe7bef968edd877b1474790ff2a1235e3 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 26 Apr 2024 08:11:44 +0200
Subject: [PATCH 108/127] Fix test (finally)

---
 test/k4FWCoreTest/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 4f2764f1..9544d67f 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -126,7 +126,8 @@ add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctio
 add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTransformerRuntimeEmpty.py)
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
-add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py PROPERTIES DEPENDS "FunctionalFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath")
+add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
+set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")

From 4c04e8f68af3677e5d21004654ced2d98ac763d0 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Fri, 26 Apr 2024 08:12:34 +0200
Subject: [PATCH 109/127] Fix style

---
 python/k4FWCore/ApplicationMgr.py               | 17 ++++++++++-------
 python/k4FWCore/IOSvc.py                        |  8 +++-----
 test/k4FWCoreTest/CMakeLists.txt                |  1 +
 test/k4FWCoreTest/options/CheckOutputFiles.py   |  4 +++-
 .../options/ExampleFunctionalMTFile.py          |  3 +--
 .../options/ExampleFunctionalMTMemory.py        |  4 +---
 .../options/ExampleFunctionalMemory.py          |  6 +++++-
 .../k4FWCoreTest/options/runEventHeaderCheck.py |  4 +---
 test/k4FWCoreTest/options/runFunctionalMix.py   |  4 +---
 .../options/runFunctionalMixIOSvc.py            |  4 +---
 10 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index bd36fc5a..3830b9e6 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -22,7 +22,6 @@
 
 
 class ApplicationMgr:
-
     def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
 
@@ -36,19 +35,23 @@ def __init__(self, **kwargs):
                         reader = alg
                     elif isinstance(alg, Writer):
                         writer = alg
-                if reader is None and props['input'][0]:
+                if reader is None and props["input"][0]:
                     reader = Reader("k4FWCore__Reader")
                     add_reader = True
                 # It seems for a single string the default without a value is '<no value>'
                 # while for a list it's an empty list
-                if writer is None and props['output'][0] and props['output'][0] != '<no value>':
+                if writer is None and props["output"][0] and props["output"][0] != "<no value>":
                     writer = Writer("k4FWCore__Writer")
                     add_writer = True
                 # Let's tell the Reader one of the input files so it can
                 # know which collections it's going to read
                 if reader is not None:
-                    if os.path.exists(props['input'][0][0]):
-                        reader.Input = props['input'][0][0]
+                    if os.path.exists(props["input"][0][0]):
+                        reader.Input = props["input"][0][0]
                     else:
-                        reader.Input = os.getcwd() + '/' + props['input'][0][0]
-                self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg + ([writer] if add_writer else [])
+                        reader.Input = os.getcwd() + "/" + props["input"][0][0]
+                self._mgr.TopAlg = (
+                    ([reader] if add_reader else [])
+                    + self._mgr.TopAlg
+                    + ([writer] if add_writer else [])
+                )
diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
index 65550ea2..db40ef52 100644
--- a/python/k4FWCore/IOSvc.py
+++ b/python/k4FWCore/IOSvc.py
@@ -21,23 +21,21 @@
 
 
 class IOSvc:
-
     def __init__(self, *args, **kwargs):
-
         self._svc = IO(**kwargs)
 
     def __getattr__(self, attr):
         return getattr(self._svc, attr)
 
     def __setattr__(self, attr, value):
-        if attr == '_svc':
+        if attr == "_svc":
             super().__setattr__(attr, value)
             return
 
-        if attr == 'input':
+        if attr == "input":
             if isinstance(value, str):
                 value = [value]
-        if attr == 'output':
+        if attr == "output":
             if os.path.dirname(value) and not os.path.exists(os.path.dirname(value)):
                 os.makedirs(os.path.dirname(value))
         setattr(self._svc, attr, value)
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 9544d67f..124ba22e 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -14,6 +14,7 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
+limitations under the License.
 ]]
 
 file(GLOB k4fwcoretest_plugin_sources src/components/*.cpp)
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index 1c9dcf5b..f3ae0166 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -63,4 +63,6 @@ def check_collections(filename, names):
     ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"],
 )
 check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
-check_collections("functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"])
+check_collections(
+    "functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"]
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 4da1f385..33164555 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -38,8 +38,7 @@
     "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
 )
 
-scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True,
-                                  OutputLevel=WARNING)
+scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True, OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
 svc.input = "output_k4test_exampledata_producer_multiple.root"
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
index 42f47c35..4d11bcb0 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py
@@ -38,9 +38,7 @@
     ForceLeaves=True,
 )
 
-slimeventloopmgr = HiveSlimEventLoopMgr(
-    SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING
-)
+slimeventloopmgr = HiveSlimEventLoopMgr(SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING)
 
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING)
 scheduler.ShowDataDependencies = True
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
index c687ae3c..c79f1090 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py
@@ -21,7 +21,11 @@
 # to check that the contents of the file are the expected ones
 
 from Gaudi.Configuration import INFO
-from Configurables import ExampleFunctionalTransformer, ExampleFunctionalProducer, ExampleFunctionalConsumer
+from Configurables import (
+    ExampleFunctionalTransformer,
+    ExampleFunctionalProducer,
+    ExampleFunctionalConsumer,
+)
 from k4FWCore import ApplicationMgr
 from Configurables import EventDataSvc
 
diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py
index b2c5ef5f..d15eefb3 100644
--- a/test/k4FWCoreTest/options/runEventHeaderCheck.py
+++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py
@@ -29,9 +29,7 @@
 
 reader = Reader("Reader")
 
-consumer = ExampleEventHeaderConsumer(
-    "EventHeaderCheck", runNumber=42, eventNumberOffset=42
-)
+consumer = ExampleEventHeaderConsumer("EventHeaderCheck", runNumber=42, eventNumberOffset=42)
 
 ApplicationMgr(
     TopAlg=[reader, consumer],
diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py
index daedfb7f..70c9f648 100644
--- a/test/k4FWCoreTest/options/runFunctionalMix.py
+++ b/test/k4FWCoreTest/options/runFunctionalMix.py
@@ -93,9 +93,7 @@
 consumer_produceralg_algorithm.keepEventNumberZero = True
 
 # Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple(
-    "FunctionalTransformerMultiple"
-)
+transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
 
 out = PodioOutput("out")
 out.filename = "output_k4test_exampledata_functional_mix.root"
diff --git a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
index 97011704..0a4581ab 100644
--- a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
+++ b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
@@ -90,9 +90,7 @@
 consumer_produceralg_algorithm.keepEventNumberZero = True
 
 # Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple(
-    "FunctionalTransformerMultiple"
-)
+transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
 
 iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root"
 

From 3a4e2163b764ce2f9fef01c08a8c5313f4bcfe55 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 08:46:37 +0200
Subject: [PATCH 110/127] Fix parrallel running with a Sequencer, add tests and
 clean up

---
 k4FWCore/components/IOSvc.cpp                 |  4 +-
 k4FWCore/components/Reader.cpp                | 68 +++++++++------
 k4FWCore/include/k4FWCore/Consumer.h          |  1 -
 k4FWCore/scripts/k4run                        |  2 +-
 python/k4FWCore/ApplicationMgr.py             | 82 ++++++++++++-------
 test/k4FWCoreTest/CMakeLists.txt              |  4 +-
 test/k4FWCoreTest/options/CheckOutputFiles.py | 12 +++
 .../options/ExampleFunctionalFileMultiple.py  |  2 +-
 .../options/ExampleFunctionalMTFile.py        | 12 +--
 .../ExampleFunctionalOutputCommands.py        |  2 +-
 .../options/ExampleFunctionalProducer.py      | 12 ++-
 .../ExampleFunctionalSeveralInputFiles.py     | 39 +++++++++
 12 files changed, 174 insertions(+), 66 deletions(-)
 create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py

diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp
index 52b3bdf3..b88858bd 100644
--- a/k4FWCore/components/IOSvc.cpp
+++ b/k4FWCore/components/IOSvc.cpp
@@ -119,7 +119,7 @@ void IOSvc::handle(const Incident& incident) {
       info() << "No context found in IOSvc" << endmsg;
       return;
     }
-    info() << "Setting store to " << incident.context().slot() << endmsg;
+    debug() << "Setting store to " << incident.context().slot() << endmsg;
     code = m_hiveWhiteBoard->selectStore(incident.context().slot());
     if (code.isFailure()) {
       error() << "Error when setting store" << endmsg;
@@ -141,7 +141,7 @@ void IOSvc::handle(const Incident& incident) {
     DataObject* collPtr;
     code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr);
     if (code.isSuccess()) {
-      info() << "Removing collection: " << coll << endmsg;
+      debug() << "Removing the collection: " << coll << " from the store" << endmsg;
       code = m_dataSvc->unregisterObject(collPtr);
     }
     // else {
diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index c8831184..378170d8 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -33,8 +33,7 @@
 
 #include <memory>
 
-template <typename Container> using vector_of_          = std::vector<Container>;
-template <typename Container> using vector_of_optional_ = std::vector<std::optional<Container>>;
+template <typename Container> using vector_of_ = std::vector<Container>;
 
 class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::Functional::Traits::useDefaults> {
   using Traits_    = Gaudi::Functional::Traits::useDefaults;
@@ -45,30 +44,49 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
   template <typename T>
   using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t<Traits_, std::remove_pointer_t<T>>;
   std::vector<OutputHandle_t<std::shared_ptr<podio::CollectionBase>>> m_outputs;
-  Gaudi::Property<std::string>                                        m_input{this, "Input", "Event", "Input file"};
+  Gaudi::Property<std::vector<std::string>>                           m_inputCollections{
+      this, "InputCollections", {"First collection"}, "List of input collections"};
+  // Gaudi::Property<std::string>                                        m_input{this, "Input", "Event", "Input file"};
 
 public:
   CollectionPusher(std::string name, ISvcLocator* locator)
       : base_class(std::move(name), locator),
-        m_input{this, "Input", "Event",
-                [this](Gaudi::Details::PropertyBase& b) {
-                  const std::string cmd = System::cmdLineArgs()[0];
-                  if (cmd.find("genconf") != std::string::npos) {
-                    return;
-                  }
-                  if (m_input.value() == "Event") {
-                    return;
-                  }
-                  auto reader = podio::ROOTReader();
-                  reader.openFile(m_input.value());
-                  auto frame = podio::Frame(reader.readNextEntry(podio::Category::Event));
-                  auto colls = frame.getAvailableCollections();
-
-                  for (auto& c : colls) {
-                      m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
-                  }
-                },
-                Gaudi::Details::Property::ImmediatelyInvokeHandler{true}} {}
+        // m_input{this, "Input", "Event",
+        //         [this](Gaudi::Details::PropertyBase& b) {
+        //           const std::string cmd = System::cmdLineArgs()[0];
+        //           if (cmd.find("genconf") != std::string::npos) {
+        //             return;
+        //           }
+        //           if (m_input.value() == "Event") {
+        //             return;
+        //           }
+        //           auto reader = podio::ROOTReader();
+        //           reader.openFile(m_input.value());
+        //           auto frame = podio::Frame(reader.readNextEntry(podio::Category::Event));
+        //           auto colls = frame.getAvailableCollections();
+
+        //           for (auto& c : colls) {
+        //               m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
+        //           }
+        //         },
+        //         Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}
+        m_inputCollections{this,
+                           "InputCollections",
+                           {"Event"},
+                           [this](Gaudi::Details::PropertyBase& b) {
+                             const std::string cmd = System::cmdLineArgs()[0];
+                             if (cmd.find("genconf") != std::string::npos) {
+                               return;
+                             }
+                             if (m_inputCollections.value().size() == 1 && m_inputCollections.value()[0] == "Event") {
+                               return;
+                             }
+
+                             for (auto& c : m_inputCollections.value()) {
+                               m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
+                             }
+                           },
+                           Gaudi::Details::Property::ImmediatelyInvokeHandler{true}} {}
 
   // derived classes can NOT implement execute
   StatusCode execute(const EventContext&) const override final {
@@ -107,7 +125,11 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
 
 class Reader final : public CollectionPusher {
 public:
-  Reader(const std::string& name, ISvcLocator* svcLoc) : CollectionPusher(name, svcLoc) {}
+  Reader(const std::string& name, ISvcLocator* svcLoc) : CollectionPusher(name, svcLoc) {
+    setProperty("Cardinality", 1).ignore();
+  }
+
+  bool isReEntrant() const override { return false; }
 
   // Gaudi doesn't run the destructor of the Services so we have to
   // manually ask for the reader to be deleted so it will call finish()
diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h
index 3a430dd5..5a9911bf 100644
--- a/k4FWCore/include/k4FWCore/Consumer.h
+++ b/k4FWCore/include/k4FWCore/Consumer.h
@@ -80,7 +80,6 @@ namespace k4FWCore {
       // derived classes are NOT allowed to implement execute ...
       StatusCode execute(const EventContext& ctx) const override final {
         try {
-          Gaudi::Algorithm::info() << "Executing " << this->name() << endmsg;
           filter_evtcontext_tt<In...>::apply(*this, ctx, m_inputs);
           return Gaudi::Functional::FilterDecision::PASSED;
         } catch (GaudiException& e) {
diff --git a/k4FWCore/scripts/k4run b/k4FWCore/scripts/k4run
index 36ed60ee..aee31945 100755
--- a/k4FWCore/scripts/k4run
+++ b/k4FWCore/scripts/k4run
@@ -232,7 +232,7 @@ def main():
         # Do the real processing
         retcode = c.run(opts.gdb)
         # User requested stop returns non-zero exit code see: https://github.com/key4hep/k4FWCore/issues/125
-        if ApplicationMgr().EvtMax == -1 and retcode == 4:
+        if retcode == 4:
             retcode = 0
         sys.exit(retcode)
 
diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 3830b9e6..a0b4b383 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -17,8 +17,9 @@
 # limitations under the License.
 #
 from Configurables import ApplicationMgr as AppMgr
-from Configurables import Reader, Writer, IOSvc
+from Configurables import Reader, Writer, IOSvc, Gaudi__Sequencer
 import os
+from podio.root_io import Reader as PodioReader
 
 
 class ApplicationMgr:
@@ -26,32 +27,57 @@ def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
 
         for conf in frozenset(self._mgr.allConfigurables.values()):
-            if isinstance(conf, IOSvc):
-                props = conf.getPropertiesWithDescription()
-                reader = writer = None
-                add_reader = add_writer = False
-                for alg in self._mgr.TopAlg:
-                    if isinstance(alg, Reader):
-                        reader = alg
-                    elif isinstance(alg, Writer):
-                        writer = alg
-                if reader is None and props["input"][0]:
-                    reader = Reader("k4FWCore__Reader")
-                    add_reader = True
-                # It seems for a single string the default without a value is '<no value>'
-                # while for a list it's an empty list
-                if writer is None and props["output"][0] and props["output"][0] != "<no value>":
-                    writer = Writer("k4FWCore__Writer")
-                    add_writer = True
-                # Let's tell the Reader one of the input files so it can
-                # know which collections it's going to read
-                if reader is not None:
+            # import pdb; pdb.set_trace()
+            if not isinstance(conf, IOSvc):
+                continue
+            props = conf.getPropertiesWithDescription()
+            reader = writer = None
+            add_reader = add_writer = False
+            for alg in self._mgr.TopAlg:
+                if isinstance(alg, Reader):
+                    reader = alg
+                elif isinstance(alg, Writer):
+                    writer = alg
+            if reader is None and props["input"][0]:
+                reader = Reader("k4FWCore__Reader")
+                add_reader = True
+            # It seems for a single string the default without a value is '<no value>'
+            # while for a list it's an empty list
+            if writer is None and props["output"][0] and props["output"][0] != "<no value>":
+                writer = Writer("k4FWCore__Writer")
+                add_writer = True
+            # Let's tell the Reader one of the input files so it can
+            # know which collections it's going to read
+            if reader is not None:
+                # Open the files and get the number of events This is necessary to
+                # avoid errors when running multithreaded since if we have, for
+                # example, 10 events and we are running 9 at the same time, then
+                # (possibly) the first 9 complete and 9 more are scheduled, out of
+                # which only one will be finished without errors. If we know the
+                # number of events in advance then we can just schedule those.
+                if props["input"][0]:
                     if os.path.exists(props["input"][0][0]):
-                        reader.Input = props["input"][0][0]
+                        path = props["input"][0][0]
                     else:
-                        reader.Input = os.getcwd() + "/" + props["input"][0][0]
-                self._mgr.TopAlg = (
-                    ([reader] if add_reader else [])
-                    + self._mgr.TopAlg
-                    + ([writer] if add_writer else [])
-                )
+                        path = os.getcwd() + "/" + props["input"][0][0]
+                    podio_reader = PodioReader(path)
+                    if self._mgr.EvtMax == -1:
+                        self._mgr.EvtMax = podio_reader._reader.getEntries("events")
+                    frame = podio_reader.get("events")[0]
+                    collections = list(frame.getAvailableCollections())
+                    reader.InputCollections = collections
+            self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
+            # Assume the writer is at the end
+            if writer:
+                self._mgr.TopAlg = [
+                    Gaudi__Sequencer(
+                        "k4FWCore__Sequencer",
+                        Sequential=True,
+                        Members=[
+                            Gaudi__Sequencer(
+                                "k4FWCore__Algs", Members=self._mgr.TopAlg, Sequential=False
+                            ),
+                            writer,
+                        ],
+                    )
+                ]
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index 124ba22e..d4a718d8 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -112,9 +112,11 @@ add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py)
 add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py)
 add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py)
 add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py)
+add_test_with_env(FunctionalProducerAnother options/ExampleFunctionalProducer.py --second)
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
 add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py)
 add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS FunctionalProducer)
+add_test_with_env(FunctionalSeveralInputFiles options/ExampleFunctionalSeveralInputFiles.py PROPERTIES "DEPENDS FunctionalProducer;FunctionalProducerAnother")
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
@@ -128,7 +130,7 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty")
+set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index f3ae0166..488ef32f 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -66,3 +66,15 @@ def check_collections(filename, names):
 check_collections(
     "functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"]
 )
+check_collections(
+    "functional_transformerMT.root",
+    [
+        "VectorFloat",
+        "MCParticles1",
+        "MCParticles2",
+        "SimTrackerHits",
+        "TrackerHits",
+        "Tracks",
+        "NewMCParticles",
+    ],
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
index 211165ba..0fae21d4 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py
@@ -26,7 +26,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ["output_k4test_exampledata_producer_multiple.root"]
+svc.input = "output_k4test_exampledata_producer_multiple.root"
 svc.output = "functional_transformer_multiple.root"
 
 transformer = ExampleFunctionalTransformerMultiple(
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 33164555..46ba6f6b 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -25,8 +25,8 @@
 from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc
 from k4FWCore import ApplicationMgr, IOSvc
 
-evtslots = 3
-threads = 3
+evtslots = 6
+threads = 6
 
 whiteboard = HiveWhiteBoard(
     "EventDataSvc",
@@ -42,20 +42,20 @@
 
 svc = IOSvc("IOSvc")
 svc.input = "output_k4test_exampledata_producer_multiple.root"
-svc.output = "functional_transformerMT.root"
+# svc.output = "functional_transformerMT.root"
 
-consumer1 = ExampleFunctionalConsumer(
+consumer = ExampleFunctionalConsumer(
     "Consumer1",
     InputCollection=["MCParticles1"],
     Offset=0,
 )
 
 transformer = ExampleFunctionalTransformer(
-    "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"]
+    "Transformer", InputCollection=["MCParticles1"], OutputCollection=["NewMCParticles"]
 )
 
 mgr = ApplicationMgr(
-    TopAlg=[consumer1, transformer],
+    TopAlg=[consumer, transformer],
     EvtSel="NONE",
     EvtMax=-1,
     ExtSvc=[whiteboard],
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
index b87343f3..d6db96aa 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py
@@ -26,7 +26,7 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ["output_k4test_exampledata_producer_multiple.root"]
+svc.input = "output_k4test_exampledata_producer_multiple.root"
 svc.output = "functional_transformer_multiple_output_commands.root"
 svc.outputCommands = [
     "drop Tracks",
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
index ec9ab87c..12ff4dac 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py
@@ -24,10 +24,18 @@
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
 from Configurables import Writer
+from k4FWCore.parseArgs import parser
 
+parser.add_argument("--second", action="store_true")
+args = parser.parse_known_args()
 
 iosvc = IOSvc("IOSvc")
-iosvc.output = "output_k4test_exampledata_producer.root"
+name = (
+    "output_k4test_exampledata_producer.root"
+    if not args[0].second
+    else "output_k4test_exampledata_producer2.root"
+)
+iosvc.output = name
 # Collections can be dropped
 # out.outputCommands = ["drop *"]
 
@@ -39,7 +47,7 @@
 ApplicationMgr(
     TopAlg=[producer, writer],
     EvtSel="NONE",
-    EvtMax=10,
+    EvtMax=10 if not args[0].second else 20,
     ExtSvc=[EventDataSvc("EventDataSvc")],
     OutputLevel=INFO,
 )
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py
new file mode 100644
index 00000000..679607e2
--- /dev/null
+++ b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py
@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example reading from a file and using a consumer with several inputs
+# to check that the contents of the file are the expected ones
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalConsumer
+from Configurables import EventDataSvc
+from k4FWCore import ApplicationMgr, IOSvc
+
+svc = IOSvc("IOSvc")
+svc.input = ["output_k4test_exampledata_producer.root", "output_k4test_exampledata_producer2.root"]
+
+consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0)
+
+mgr = ApplicationMgr(
+    TopAlg=[consumer],
+    EvtSel="NONE",
+    EvtMax=30,
+    ExtSvc=[EventDataSvc("EventDataSvc")],
+    OutputLevel=INFO,
+)

From 3765ea207674a7eba2a33fd515d525b4147730db Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 09:05:40 +0200
Subject: [PATCH 111/127] Add comments in the python files

---
 python/k4FWCore/ApplicationMgr.py | 16 +++++++++++++++-
 python/k4FWCore/IOSvc.py          |  1 +
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index a0b4b383..fd8e94a3 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -23,6 +23,17 @@
 
 
 class ApplicationMgr:
+    """ApplicationMgr is a class that wraps the Gaudi ApplicationMgr class to
+    - Give the reader the collections it's going to read so that
+      the scheduler can know which algorithms can't run until the
+      collections are available
+    - When running multithreaded and EvtMax is -1, set the number of events to
+      be run to the number of events in the file so that no more events than
+      necessary are scheduled
+    - Wrap inside a sequencer the set of algorithms and a Writer (if any) so that
+      when running multithreaded the writer runs after the algorithms
+    """
+
     def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
 
@@ -49,7 +60,7 @@ def __init__(self, **kwargs):
             # Let's tell the Reader one of the input files so it can
             # know which collections it's going to read
             if reader is not None:
-                # Open the files and get the number of events This is necessary to
+                # Open the files and get the number of events. This is necessary to
                 # avoid errors when running multithreaded since if we have, for
                 # example, 10 events and we are running 9 at the same time, then
                 # (possibly) the first 9 complete and 9 more are scheduled, out of
@@ -68,6 +79,9 @@ def __init__(self, **kwargs):
                     reader.InputCollections = collections
             self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
             # Assume the writer is at the end
+            # Algorithms are wrapped with Sequential=False so that they can run in parallel
+            # The algorithms and Writer are wrapped with Sequential=True so that the can not
+            # run in parallel
             if writer:
                 self._mgr.TopAlg = [
                     Gaudi__Sequencer(
diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py
index db40ef52..edae89fb 100644
--- a/python/k4FWCore/IOSvc.py
+++ b/python/k4FWCore/IOSvc.py
@@ -32,6 +32,7 @@ def __setattr__(self, attr, value):
             super().__setattr__(attr, value)
             return
 
+        # Allow to specify a single string for input when what we want is a list
         if attr == "input":
             if isinstance(value, str):
                 value = [value]

From 8897c205d7a3b42f7b967e58ded35ccf98a2a979 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 09:14:16 +0200
Subject: [PATCH 112/127] Bring back the output from a test

---
 test/k4FWCoreTest/options/ExampleFunctionalMTFile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
index 46ba6f6b..2af168f6 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py
@@ -42,7 +42,7 @@
 
 svc = IOSvc("IOSvc")
 svc.input = "output_k4test_exampledata_producer_multiple.root"
-# svc.output = "functional_transformerMT.root"
+svc.output = "functional_transformerMT.root"
 
 consumer = ExampleFunctionalConsumer(
     "Consumer1",

From adc3d68b3e204df0fffa81e3d3f6a24d65eed58d Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 10:35:01 +0200
Subject: [PATCH 113/127] Add new line

Co-authored-by: Andre Sailer <andre.philippe.sailer@cern.ch>
---
 python/k4FWCore/ApplicationMgr.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index fd8e94a3..d08a84ba 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -24,6 +24,7 @@
 
 class ApplicationMgr:
     """ApplicationMgr is a class that wraps the Gaudi ApplicationMgr class to
+
     - Give the reader the collections it's going to read so that
       the scheduler can know which algorithms can't run until the
       collections are available

From 12ea14fce024a5532ef5884e5c36ef3999f128c2 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 15:00:09 +0200
Subject: [PATCH 114/127] Try to fix failing test

---
 test/k4FWCoreTest/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index d4a718d8..c44a9d33 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -116,7 +116,8 @@ add_test_with_env(FunctionalProducerAnother options/ExampleFunctionalProducer.py
 add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py)
 add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py)
 add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS FunctionalProducer)
-add_test_with_env(FunctionalSeveralInputFiles options/ExampleFunctionalSeveralInputFiles.py PROPERTIES "DEPENDS FunctionalProducer;FunctionalProducerAnother")
+add_test_with_env(FunctionalSeveralInputFiles options/ExampleFunctionalSeveralInputFiles.py)
+set_tests_properties(FunctionalSeveralInputFiles PROPERTIES DEPENDS "FunctionalProducer;FunctionalProducerAnother")
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)

From 7dd30556aa8e26f53cf0475aabdc0c6bd0b3c523 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 15:22:39 +0200
Subject: [PATCH 115/127] Clean up the Writer and add comments

---
 k4FWCore/components/Writer.cpp | 37 ++++++++++++++--------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 519ab096..7fdffb17 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -41,13 +41,14 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     setProperty("Cardinality", 1).ignore();
   }
 
-  mutable Gaudi::Property<std::vector<std::string>> m_OutputNames{this, "CollectionNames", {}};
-  mutable std::set<std::string>                     m_availableCollections;
-  mutable std::vector<std::string>                  m_collectionsToAdd;
-  mutable std::vector<std::string>                  m_collectionsRemaining;
-  mutable std::vector<std::string>                  m_collectionsToSave;
+  // bool isReEntrant() const override { return false; }
 
-  mutable std::mutex m_mutex;
+  // Many members are mutable because it's assumed that the Writer is called only once
+  mutable std::set<std::string>    m_availableCollections;
+  // These are the collections that are not in the frame and we want to add to the frame
+  mutable std::vector<std::string> m_collectionsToAdd;
+  // These are the collections we want to save to the frame
+  mutable std::vector<std::string> m_collectionsToSave;
 
   ServiceHandle<IIOSvc>     iosvc{this, "IOSvc", "IOSvc"};
   SmartIF<IHiveWhiteBoard>  m_hiveWhiteBoard;
@@ -158,17 +159,13 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
   }
 
   void operator()(const EventContext& ctx) const override {
-    // It seems that even when setting Cardinality to 1,
-    // more than one instance is created
-    std::scoped_lock<std::mutex> lock(m_mutex);
-
     if (m_hiveWhiteBoard) {
       // It's never set to valid but it has the slot information
       // if (ctx.valid()) {
       //   info() << "No context found in Writer" << endmsg;
       //   return;
       // }
-      info() << "Setting store to " << ctx.slot() << endmsg;
+      debug() << "Setting store to " << ctx.slot() << endmsg;
       if (m_hiveWhiteBoard->selectStore(ctx.slot()).isFailure()) {
         error() << "Error when setting store" << endmsg;
         throw GaudiException("Error when setting store", name(), StatusCode::FAILURE);
@@ -184,11 +181,12 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       ptr = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
     }
     // This is the case when no reading is being done
-    // needs to be fixed? (new without delete)
+    // Will be deleted by the store
     else {
       ptr = new AnyDataWrapper<podio::Frame>(podio::Frame());
     }
 
+    const auto& frameCollections = ptr->getData().getAvailableCollections();
     if (m_first) {
       // Assume all the output collections are the same for all events
       // and cache them
@@ -196,19 +194,18 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       for (const auto& coll : m_availableCollections) {
         if (iosvc->checkIfWriteCollection(coll)) {
           m_collectionsToSave.push_back(coll);
-          const auto& frameCollections = ptr->getData().getAvailableCollections();
           if (std::find(frameCollections.begin(), frameCollections.end(), coll) == frameCollections.end()) {
             m_collectionsToAdd.push_back(coll);
-          } else {
-            m_collectionsRemaining.push_back(coll);
           }
         }
       }
-
       m_first = false;
     }
 
-    for (auto& coll : ptr->getData().getAvailableCollections()) {
+    // Remove the collections owned by a Frame (if any) so that they are not
+    // deleted by the store (and later deleted by the Frame, triggering a double
+    // delete)
+    for (auto& coll : frameCollections) {
       DataObject* storeCollection;
       if (m_dataSvc->retrieveObject("/Event/" + coll, storeCollection).isFailure()) {
         error() << "Failed to retrieve collection " << coll << endmsg;
@@ -251,11 +248,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
       }
     }
 
-    debug() << "Writing frame, with time: "
-            << std::chrono::duration_cast<std::chrono::milliseconds>(
-                   std::chrono::system_clock::now().time_since_epoch())
-                   .count()
-            << endmsg;
+    debug() << "Writing frame" << endmsg;
     iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave);
   }
 };

From 1c4690f40b77107e138582c431155413e8515c87 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Mon, 29 Apr 2024 15:30:33 +0200
Subject: [PATCH 116/127] Fix style and clean up

---
 k4FWCore/components/Writer.cpp    | 2 +-
 python/k4FWCore/ApplicationMgr.py | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index 7fdffb17..b4036e65 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -44,7 +44,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
   // bool isReEntrant() const override { return false; }
 
   // Many members are mutable because it's assumed that the Writer is called only once
-  mutable std::set<std::string>    m_availableCollections;
+  mutable std::set<std::string> m_availableCollections;
   // These are the collections that are not in the frame and we want to add to the frame
   mutable std::vector<std::string> m_collectionsToAdd;
   // These are the collections we want to save to the frame
diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index d08a84ba..98c06741 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -39,7 +39,6 @@ def __init__(self, **kwargs):
         self._mgr = AppMgr(**kwargs)
 
         for conf in frozenset(self._mgr.allConfigurables.values()):
-            # import pdb; pdb.set_trace()
             if not isinstance(conf, IOSvc):
                 continue
             props = conf.getPropertiesWithDescription()
@@ -57,7 +56,6 @@ def __init__(self, **kwargs):
             # while for a list it's an empty list
             if writer is None and props["output"][0] and props["output"][0] != "<no value>":
                 writer = Writer("k4FWCore__Writer")
-                add_writer = True
             # Let's tell the Reader one of the input files so it can
             # know which collections it's going to read
             if reader is not None:

From 312725d3164abe15826b3538d9e1c903b9b34fe5 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Wed, 15 May 2024 14:37:47 +0200
Subject: [PATCH 117/127] Have the runFunctionalMix tests run with a single
 steering file

---
 k4FWCore/scripts/k4run                        |   1 -
 test/k4FWCoreTest/CMakeLists.txt              |   4 +-
 test/k4FWCoreTest/options/CheckOutputFiles.py |  42 ++++++-
 .../ExampleFunctionalSeveralInputFiles.py     |   5 +-
 test/k4FWCoreTest/options/runFunctionalMix.py | 116 ++++++++++++------
 .../options/runFunctionalMixIOSvc.py          | 116 ------------------
 .../components/ExampleFunctionalConsumer.cpp  |   1 +
 .../k4FWCoreTest_CheckExampleEventData.cpp    |   4 +-
 .../k4FWCoreTest_CreateExampleEventData.cpp   |   9 +-
 9 files changed, 137 insertions(+), 161 deletions(-)
 delete mode 100644 test/k4FWCoreTest/options/runFunctionalMixIOSvc.py

diff --git a/k4FWCore/scripts/k4run b/k4FWCore/scripts/k4run
index aee31945..3facf5d5 100755
--- a/k4FWCore/scripts/k4run
+++ b/k4FWCore/scripts/k4run
@@ -3,7 +3,6 @@
 import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 import logging
 
 from k4FWCore.utils import load_file
diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt
index c44a9d33..88962474 100644
--- a/test/k4FWCoreTest/CMakeLists.txt
+++ b/test/k4FWCoreTest/CMakeLists.txt
@@ -121,7 +121,7 @@ set_tests_properties(FunctionalSeveralInputFiles PROPERTIES DEPENDS "FunctionalP
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
-add_test_with_env(FunctionalMixIOSvc options/runFunctionalMixIOSvc.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalMixIOSvc options/runFunctionalMix.py --iosvc PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
 add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
@@ -131,7 +131,7 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty")
+set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")
diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py
index 488ef32f..1cd73238 100644
--- a/test/k4FWCoreTest/options/CheckOutputFiles.py
+++ b/test/k4FWCoreTest/options/CheckOutputFiles.py
@@ -64,7 +64,8 @@ def check_collections(filename, names):
 )
 check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
 check_collections(
-    "functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"]
+    "functional_transformer_runtime_empty.root",
+    ["MCParticles0", "MCParticles1", "MCParticles2"],
 )
 check_collections(
     "functional_transformerMT.root",
@@ -78,3 +79,42 @@ def check_collections(filename, names):
         "NewMCParticles",
     ],
 )
+
+
+mix_collections = [
+    # From file
+    "VectorFloat",
+    "MCParticles1",
+    "MCParticles2",
+    "SimTrackerHits",
+    "TrackerHits",
+    "Tracks",
+    # Produced by functional
+    "FunctionalVectorFloat",
+    "FunctionalMCParticles",
+    "FunctionalMCParticles2",
+    "FunctionalSimTrackerHits",
+    "FunctionalTrackerHits",
+    "FunctionalTracks",
+    # Produced by an old algorithm
+    "OldAlgorithmMCParticles",
+    "OldAlgorithmSimTrackerHits",
+    "OldAlgorithmTrackerHits",
+    "OldAlgorithmTracks",
+    "OldAlgorithmVectorFloat",
+    # Produced by the last transformer
+    "Counter",
+    "TransformedFunctionalMCParticles1",
+]
+
+
+# Not working, collections produced by functional algorithms are not being written to the file
+# check_collections(
+#     "output_k4test_exampledata_functional_mix.root",
+#     mix_collections,
+# )
+
+check_collections(
+    "output_k4test_exampledata_functional_mix_iosvc.root",
+    mix_collections,
+)
diff --git a/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py
index 679607e2..fc35473c 100644
--- a/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py
+++ b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py
@@ -26,7 +26,10 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ["output_k4test_exampledata_producer.root", "output_k4test_exampledata_producer2.root"]
+svc.input = [
+    "output_k4test_exampledata_producer.root",
+    "output_k4test_exampledata_producer2.root",
+]
 
 consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0)
 
diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py
index 70c9f648..d80bac86 100644
--- a/test/k4FWCoreTest/options/runFunctionalMix.py
+++ b/test/k4FWCoreTest/options/runFunctionalMix.py
@@ -20,7 +20,7 @@
 # This is an example mixing functional and non-functional algorithms
 #
 
-from Gaudi.Configuration import INFO
+from Gaudi.Configuration import INFO, DEBUG
 from Configurables import (
     ExampleFunctionalConsumerMultiple,
     ExampleFunctionalTransformerMultiple,
@@ -33,19 +33,44 @@
 from Configurables import ApplicationMgr
 from Configurables import k4DataSvc
 from Configurables import PodioInput, PodioOutput
+from k4FWCore.parseArgs import parser
 
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer_multiple.root"
+parser.add_argument(
+    "--iosvc",
+    help="Use the IOSvc instead of PodioInput and PodioOutput",
+    action="store_true",
+    default=False,
+)
+args = parser.parse_known_args()[0]
+
+print(args.iosvc)
+
+if not args.iosvc:
+    podioevent = k4DataSvc("EventDataSvc")
+    podioevent.input = "output_k4test_exampledata_producer_multiple.root"
+
+    inp = PodioInput()
+    inp.collections = [
+        "VectorFloat",
+        "MCParticles1",
+        "MCParticles2",
+        "SimTrackerHits",
+        "TrackerHits",
+        "Tracks",
+    ]
+
+    out = PodioOutput()
+    out.filename = "output_k4test_exampledata_functional_mix.root"
+    out.outputCommands = ["keep *"]
+
+else:
+    from k4FWCore import IOSvc, ApplicationMgr
+
+    iosvc = IOSvc("IOSvc")
+    iosvc.input = "output_k4test_exampledata_producer_multiple.root"
+    iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root"
 
-inp = PodioInput()
-inp.collections = [
-    "VectorFloat",
-    "MCParticles1",
-    "MCParticles2",
-    "SimTrackerHits",
-    "TrackerHits",
-    "Tracks",
-]
+# Check input with functional and old algorithms
 
 consumer_input_functional = ExampleFunctionalConsumerMultiple(
     "ExampleFunctionalConsumerMultiple",
@@ -55,20 +80,25 @@
 consumer_input_algorithm.mcparticles = "MCParticles1"
 consumer_input_algorithm.keepEventNumberZero = True
 
-# We only care about the new FunctionalMCParticles collection in this example
+###############################
+
 producer_functional = ExampleFunctionalProducerMultiple(
     "ProducerFunctional",
-    OutputCollectionFloat=["VectorFloat_"],
+    OutputCollectionFloat=["FunctionalVectorFloat"],
     OutputCollectionParticles1=["FunctionalMCParticles"],
-    OutputCollectionParticles2=["MCParticles2_"],
-    OutputCollectionSimTrackerHits=["SimTrackerHits_"],
-    OutputCollectionTrackerHits=["TrackerHits_"],
-    OutputCollectionTracks=["Tracks_"],
+    OutputCollectionParticles2=["FunctionalMCParticles2"],
+    OutputCollectionSimTrackerHits=["FunctionalSimTrackerHits"],
+    OutputCollectionTrackerHits=["FunctionalTrackerHits"],
+    OutputCollectionTracks=["FunctionalTracks"],
     ExampleInt=5,
 )
 
+# Check the functional-produced collections with functional and old algorithms
+
+# Here we check the new FunctionalMCParticles and the others that are
+# read from the file
 consumer_producerfun_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerFunctional",
+    "FunctionalConsumerFromFunctional",
     InputCollectionParticles=["FunctionalMCParticles"],
     Offset=0,
 )
@@ -76,31 +106,43 @@
 consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles"
 consumer_producerfun_algorithm.keepEventNumberZero = True
 
+###############################
+
 producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData")
 # We only care about the MCParticles collection
-producer_algorithm.mcparticles = "AlgorithmMCParticles"
-producer_algorithm.simtrackhits = "SimTrackerHits__"
-producer_algorithm.trackhits = "TrackerHits__"
-producer_algorithm.tracks = "Tracks__"
-producer_algorithm.vectorfloat = "VectorFloat__"
+producer_algorithm.mcparticles = "OldAlgorithmMCParticles"
+producer_algorithm.simtrackhits = "OldAlgorithmSimTrackerHits"
+producer_algorithm.trackhits = "OldAlgorithmTrackerHits"
+producer_algorithm.tracks = "OldAlgorithmTracks"
+producer_algorithm.vectorfloat = "OldAlgorithmVectorFloat"
+
+# Check the functional-produced collections with functional and old algorithms
 
 consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerAlgorithm",
+    "FunctionalConsumerFromAlgorithm",
+    InputCollectionParticles=["OldAlgorithmMCParticles"],
     Offset=0,
 )
 consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
-consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
-consumer_produceralg_algorithm.keepEventNumberZero = True
+consumer_produceralg_algorithm.mcparticles = "OldAlgorithmMCParticles"
 
-# Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
+###############################
+
+# Let's also run the transformer on collections that are either read, produced by a functional or an algorithm
+transformer_functional = ExampleFunctionalTransformerMultiple(
+    "FunctionalTransformerMultiple",
+    InputCollectionFloat=["VectorFloat"],
+    InputCollectionParticles=["FunctionalMCParticles"],
+    InputCollectionSimTrackerHits=["OldAlgorithmSimTrackerHits"],
+    InputCollectionTrackerHits=["TrackerHits"],
+    OutputCollectionCounter=["Counter"],
+    OutputCollectionParticles=["TransformedFunctionalMCParticles1"],
+)
 
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_functional_mix.root"
 
 ApplicationMgr(
-    TopAlg=[
-        inp,
+    TopAlg=([inp] if not args.iosvc else [])
+    + [
         # Check we can read input
         consumer_input_functional,
         consumer_input_algorithm,
@@ -113,10 +155,10 @@
         consumer_produceralg_functional,
         consumer_produceralg_algorithm,
         transformer_functional,
-        out,
-    ],
+    ]
+    + ([out] if not args.iosvc else []),
     EvtSel="NONE",
     EvtMax=10,
-    ExtSvc=[podioevent],
-    OutputLevel=INFO,
+    ExtSvc=[iosvc if args.iosvc else podioevent],
+    OutputLevel=DEBUG,
 )
diff --git a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
deleted file mode 100644
index 0a4581ab..00000000
--- a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#
-# Copyright (c) 2014-2024 Key4hep-Project.
-#
-# This file is part of Key4hep.
-# See https://key4hep.github.io/key4hep-doc/ for further info.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is an example mixing functional and non-functional algorithms
-#
-
-from Gaudi.Configuration import INFO
-from Configurables import (
-    ExampleFunctionalConsumerMultiple,
-    ExampleFunctionalTransformerMultiple,
-)
-from Configurables import (
-    ExampleFunctionalProducerMultiple,
-    k4FWCoreTest_CreateExampleEventData,
-)
-from Configurables import k4FWCoreTest_CheckExampleEventData
-from k4FWCore import ApplicationMgr, IOSvc
-
-iosvc = IOSvc("IOSvc")
-iosvc.input = "output_k4test_exampledata_producer_multiple.root"
-
-# inp.collections = [
-#     "VectorFloat",
-#     "MCParticles1",
-#     "MCParticles2",
-#     "SimTrackerHits",
-#     "TrackerHits",
-#     "Tracks",
-# ]
-
-consumer_input_functional = ExampleFunctionalConsumerMultiple(
-    "ExampleFunctionalConsumerMultiple",
-    Offset=0,
-)
-consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData")
-consumer_input_algorithm.mcparticles = "MCParticles1"
-consumer_input_algorithm.keepEventNumberZero = True
-
-# We only care about the new FunctionalMCParticles collection in this example
-producer_functional = ExampleFunctionalProducerMultiple(
-    "ProducerFunctional",
-    OutputCollectionFloat=["VectorFloat_"],
-    OutputCollectionParticles1=["FunctionalMCParticles"],
-    OutputCollectionParticles2=["MCParticles2_"],
-    OutputCollectionSimTrackerHits=["SimTrackerHits_"],
-    OutputCollectionTrackerHits=["TrackerHits_"],
-    OutputCollectionTracks=["Tracks_"],
-    ExampleInt=5,
-)
-
-consumer_producerfun_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerFunctional",
-    InputCollectionParticles=["FunctionalMCParticles"],
-    Offset=0,
-)
-consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional")
-consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles"
-consumer_producerfun_algorithm.keepEventNumberZero = True
-
-producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData")
-# We only care about the MCParticles collection
-producer_algorithm.mcparticles = "AlgorithmMCParticles"
-producer_algorithm.simtrackhits = "SimTrackerHits__"
-producer_algorithm.trackhits = "TrackerHits__"
-producer_algorithm.tracks = "Tracks__"
-producer_algorithm.vectorfloat = "VectorFloat__"
-
-consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerAlgorithm",
-    Offset=0,
-)
-consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
-consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
-consumer_produceralg_algorithm.keepEventNumberZero = True
-
-# Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
-
-iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root"
-
-ApplicationMgr(
-    TopAlg=[
-        # Check we can read input
-        consumer_input_functional,
-        consumer_input_algorithm,
-        producer_functional,
-        # Check we can read what's produced by a functional
-        consumer_producerfun_functional,
-        consumer_producerfun_algorithm,
-        producer_algorithm,
-        # Check we can read what's produced by an algorithm
-        consumer_produceralg_functional,
-        consumer_produceralg_algorithm,
-        transformer_functional,
-    ],
-    EvtSel="NONE",
-    EvtMax=-1,
-    ExtSvc=[iosvc],
-    OutputLevel=INFO,
-)
diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
index 903c3bcc..a81fb702 100644
--- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
+++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp
@@ -36,6 +36,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
   // Note that the function has to be const, as well as the collections
   // we get from the input
   void operator()(const edm4hep::MCParticleCollection& input) const override {
+    debug() << "Received MCParticle collection with " << input.size() << " elements" << endmsg;
     if (input.size() != 2) {
       fatal() << "Wrong size of MCParticle collection, expected 2 got " << input.size() << endmsg;
       throw std::runtime_error("Wrong size of MCParticle collection");
diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
index fda76d93..172d51f1 100644
--- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
+++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp
@@ -56,11 +56,11 @@ StatusCode k4FWCoreTest_CheckExampleEventData::execute(const EventContext&) cons
   auto particles = m_mcParticleHandle.get();
   auto particle  = (*particles)[0];
   if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 0) ||
-      (particle.getMass() != m_magicNumberOffset + m_event + 6)) {
+      (particle.getMass() != m_magicNumberOffset + 6)) {
     std::stringstream error;
     error << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x
           << " (expected " << m_magicNumberOffset + m_event + 0 << "), mass = " << particle.getMass() << " (expected "
-          << m_magicNumberOffset + m_event + 6 << ")";
+          << m_magicNumberOffset + 6 << ")";
     throw std::runtime_error(error.str());
   }
 
diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
index a11140db..f52bbf75 100644
--- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
+++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp
@@ -64,8 +64,15 @@ StatusCode k4FWCoreTest_CreateExampleEventData::execute(const EventContext&) con
   edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut();
 
   auto particle = particles->create();
+  particle.setPDG(1);
+  particle.setGeneratorStatus(2);
+  particle.setSimulatorStatus(3);
+  particle.setCharge(4);
+  particle.setTime(5);
+  particle.setMass(m_magicNumberOffset + 6);
   particle.setMomentum({m_magicNumberOffset + m_event + 0.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0});
-  particle.setMass(m_magicNumberOffset + m_event + 6);
+
+  particles->create(2, 3, 4, 5.f, 6.f, 7.f);
 
   edm4hep::SimTrackerHitCollection* simTrackerHits = m_simTrackerHitHandle.createAndPut();
   auto                              hit            = simTrackerHits->create();

From cd6b9381d82c58582fd9ce642a4a11cf2ab3b912 Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:38:22 +0200
Subject: [PATCH 118/127] Use ShortCircuit=False

Co-authored-by: Mateusz Jakub Fila <37295697+m-fila@users.noreply.github.com>
---
 python/k4FWCore/ApplicationMgr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 98c06741..62066d3f 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -88,7 +88,7 @@ def __init__(self, **kwargs):
                         Sequential=True,
                         Members=[
                             Gaudi__Sequencer(
-                                "k4FWCore__Algs", Members=self._mgr.TopAlg, Sequential=False
+                                "k4FWCore__Algs", Members=self._mgr.TopAlg, Sequential=False, ShortCircuit=False
                             ),
                             writer,
                         ],

From 663b097aae63e437c8e07d9c543034a8f22aead0 Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:38:35 +0200
Subject: [PATCH 119/127] Fix typo

Co-authored-by: Mateusz Jakub Fila <37295697+m-fila@users.noreply.github.com>
---
 python/k4FWCore/ApplicationMgr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 62066d3f..18d0086d 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -79,7 +79,7 @@ def __init__(self, **kwargs):
             self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg
             # Assume the writer is at the end
             # Algorithms are wrapped with Sequential=False so that they can run in parallel
-            # The algorithms and Writer are wrapped with Sequential=True so that the can not
+            # The algorithms and Writer are wrapped with Sequential=True so that they can not
             # run in parallel
             if writer:
                 self._mgr.TopAlg = [

From 8c6893e2e2af94b83fc7d9255fb64e4c2400d97b Mon Sep 17 00:00:00 2001
From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:38:53 +0200
Subject: [PATCH 120/127] Use emplace_back

Co-authored-by: Mateusz Jakub Fila <37295697+m-fila@users.noreply.github.com>
---
 k4FWCore/components/Reader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 378170d8..7c194d85 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -83,7 +83,7 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
                              }
 
                              for (auto& c : m_inputCollections.value()) {
-                               m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
+                               m_outputs.emplace_back(c, this);
                              }
                            },
                            Gaudi::Details::Property::ImmediatelyInvokeHandler{true}} {}

From f5fdcf371703a992f2b10fadfdd7d1c8806a5a13 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:43:48 +0200
Subject: [PATCH 121/127] Remove some comments

---
 k4FWCore/components/Reader.cpp | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp
index 7c194d85..c05c5ce2 100644
--- a/k4FWCore/components/Reader.cpp
+++ b/k4FWCore/components/Reader.cpp
@@ -51,25 +51,6 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
 public:
   CollectionPusher(std::string name, ISvcLocator* locator)
       : base_class(std::move(name), locator),
-        // m_input{this, "Input", "Event",
-        //         [this](Gaudi::Details::PropertyBase& b) {
-        //           const std::string cmd = System::cmdLineArgs()[0];
-        //           if (cmd.find("genconf") != std::string::npos) {
-        //             return;
-        //           }
-        //           if (m_input.value() == "Event") {
-        //             return;
-        //           }
-        //           auto reader = podio::ROOTReader();
-        //           reader.openFile(m_input.value());
-        //           auto frame = podio::Frame(reader.readNextEntry(podio::Category::Event));
-        //           auto colls = frame.getAvailableCollections();
-
-        //           for (auto& c : colls) {
-        //               m_outputs.push_back(OutputHandle_t<std::shared_ptr<podio::CollectionBase>>(c, this));
-        //           }
-        //         },
-        //         Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}
         m_inputCollections{this,
                            "InputCollections",
                            {"Event"},
@@ -114,12 +95,6 @@ class CollectionPusher : public Gaudi::Functional::details::BaseClass_t<Gaudi::F
   virtual std::tuple<vector_of_<Out>, std::vector<std::string>> operator()() const = 0;
 
 private:
-  // if In is a pointer, it signals optional (as opposed to mandatory) input
-  // template <typename T>
-  // using InputHandle_t = InputHandle_t<Traits_, std::remove_pointer_t<T>>;
-  // Gaudi::Property<std::vector<DataObjID>> m_inputLocations; // TODO/FIXME: remove this duplication...
-  // TODO/FIXME: replace vector of DataObjID property + call-back with a
-  //             vector<handle> property ... as soon as declareProperty can deal with that.
   ServiceHandle<IDataProviderSvc> m_dataSvc{this, "EventDataSvc", "EventDataSvc"};
 };
 

From 1e8d0cc8537364574a03108868bd095d9614ffb6 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:51:15 +0200
Subject: [PATCH 122/127] Use isReEntrant for the Writer

---
 k4FWCore/components/Writer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp
index b4036e65..38ae00e5 100644
--- a/k4FWCore/components/Writer.cpp
+++ b/k4FWCore/components/Writer.cpp
@@ -41,7 +41,7 @@ class Writer final : public Gaudi::Functional::Consumer<void(const EventContext&
     setProperty("Cardinality", 1).ignore();
   }
 
-  // bool isReEntrant() const override { return false; }
+  bool isReEntrant() const override { return false; }
 
   // Many members are mutable because it's assumed that the Writer is called only once
   mutable std::set<std::string> m_availableCollections;

From e796ac33ca9ccec4cd58b89ff99a807c368c641a Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 20:52:10 +0200
Subject: [PATCH 123/127] Fix pre-commit

---
 python/k4FWCore/ApplicationMgr.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py
index 18d0086d..8c5dc2f3 100644
--- a/python/k4FWCore/ApplicationMgr.py
+++ b/python/k4FWCore/ApplicationMgr.py
@@ -88,7 +88,10 @@ def __init__(self, **kwargs):
                         Sequential=True,
                         Members=[
                             Gaudi__Sequencer(
-                                "k4FWCore__Algs", Members=self._mgr.TopAlg, Sequential=False, ShortCircuit=False
+                                "k4FWCore__Algs",
+                                Members=self._mgr.TopAlg,
+                                Sequential=False,
+                                ShortCircuit=False,
                             ),
                             writer,
                         ],

From dc6e736df1889bba5bd760d77d0f0854e8c8e117 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 21:02:47 +0200
Subject: [PATCH 124/127] Fix workflow files

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1a9c319c..35c14d59 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         cvmfs_base: ['sft.cern.ch/lcg/views']
-        ENVIRONMENT: ['dev3/latest/x86_64-centos7-gcc11-opt']
+        ENVIRONMENT: ['dev3/latest/x86_64-el9-gcc13-opt']
     steps:
     - uses: actions/checkout@v2
     - uses: cvmfs-contrib/github-action-cvmfs@v2
@@ -22,7 +22,7 @@ jobs:
          mkdir -p build install;\
          source /cvmfs/${{ matrix.cvmfs_base }}/${{ matrix.ENVIRONMENT }}/setup.sh;\
          cd build;\
-         cmake -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=17  -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always " -G Ninja ..;'
+         cmake -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=20  -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always " -G Ninja ..;'
     - name: Compile
       run: |
         docker exec CI_container /bin/bash -c 'cd ./Package;\

From 6409c2f79107cb73a60cc01959d9d9d8b6ae6879 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 21:10:09 +0200
Subject: [PATCH 125/127] Use alma9

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 35c14d59..430b9f73 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,7 @@ jobs:
     - uses: cvmfs-contrib/github-action-cvmfs@v2
     - name: Start container
       run: |
-        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/aidasoft/centos7:latest /bin/bash
+        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/key4hep/key4hep-images/alma9:latest /bin/bash
     - name: CMake Configure
       run: |
          docker exec CI_container /bin/bash -c 'cd Package;\

From e073c00ef2f21d007ade54761125deeda744e95d Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 21:34:37 +0200
Subject: [PATCH 126/127] Go back to centos7

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 430b9f73..12ebecbc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,13 +9,13 @@ jobs:
       fail-fast: false
       matrix:
         cvmfs_base: ['sft.cern.ch/lcg/views']
-        ENVIRONMENT: ['dev3/latest/x86_64-el9-gcc13-opt']
+        ENVIRONMENT: ['dev3/latest/x86_64-centos7-gcc11-opt']
     steps:
     - uses: actions/checkout@v2
     - uses: cvmfs-contrib/github-action-cvmfs@v2
     - name: Start container
       run: |
-        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/key4hep/key4hep-images/alma9:latest /bin/bash
+        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/key4hep/centos7:latest /bin/bash
     - name: CMake Configure
       run: |
          docker exec CI_container /bin/bash -c 'cd Package;\

From afd2856c02562b907880c47facf164b623106772 Mon Sep 17 00:00:00 2001
From: jmcarcell <jmcarcell@users.noreply.github.com>
Date: Tue, 21 May 2024 21:39:25 +0200
Subject: [PATCH 127/127] Fix path

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 12ebecbc..d96db79d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,7 @@ jobs:
     - uses: cvmfs-contrib/github-action-cvmfs@v2
     - name: Start container
       run: |
-        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/key4hep/centos7:latest /bin/bash
+        docker run -it --name CI_container -v ${GITHUB_WORKSPACE}:/Package -v /cvmfs:/cvmfs:shared -d ghcr.io/aidasoft/centos7:latest /bin/bash
     - name: CMake Configure
       run: |
          docker exec CI_container /bin/bash -c 'cd Package;\