Have the runFunctionalMix tests run with a single steering file

k4FWCore/scripts/k4run

@@ -3,7 +3,6 @@
 import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 import logging
 
 from k4FWCore.utils import load_file

test/k4FWCoreTest/CMakeLists.txt

@@ -121,7 +121,7 @@ set_tests_properties(FunctionalSeveralInputFiles PROPERTIES DEPENDS "FunctionalP
 add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer)
 add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple)
-add_test_with_env(FunctionalMixIOSvc options/runFunctionalMixIOSvc.py PROPERTIES DEPENDS FunctionalProducerMultiple)
+add_test_with_env(FunctionalMixIOSvc options/runFunctionalMix.py --iosvc PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple)
 add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py)
 add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py)
@@ -131,7 +131,7 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty")
+set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")

test/k4FWCoreTest/options/CheckOutputFiles.py

@@ -64,7 +64,8 @@ def check_collections(filename, names):
 )
 check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
 check_collections(
-    "functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"]
+    "functional_transformer_runtime_empty.root",
+    ["MCParticles0", "MCParticles1", "MCParticles2"],
 )
 check_collections(
     "functional_transformerMT.root",
@@ -78,3 +79,42 @@ def check_collections(filename, names):
         "NewMCParticles",
     ],
 )
+
+
+mix_collections = [
+    # From file
+    "VectorFloat",
+    "MCParticles1",
+    "MCParticles2",
+    "SimTrackerHits",
+    "TrackerHits",
+    "Tracks",
+    # Produced by functional
+    "FunctionalVectorFloat",
+    "FunctionalMCParticles",
+    "FunctionalMCParticles2",
+    "FunctionalSimTrackerHits",
+    "FunctionalTrackerHits",
+    "FunctionalTracks",
+    # Produced by an old algorithm
+    "OldAlgorithmMCParticles",
+    "OldAlgorithmSimTrackerHits",
+    "OldAlgorithmTrackerHits",
+    "OldAlgorithmTracks",
+    "OldAlgorithmVectorFloat",
+    # Produced by the last transformer
+    "Counter",
+    "TransformedFunctionalMCParticles1",
+]
+
+
+# Not working, collections produced by functional algorithms are not being written to the file
+# check_collections(
+#     "output_k4test_exampledata_functional_mix.root",
+#     mix_collections,
+# )
+
+check_collections(
+    "output_k4test_exampledata_functional_mix_iosvc.root",
+    mix_collections,
+)

test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py

@@ -26,7 +26,10 @@
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = ["output_k4test_exampledata_producer.root", "output_k4test_exampledata_producer2.root"]
+svc.input = [
+    "output_k4test_exampledata_producer.root",
+    "output_k4test_exampledata_producer2.root",
+]
 
 consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0)
 

test/k4FWCoreTest/options/runFunctionalMix.py

@@ -20,7 +20,7 @@
 # This is an example mixing functional and non-functional algorithms
 #
 
-from Gaudi.Configuration import INFO
+from Gaudi.Configuration import INFO, DEBUG
 from Configurables import (
     ExampleFunctionalConsumerMultiple,
     ExampleFunctionalTransformerMultiple,
@@ -33,19 +33,44 @@
 from Configurables import ApplicationMgr
 from Configurables import k4DataSvc
 from Configurables import PodioInput, PodioOutput
+from k4FWCore.parseArgs import parser
 
-podioevent = k4DataSvc("EventDataSvc")
-podioevent.input = "output_k4test_exampledata_producer_multiple.root"
+parser.add_argument(
+    "--iosvc",
+    help="Use the IOSvc instead of PodioInput and PodioOutput",
+    action="store_true",
+    default=False,
+)
+args = parser.parse_known_args()[0]
+
+print(args.iosvc)
+
+if not args.iosvc:
+    podioevent = k4DataSvc("EventDataSvc")
+    podioevent.input = "output_k4test_exampledata_producer_multiple.root"
+
+    inp = PodioInput()
+    inp.collections = [
+        "VectorFloat",
+        "MCParticles1",
+        "MCParticles2",
+        "SimTrackerHits",
+        "TrackerHits",
+        "Tracks",
+    ]
+
+    out = PodioOutput()
+    out.filename = "output_k4test_exampledata_functional_mix.root"
+    out.outputCommands = ["keep *"]
+
+else:
+    from k4FWCore import IOSvc, ApplicationMgr
+
+    iosvc = IOSvc("IOSvc")
+    iosvc.input = "output_k4test_exampledata_producer_multiple.root"
+    iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root"
 
-inp = PodioInput()
-inp.collections = [
-    "VectorFloat",
-    "MCParticles1",
-    "MCParticles2",
-    "SimTrackerHits",
-    "TrackerHits",
-    "Tracks",
-]
+# Check input with functional and old algorithms
 
 consumer_input_functional = ExampleFunctionalConsumerMultiple(
     "ExampleFunctionalConsumerMultiple",
@@ -55,52 +80,69 @@
 consumer_input_algorithm.mcparticles = "MCParticles1"
 consumer_input_algorithm.keepEventNumberZero = True
 
-# We only care about the new FunctionalMCParticles collection in this example
+###############################
+
 producer_functional = ExampleFunctionalProducerMultiple(
     "ProducerFunctional",
-    OutputCollectionFloat=["VectorFloat_"],
+    OutputCollectionFloat=["FunctionalVectorFloat"],
     OutputCollectionParticles1=["FunctionalMCParticles"],
-    OutputCollectionParticles2=["MCParticles2_"],
-    OutputCollectionSimTrackerHits=["SimTrackerHits_"],
-    OutputCollectionTrackerHits=["TrackerHits_"],
-    OutputCollectionTracks=["Tracks_"],
+    OutputCollectionParticles2=["FunctionalMCParticles2"],
+    OutputCollectionSimTrackerHits=["FunctionalSimTrackerHits"],
+    OutputCollectionTrackerHits=["FunctionalTrackerHits"],
+    OutputCollectionTracks=["FunctionalTracks"],
     ExampleInt=5,
 )
 
+# Check the functional-produced collections with functional and old algorithms
+
+# Here we check the new FunctionalMCParticles and the others that are
+# read from the file
 consumer_producerfun_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerFunctional",
+    "FunctionalConsumerFromFunctional",
     InputCollectionParticles=["FunctionalMCParticles"],
     Offset=0,
 )
 consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional")
 consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles"
 consumer_producerfun_algorithm.keepEventNumberZero = True
 
+###############################
+
 producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData")
 # We only care about the MCParticles collection
-producer_algorithm.mcparticles = "AlgorithmMCParticles"
-producer_algorithm.simtrackhits = "SimTrackerHits__"
-producer_algorithm.trackhits = "TrackerHits__"
-producer_algorithm.tracks = "Tracks__"
-producer_algorithm.vectorfloat = "VectorFloat__"
+producer_algorithm.mcparticles = "OldAlgorithmMCParticles"
+producer_algorithm.simtrackhits = "OldAlgorithmSimTrackerHits"
+producer_algorithm.trackhits = "OldAlgorithmTrackerHits"
+producer_algorithm.tracks = "OldAlgorithmTracks"
+producer_algorithm.vectorfloat = "OldAlgorithmVectorFloat"
+
+# Check the functional-produced collections with functional and old algorithms
 
 consumer_produceralg_functional = ExampleFunctionalConsumerMultiple(
-    "FunctionalConsumerAlgorithm",
+    "FunctionalConsumerFromAlgorithm",
+    InputCollectionParticles=["OldAlgorithmMCParticles"],
     Offset=0,
 )
 consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm")
-consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles"
-consumer_produceralg_algorithm.keepEventNumberZero = True
+consumer_produceralg_algorithm.mcparticles = "OldAlgorithmMCParticles"
 
-# Let's also run the transformer, why not
-transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple")
+###############################
+
+# Let's also run the transformer on collections that are either read, produced by a functional or an algorithm
+transformer_functional = ExampleFunctionalTransformerMultiple(
+    "FunctionalTransformerMultiple",
+    InputCollectionFloat=["VectorFloat"],
+    InputCollectionParticles=["FunctionalMCParticles"],
+    InputCollectionSimTrackerHits=["OldAlgorithmSimTrackerHits"],
+    InputCollectionTrackerHits=["TrackerHits"],
+    OutputCollectionCounter=["Counter"],
+    OutputCollectionParticles=["TransformedFunctionalMCParticles1"],
+)
 
-out = PodioOutput("out")
-out.filename = "output_k4test_exampledata_functional_mix.root"
 
 ApplicationMgr(
-    TopAlg=[
-        inp,
+    TopAlg=([inp] if not args.iosvc else [])
+    + [
         # Check we can read input
         consumer_input_functional,
         consumer_input_algorithm,
@@ -113,10 +155,10 @@
         consumer_produceralg_functional,
         consumer_produceralg_algorithm,
         transformer_functional,
-        out,
-    ],
+    ]
+    + ([out] if not args.iosvc else []),
     EvtSel="NONE",
     EvtMax=10,
-    ExtSvc=[podioevent],
-    OutputLevel=INFO,
+    ExtSvc=[iosvc if args.iosvc else podioevent],
+    OutputLevel=DEBUG,
 )

test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp

@@ -36,6 +36,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumer<void(const edm4hep::
   // Note that the function has to be const, as well as the collections
   // we get from the input
   void operator()(const edm4hep::MCParticleCollection& input) const override {
+    debug() << "Received MCParticle collection with " << input.size() << " elements" << endmsg;
     if (input.size() != 2) {
       fatal() << "Wrong size of MCParticle collection, expected 2 got " << input.size() << endmsg;
       throw std::runtime_error("Wrong size of MCParticle collection");