From 312725d3164abe15826b3538d9e1c903b9b34fe5 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 15 May 2024 14:37:47 +0200 Subject: [PATCH] Have the runFunctionalMix tests run with a single steering file --- k4FWCore/scripts/k4run | 1 - test/k4FWCoreTest/CMakeLists.txt | 4 +- test/k4FWCoreTest/options/CheckOutputFiles.py | 42 ++++++- .../ExampleFunctionalSeveralInputFiles.py | 5 +- test/k4FWCoreTest/options/runFunctionalMix.py | 116 ++++++++++++------ .../options/runFunctionalMixIOSvc.py | 116 ------------------ .../components/ExampleFunctionalConsumer.cpp | 1 + .../k4FWCoreTest_CheckExampleEventData.cpp | 4 +- .../k4FWCoreTest_CreateExampleEventData.cpp | 9 +- 9 files changed, 137 insertions(+), 161 deletions(-) delete mode 100644 test/k4FWCoreTest/options/runFunctionalMixIOSvc.py diff --git a/k4FWCore/scripts/k4run b/k4FWCore/scripts/k4run index aee31945..3facf5d5 100755 --- a/k4FWCore/scripts/k4run +++ b/k4FWCore/scripts/k4run @@ -3,7 +3,6 @@ import os import sys import argparse -from multiprocessing import cpu_count import logging from k4FWCore.utils import load_file diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index c44a9d33..88962474 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -121,7 +121,7 @@ set_tests_properties(FunctionalSeveralInputFiles PROPERTIES DEPENDS "FunctionalP add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer) add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple) add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple) -add_test_with_env(FunctionalMixIOSvc options/runFunctionalMixIOSvc.py PROPERTIES DEPENDS FunctionalProducerMultiple) +add_test_with_env(FunctionalMixIOSvc options/runFunctionalMix.py --iosvc PROPERTIES DEPENDS FunctionalProducerMultiple) add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple) add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py) add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py) @@ -131,7 +131,7 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py) add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py) -set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty") +set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc") # Do this after checking the files not to overwrite them add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop") diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py index 488ef32f..1cd73238 100644 --- a/test/k4FWCoreTest/options/CheckOutputFiles.py +++ b/test/k4FWCoreTest/options/CheckOutputFiles.py @@ -64,7 +64,8 @@ def check_collections(filename, names): ) check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"]) check_collections( - "functional_transformer_runtime_empty.root", ["MCParticles0", "MCParticles1", "MCParticles2"] + "functional_transformer_runtime_empty.root", + ["MCParticles0", "MCParticles1", "MCParticles2"], ) check_collections( "functional_transformerMT.root", @@ -78,3 +79,42 @@ def check_collections(filename, names): "NewMCParticles", ], ) + + +mix_collections = [ + # From file + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + # Produced by functional + "FunctionalVectorFloat", + "FunctionalMCParticles", + "FunctionalMCParticles2", + "FunctionalSimTrackerHits", + "FunctionalTrackerHits", + "FunctionalTracks", + # Produced by an old algorithm + "OldAlgorithmMCParticles", + "OldAlgorithmSimTrackerHits", + "OldAlgorithmTrackerHits", + "OldAlgorithmTracks", + "OldAlgorithmVectorFloat", + # Produced by the last transformer + "Counter", + "TransformedFunctionalMCParticles1", +] + + +# Not working, collections produced by functional algorithms are not being written to the file +# check_collections( +# "output_k4test_exampledata_functional_mix.root", +# mix_collections, +# ) + +check_collections( + "output_k4test_exampledata_functional_mix_iosvc.root", + mix_collections, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py index 679607e2..fc35473c 100644 --- a/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py @@ -26,7 +26,10 @@ from k4FWCore import ApplicationMgr, IOSvc svc = IOSvc("IOSvc") -svc.input = ["output_k4test_exampledata_producer.root", "output_k4test_exampledata_producer2.root"] +svc.input = [ + "output_k4test_exampledata_producer.root", + "output_k4test_exampledata_producer2.root", +] consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0) diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py index 70c9f648..d80bac86 100644 --- a/test/k4FWCoreTest/options/runFunctionalMix.py +++ b/test/k4FWCoreTest/options/runFunctionalMix.py @@ -20,7 +20,7 @@ # This is an example mixing functional and non-functional algorithms # -from Gaudi.Configuration import INFO +from Gaudi.Configuration import INFO, DEBUG from Configurables import ( ExampleFunctionalConsumerMultiple, ExampleFunctionalTransformerMultiple, @@ -33,19 +33,44 @@ from Configurables import ApplicationMgr from Configurables import k4DataSvc from Configurables import PodioInput, PodioOutput +from k4FWCore.parseArgs import parser -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer_multiple.root" +parser.add_argument( + "--iosvc", + help="Use the IOSvc instead of PodioInput and PodioOutput", + action="store_true", + default=False, +) +args = parser.parse_known_args()[0] + +print(args.iosvc) + +if not args.iosvc: + podioevent = k4DataSvc("EventDataSvc") + podioevent.input = "output_k4test_exampledata_producer_multiple.root" + + inp = PodioInput() + inp.collections = [ + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + ] + + out = PodioOutput() + out.filename = "output_k4test_exampledata_functional_mix.root" + out.outputCommands = ["keep *"] + +else: + from k4FWCore import IOSvc, ApplicationMgr + + iosvc = IOSvc("IOSvc") + iosvc.input = "output_k4test_exampledata_producer_multiple.root" + iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root" -inp = PodioInput() -inp.collections = [ - "VectorFloat", - "MCParticles1", - "MCParticles2", - "SimTrackerHits", - "TrackerHits", - "Tracks", -] +# Check input with functional and old algorithms consumer_input_functional = ExampleFunctionalConsumerMultiple( "ExampleFunctionalConsumerMultiple", @@ -55,20 +80,25 @@ consumer_input_algorithm.mcparticles = "MCParticles1" consumer_input_algorithm.keepEventNumberZero = True -# We only care about the new FunctionalMCParticles collection in this example +############################### + producer_functional = ExampleFunctionalProducerMultiple( "ProducerFunctional", - OutputCollectionFloat=["VectorFloat_"], + OutputCollectionFloat=["FunctionalVectorFloat"], OutputCollectionParticles1=["FunctionalMCParticles"], - OutputCollectionParticles2=["MCParticles2_"], - OutputCollectionSimTrackerHits=["SimTrackerHits_"], - OutputCollectionTrackerHits=["TrackerHits_"], - OutputCollectionTracks=["Tracks_"], + OutputCollectionParticles2=["FunctionalMCParticles2"], + OutputCollectionSimTrackerHits=["FunctionalSimTrackerHits"], + OutputCollectionTrackerHits=["FunctionalTrackerHits"], + OutputCollectionTracks=["FunctionalTracks"], ExampleInt=5, ) +# Check the functional-produced collections with functional and old algorithms + +# Here we check the new FunctionalMCParticles and the others that are +# read from the file consumer_producerfun_functional = ExampleFunctionalConsumerMultiple( - "FunctionalConsumerFunctional", + "FunctionalConsumerFromFunctional", InputCollectionParticles=["FunctionalMCParticles"], Offset=0, ) @@ -76,31 +106,43 @@ consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles" consumer_producerfun_algorithm.keepEventNumberZero = True +############################### + producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData") # We only care about the MCParticles collection -producer_algorithm.mcparticles = "AlgorithmMCParticles" -producer_algorithm.simtrackhits = "SimTrackerHits__" -producer_algorithm.trackhits = "TrackerHits__" -producer_algorithm.tracks = "Tracks__" -producer_algorithm.vectorfloat = "VectorFloat__" +producer_algorithm.mcparticles = "OldAlgorithmMCParticles" +producer_algorithm.simtrackhits = "OldAlgorithmSimTrackerHits" +producer_algorithm.trackhits = "OldAlgorithmTrackerHits" +producer_algorithm.tracks = "OldAlgorithmTracks" +producer_algorithm.vectorfloat = "OldAlgorithmVectorFloat" + +# Check the functional-produced collections with functional and old algorithms consumer_produceralg_functional = ExampleFunctionalConsumerMultiple( - "FunctionalConsumerAlgorithm", + "FunctionalConsumerFromAlgorithm", + InputCollectionParticles=["OldAlgorithmMCParticles"], Offset=0, ) consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm") -consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles" -consumer_produceralg_algorithm.keepEventNumberZero = True +consumer_produceralg_algorithm.mcparticles = "OldAlgorithmMCParticles" -# Let's also run the transformer, why not -transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple") +############################### + +# Let's also run the transformer on collections that are either read, produced by a functional or an algorithm +transformer_functional = ExampleFunctionalTransformerMultiple( + "FunctionalTransformerMultiple", + InputCollectionFloat=["VectorFloat"], + InputCollectionParticles=["FunctionalMCParticles"], + InputCollectionSimTrackerHits=["OldAlgorithmSimTrackerHits"], + InputCollectionTrackerHits=["TrackerHits"], + OutputCollectionCounter=["Counter"], + OutputCollectionParticles=["TransformedFunctionalMCParticles1"], +) -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_functional_mix.root" ApplicationMgr( - TopAlg=[ - inp, + TopAlg=([inp] if not args.iosvc else []) + + [ # Check we can read input consumer_input_functional, consumer_input_algorithm, @@ -113,10 +155,10 @@ consumer_produceralg_functional, consumer_produceralg_algorithm, transformer_functional, - out, - ], + ] + + ([out] if not args.iosvc else []), EvtSel="NONE", EvtMax=10, - ExtSvc=[podioevent], - OutputLevel=INFO, + ExtSvc=[iosvc if args.iosvc else podioevent], + OutputLevel=DEBUG, ) diff --git a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py b/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py deleted file mode 100644 index 0a4581ab..00000000 --- a/test/k4FWCoreTest/options/runFunctionalMixIOSvc.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright (c) 2014-2024 Key4hep-Project. -# -# This file is part of Key4hep. -# See https://key4hep.github.io/key4hep-doc/ for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is an example mixing functional and non-functional algorithms -# - -from Gaudi.Configuration import INFO -from Configurables import ( - ExampleFunctionalConsumerMultiple, - ExampleFunctionalTransformerMultiple, -) -from Configurables import ( - ExampleFunctionalProducerMultiple, - k4FWCoreTest_CreateExampleEventData, -) -from Configurables import k4FWCoreTest_CheckExampleEventData -from k4FWCore import ApplicationMgr, IOSvc - -iosvc = IOSvc("IOSvc") -iosvc.input = "output_k4test_exampledata_producer_multiple.root" - -# inp.collections = [ -# "VectorFloat", -# "MCParticles1", -# "MCParticles2", -# "SimTrackerHits", -# "TrackerHits", -# "Tracks", -# ] - -consumer_input_functional = ExampleFunctionalConsumerMultiple( - "ExampleFunctionalConsumerMultiple", - Offset=0, -) -consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData") -consumer_input_algorithm.mcparticles = "MCParticles1" -consumer_input_algorithm.keepEventNumberZero = True - -# We only care about the new FunctionalMCParticles collection in this example -producer_functional = ExampleFunctionalProducerMultiple( - "ProducerFunctional", - OutputCollectionFloat=["VectorFloat_"], - OutputCollectionParticles1=["FunctionalMCParticles"], - OutputCollectionParticles2=["MCParticles2_"], - OutputCollectionSimTrackerHits=["SimTrackerHits_"], - OutputCollectionTrackerHits=["TrackerHits_"], - OutputCollectionTracks=["Tracks_"], - ExampleInt=5, -) - -consumer_producerfun_functional = ExampleFunctionalConsumerMultiple( - "FunctionalConsumerFunctional", - InputCollectionParticles=["FunctionalMCParticles"], - Offset=0, -) -consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional") -consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles" -consumer_producerfun_algorithm.keepEventNumberZero = True - -producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData") -# We only care about the MCParticles collection -producer_algorithm.mcparticles = "AlgorithmMCParticles" -producer_algorithm.simtrackhits = "SimTrackerHits__" -producer_algorithm.trackhits = "TrackerHits__" -producer_algorithm.tracks = "Tracks__" -producer_algorithm.vectorfloat = "VectorFloat__" - -consumer_produceralg_functional = ExampleFunctionalConsumerMultiple( - "FunctionalConsumerAlgorithm", - Offset=0, -) -consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm") -consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles" -consumer_produceralg_algorithm.keepEventNumberZero = True - -# Let's also run the transformer, why not -transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple") - -iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root" - -ApplicationMgr( - TopAlg=[ - # Check we can read input - consumer_input_functional, - consumer_input_algorithm, - producer_functional, - # Check we can read what's produced by a functional - consumer_producerfun_functional, - consumer_producerfun_algorithm, - producer_algorithm, - # Check we can read what's produced by an algorithm - consumer_produceralg_functional, - consumer_produceralg_algorithm, - transformer_functional, - ], - EvtSel="NONE", - EvtMax=-1, - ExtSvc=[iosvc], - OutputLevel=INFO, -) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp index 903c3bcc..a81fb702 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp @@ -36,6 +36,7 @@ struct ExampleFunctionalConsumer final : k4FWCore::Consumercreate(); + particle.setPDG(1); + particle.setGeneratorStatus(2); + particle.setSimulatorStatus(3); + particle.setCharge(4); + particle.setTime(5); + particle.setMass(m_magicNumberOffset + 6); particle.setMomentum({m_magicNumberOffset + m_event + 0.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0}); - particle.setMass(m_magicNumberOffset + m_event + 6); + + particles->create(2, 3, 4, 5.f, 6.f, 7.f); edm4hep::SimTrackerHitCollection* simTrackerHits = m_simTrackerHitHandle.createAndPut(); auto hit = simTrackerHits->create();