From 61c82442e8966df8b6b95683234760923866ae31 Mon Sep 17 00:00:00 2001 From: noferini <9963644+noferini@users.noreply.github.com> Date: Sun, 19 Nov 2023 12:23:34 +0100 Subject: [PATCH] TOF matching multi-threaded --- .../include/GlobalTracking/MatchTOF.h | 13 +++-- Detectors/GlobalTracking/src/MatchTOF.cxx | 54 +++++++++++-------- .../GlobalTrackingWorkflow/TOFMatcherSpec.h | 2 +- .../src/TOFMatcherSpec.cxx | 8 +-- .../src/tof-matcher-workflow.cxx | 5 +- 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/Detectors/GlobalTracking/include/GlobalTracking/MatchTOF.h b/Detectors/GlobalTracking/include/GlobalTracking/MatchTOF.h index 0e19b15d9a401..2806dbb98a4cf 100644 --- a/Detectors/GlobalTracking/include/GlobalTracking/MatchTOF.h +++ b/Detectors/GlobalTracking/include/GlobalTracking/MatchTOF.h @@ -201,6 +201,8 @@ class MatchTOF void storeMatchable(bool val = true) { mStoreMatchable = val; } + void setNlanes(int lanes) { mNlanes = lanes; } + private: bool prepareFITData(); int prepareInteractionTimes(); @@ -215,9 +217,9 @@ class MatchTOF void doMatching(int sec); void doMatchingForTPC(int sec); - void selectBestMatches(); - void BestMatches(std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, const std::vector* TracksWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels, float calibMaxChi2); - void BestMatchesHP(std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels); + void selectBestMatches(int sec); + void BestMatches(int sec, std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, const std::vector* TracksWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels, float calibMaxChi2); + void BestMatchesHP(int sec, std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels); bool propagateToRefX(o2::track::TrackParCov& trc, float xRef /*in cm*/, float stepInCm /*in cm*/, o2::track::TrackLTIntegral& intLT); bool propagateToRefXWithoutCov(o2::track::TrackParCov& trc, float xRef /*in cm*/, float stepInCm /*in cm*/, float bz); @@ -232,6 +234,8 @@ class MatchTOF const o2::globaltracking::RecoContainer* mRecoCont = nullptr; o2::InteractionRecord mStartIR{0, 0}; ///< IR corresponding to the start of the TF + int mNlanes = 3; ///< for multi-threading in matching + // TOF matching params (work in progress) const MatchTOFParams* mMatchParams = nullptr; @@ -313,8 +317,7 @@ class MatchTOF ///< per sector indices of TOF cluster entry in mTOFClusWork std::array, o2::constants::math::NSectors> mTOFClusSectIndexCache; - /// mMatchedTracksPairs; + ///< array of track-TOFCluster pairs from the matching std::vector mMatchedTracksPairsSec[o2::constants::math::NSectors]; /// + using namespace o2::globaltracking; using evGIdx = o2::dataformats::EvIndex; using trkType = o2::dataformats::MatchInfoTOFReco::TrackType; @@ -127,25 +129,35 @@ void MatchTOF::run(const o2::globaltracking::RecoContainer& inp, unsigned long f mTimerTot.Start(); std::array nMatches = {0}; + // run matching (this can be multi-thread) + mTimerMatchITSTPC.Start(); + for (int sec = o2::constants::math::NSectors; sec--;) { - mMatchedTracksPairs.clear(); // new sector - LOG(debug) << "Doing matching for sector " << sec << "..."; - if (mIsITSTPCused || mIsTPCTRDused || mIsITSTPCTRDused) { - mTimerMatchITSTPC.Start(sec == o2::constants::math::NSectors - 1); + mMatchedTracksPairsSec[sec].clear(); // new sector + } + + o2::tof::Geo::Init(); + + if (mIsITSTPCused || mIsTPCTRDused || mIsITSTPCTRDused) { +#pragma omp parallel for schedule(dynamic) num_threads(mNlanes) + for (int sec = o2::constants::math::NSectors - 1; sec > -1; sec--) { doMatching(sec); - mTimerMatchITSTPC.Stop(); } - if (mIsTPCused) { - mTimerMatchTPC.Start(sec == o2::constants::math::NSectors - 1); + } + mTimerMatchITSTPC.Stop(); + + mTimerMatchTPC.Start(); + if (mIsTPCused) { +#pragma omp parallel for schedule(dynamic) num_threads(mNlanes) + for (int sec = o2::constants::math::NSectors - 1; sec > -1; sec--) { doMatchingForTPC(sec); - mTimerMatchTPC.Stop(); } + } + mTimerMatchTPC.Stop(); + // finalize + for (int sec = o2::constants::math::NSectors; sec--;) { if (mStoreMatchable) { - // fill per sector - mMatchedTracksPairsSec[sec].clear(); - mMatchedTracksPairsSec[sec].insert(mMatchedTracksPairsSec[sec].end(), mMatchedTracksPairs.begin(), mMatchedTracksPairs.end()); - // if MC check if good or fake matches if (mMCTruthON) { for (auto& matchingPair : mMatchedTracksPairsSec[sec]) { @@ -169,8 +181,8 @@ void MatchTOF::run(const o2::globaltracking::RecoContainer& inp, unsigned long f } LOG(debug) << "...done. Now check the best matches"; - nMatches[sec] = mMatchedTracksPairs.size(); - selectBestMatches(); + nMatches[sec] = mMatchedTracksPairsSec[sec].size(); + selectBestMatches(sec); } std::string nMatchesStr = "Number of pairs matched per sector: "; for (int sec = o2::constants::math::NSectors; sec--;) { @@ -850,7 +862,7 @@ void MatchTOF::doMatching(int sec) foundCluster = true; // set event indexes (to be checked) int eventIndexTOFCluster = mTOFClusSectIndexCache[indices[0]][itof]; - mMatchedTracksPairs.emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[iPropagation], mTrackGid[type][cacheTrk[itrk]], type, (trefTOF.getTime() - (minTrkTime + maxTrkTime - 100E3) * 0.5) * 1E-6, 0., resX, resZ); // subracting 100 ns to max track which was artificially added + mMatchedTracksPairsSec[sec].emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[iPropagation], mTrackGid[type][cacheTrk[itrk]], type, (trefTOF.getTime() - (minTrkTime + maxTrkTime - 100E3) * 0.5) * 1E-6, 0., resX, resZ); // subracting 100 ns to max track which was artificially added } } } @@ -1179,7 +1191,7 @@ void MatchTOF::doMatchingForTPC(int sec) foundCluster = true; // set event indexes (to be checked) int eventIndexTOFCluster = mTOFClusSectIndexCache[indices[0]][itof]; - mMatchedTracksPairs.emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[ibc][iPropagation], mTrackGid[trkType::UNCONS][cacheTrk[itrk]], trkType::UNCONS, resZ / mTPCVDrift * side, trefTOF.getZ(), resX, resZ); // TODO: check if this is correct! + mMatchedTracksPairsSec[sec].emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[ibc][iPropagation], mTrackGid[trkType::UNCONS][cacheTrk[itrk]], trkType::UNCONS, resZ / mTPCVDrift * side, trefTOF.getZ(), resX, resZ); // TODO: check if this is correct! } } } @@ -1231,16 +1243,16 @@ int MatchTOF::findFITIndex(int bc, const gsl::span& FI return index; } //______________________________________________ -void MatchTOF::selectBestMatches() +void MatchTOF::selectBestMatches(int sec) { if (mSetHighPurity) { - BestMatchesHP(mMatchedTracksPairs, mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels); + BestMatchesHP(sec, mMatchedTracksPairsSec[sec], mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels); return; } - BestMatches(mMatchedTracksPairs, mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mTracksWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels, mMatchParams->calibMaxChi2); + BestMatches(sec, mMatchedTracksPairsSec[sec], mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mTracksWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels, mMatchParams->calibMaxChi2); } //______________________________________________ -void MatchTOF::BestMatches(std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, const std::vector* TracksWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels, float calibMaxChi2) +void MatchTOF::BestMatches(int sec, std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, const std::vector* TracksWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels, float calibMaxChi2) { ///< define the track-TOFcluster pair per sector @@ -1355,7 +1367,7 @@ void MatchTOF::BestMatches(std::vector& match } } //______________________________________________ -void MatchTOF::BestMatchesHP(std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels) +void MatchTOF::BestMatchesHP(int sec, std::vector& matchedTracksPairs, std::vector* matchedTracks, std::vector* matchedTracksIndex, int* matchedClustersIndex, const gsl::span& FITRecPoints, const std::vector& TOFClusWork, std::vector& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer* TOFClusLabels, const std::vector* TracksLblWork, std::vector* OutTOFLabels) { ///< define the track-TOFcluster pair per sector float chi2SeparationCut = 2; diff --git a/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/TOFMatcherSpec.h b/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/TOFMatcherSpec.h index a1fc7062a2fe9..dff2a6d416354 100644 --- a/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/TOFMatcherSpec.h +++ b/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/TOFMatcherSpec.h @@ -25,7 +25,7 @@ namespace globaltracking { /// create a processor spec -framework::DataProcessorSpec getTOFMatcherSpec(o2::dataformats::GlobalTrackID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType); +framework::DataProcessorSpec getTOFMatcherSpec(o2::dataformats::GlobalTrackID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType, int nlanes = 1); } // namespace globaltracking } // namespace o2 diff --git a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx index 6453e79c635f8..51af89e027cc8 100644 --- a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx @@ -58,7 +58,7 @@ namespace globaltracking class TOFMatcherSpec : public Task { public: - TOFMatcherSpec(std::shared_ptr dr, std::shared_ptr gr, bool useMC, bool useFIT, bool tpcRefit, bool strict, bool pushMatchable) : mDataRequest(dr), mGGCCDBRequest(gr), mUseMC(useMC), mUseFIT(useFIT), mDoTPCRefit(tpcRefit), mStrict(strict), mPushMatchable(pushMatchable) {} + TOFMatcherSpec(std::shared_ptr dr, std::shared_ptr gr, bool useMC, bool useFIT, bool tpcRefit, bool strict, bool pushMatchable, int lanes = 1) : mDataRequest(dr), mGGCCDBRequest(gr), mUseMC(useMC), mUseFIT(useFIT), mDoTPCRefit(tpcRefit), mStrict(strict), mPushMatchable(pushMatchable), mNlanes(lanes) {} ~TOFMatcherSpec() override = default; void init(InitContext& ic) final; void run(ProcessingContext& pc) final; @@ -77,6 +77,7 @@ class TOFMatcherSpec : public Task bool mStrict = false; bool mPushMatchable = false; float mExtraTolTRD = 0.; + int mNlanes = 1; MatchTOF mMatcher; ///< Cluster finder TStopwatch mTimer; }; @@ -92,6 +93,7 @@ void TOFMatcherSpec::init(InitContext& ic) mTPCCorrMapsLoader.init(ic); mMatcher.storeMatchable(mPushMatchable); mMatcher.setExtraTimeToleranceTRD(mExtraTolTRD); + mMatcher.setNlanes(mNlanes); } void TOFMatcherSpec::updateTimeDependentParams(ProcessingContext& pc) @@ -237,7 +239,7 @@ void TOFMatcherSpec::endOfStream(EndOfStreamContext& ec) mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1); } -DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType) +DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType, int nlanes) { uint32_t ss = o2::globaltracking::getSubSpec(strict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard); Options opts; @@ -314,7 +316,7 @@ DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bo "tof-matcher", dataRequest->inputs, outputs, - AlgorithmSpec{adaptFromTask(dataRequest, ggRequest, useMC, useFIT, tpcRefit, strict, pushMatchable)}, + AlgorithmSpec{adaptFromTask(dataRequest, ggRequest, useMC, useFIT, tpcRefit, strict, pushMatchable, nlanes)}, opts}; } diff --git a/Detectors/GlobalTrackingWorkflow/src/tof-matcher-workflow.cxx b/Detectors/GlobalTrackingWorkflow/src/tof-matcher-workflow.cxx index bfe65fe32eb19..9de34a1aa5437 100644 --- a/Detectors/GlobalTrackingWorkflow/src/tof-matcher-workflow.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/tof-matcher-workflow.cxx @@ -54,6 +54,7 @@ void customize(std::vector& workflowOptions) // option allowing to set parameters std::vector options{ {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}}, + {"tof-lanes", o2::framework::VariantType::Int, 3, {"number of parallel lanes up to the matcher"}}, {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}}, {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}}, {"track-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources to use: allowed TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD (all)"}}, @@ -97,6 +98,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext) bool writematching = 0; bool writecalib = 0; auto outputType = configcontext.options().get("output-type"); + auto nLanes = configcontext.options().get("tof-lanes"); if (outputType.rfind("matching-info") < outputType.size()) { writematching = 1; } @@ -123,6 +125,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext) LOG(debug) << "TOF matching in strict mode = " << strict; LOG(debug) << "TOF extra time tolerance for TRD tracks = " << extratolerancetrd; LOG(debug) << "Store all matchables = " << writeMatchable; + LOG(debug) << "TOF Nlanes for matcher = " << nLanes; //GID::mask_t alowedSources = GID::getSourcesMask("TPC,ITS-TPC"); GID::mask_t alowedSources = GID::getSourcesMask("TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD"); @@ -159,7 +162,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext) } } - specs.emplace_back(o2::globaltracking::getTOFMatcherSpec(src, useMC, useFIT, false, strict, extratolerancetrd, writeMatchable, lumiType)); // doTPCrefit not yet supported (need to load TPC clusters?) + specs.emplace_back(o2::globaltracking::getTOFMatcherSpec(src, useMC, useFIT, false, strict, extratolerancetrd, writeMatchable, lumiType, nLanes)); // doTPCrefit not yet supported (need to load TPC clusters?) if (!disableRootOut) { std::vector writers;