GPU ITS tracking: make tracking mode available in configKeyVals + fix…

…es (#13056)

* Fix usage of VertexerTraits from within the GPU reco WF

* Prepare settings and cleanup interface

* Temporarily re-enable the gpu-wf-spec in itsreco-wf

* Extend configuration

Detectors/ITSMFT/ITS/tracking/GPU/cuda/VertexerTraitsGPU.cu

@@ -93,7 +93,6 @@ VertexerTraitsGPU::VertexerTraitsGPU()
 
 VertexerTraitsGPU::~VertexerTraitsGPU()
 {
-  gpu::utils::gpuFree(mDeviceIndexTableUtils);
 }
 
 void VertexerTraitsGPU::initialise(const TrackingParameters& trackingParams)

Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h

@@ -34,7 +34,8 @@ namespace its
 enum class TrackingMode {
   Sync,
   Async,
-  Cosmics
+  Cosmics,
+  Unset, // Special value to leave a default in case we want to override via Configurable Params
 };
 
 std::string asString(TrackingMode mode);
@@ -146,6 +147,7 @@ struct TimeFrameGPUParameters {
   size_t maxVerticesCapacity = 5e4;
   size_t nMaxROFs = 1e3;
   size_t nTimeFrameChunks = 3;
+  size_t nROFsPerChunk = 768; // pp defaults
   int maxGPUMemoryGB = -1;
 };
 

Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h

@@ -78,6 +78,8 @@ struct TrackerParamConfig : public o2::conf::ConfigurableParamHelper<TrackerPara
   int nROFsPerIterations = 0;
   bool perPrimaryVertexProcessing = false;
   bool saveTimeBenchmarks = false;
+  bool overrideBeamEstimation = false; // used by gpuwf only
+  int trackingMode = -1;               // -1: unset, 0=sync, 1=async, 2=cosmics used by gpuwf only
 
   O2ParamDef(TrackerParamConfig, "ITSCATrackerParam");
 };

Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h

@@ -34,20 +34,21 @@ class ITSTrackingInterface
  public:
   ITSTrackingInterface(bool isMC,
                        int trgType,
-                       const TrackingMode trMode,
                        const bool overrBeamEst)
     : mIsMC{isMC},
       mUseTriggers{trgType},
-      mMode{trMode},
       mOverrideBeamEstimation{overrBeamEst}
   {
   }
 
   void setClusterDictionary(const o2::itsmft::TopologyDictionary* d) { mDict = d; }
   void setMeanVertex(const o2::dataformats::MeanVertexObject* v)
   {
-    if (!v) {
+    if (v == nullptr) {
+      LOGP(error, "Mean Vertex Object is nullptr");
       return;
+    } else {
+      LOGP(info, "Mean Vertex set with x: {} y: {}", v->getX(), v->getY());
     }
     mMeanVertex = v;
   }
@@ -61,13 +62,20 @@ class ITSTrackingInterface
 
   // Custom
   void setTraitsFromProvider(VertexerTraits*, TrackerTraits*, TimeFrame*);
+  void setTrackingMode(TrackingMode mode = TrackingMode::Unset)
+  {
+    if (mode == TrackingMode::Unset) {
+      LOGP(fatal, "ITS Tracking mode Unset is meant to be a default. Specify the mode");
+    }
+    mMode = mode;
+  }
 
  private:
   bool mIsMC = false;
   bool mRunVertexer = true;
   bool mCosmicsProcessing = false;
   int mUseTriggers = 0;
-  TrackingMode mMode = TrackingMode::Sync;
+  TrackingMode mMode = TrackingMode::Unset;
   bool mOverrideBeamEstimation = false;
   const o2::itsmft::TopologyDictionary* mDict = nullptr;
   std::unique_ptr<Tracker> mTracker = nullptr;

Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h

@@ -85,13 +85,13 @@ class VertexerTraits
                             TimeFrame*,
                             std::vector<o2::MCCompLabel>*);
 
-  VertexingParameters getVertexingParameters() const { return mVrtParams; }
   static const std::vector<std::pair<int, int>> selectClusters(const int* indexTable,
                                                                const std::array<int, 4>& selectedBinsRect,
                                                                const IndexTableUtils& utils);
 
   // utils
   VertexingParameters& getVertexingParameters() { return mVrtParams; }
+  VertexingParameters getVertexingParameters() const { return mVrtParams; }
   void setIsGPU(const unsigned char isgpu) { mIsGPU = isgpu; };
   unsigned char getIsGPU() const { return mIsGPU; };
   void dumpVertexerTraits();

Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx

@@ -22,6 +22,8 @@ std::string asString(TrackingMode mode)
       return "async";
     case TrackingMode::Cosmics:
       return "cosmics";
+    case TrackingMode::Unset:
+      return "unset";
   }
   return "unknown";
 }

Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx

@@ -455,6 +455,7 @@ void Tracker::rectifyClusterIndices()
 void Tracker::getGlobalConfiguration()
 {
   auto& tc = o2::its::TrackerParamConfig::Instance();
+  tc.printKeyValues(true, true);
   if (tc.useMatCorrTGeo) {
     mTraits->setCorrType(o2::base::PropagatorImpl<float>::MatCorrType::USEMatCorrTGeo);
   } else if (tc.useFastMaterial) {

Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx

@@ -22,6 +22,7 @@
 #include "DataFormatsTRD/TriggerRecord.h"
 #include "CommonDataFormat/IRFrame.h"
 #include "DetectorsBase/GRPGeomHelper.h"
+#include "ITStracking/TrackingConfigParam.h"
 
 namespace o2
 {
@@ -33,9 +34,11 @@ void ITSTrackingInterface::initialise()
   mRunVertexer = true;
   mCosmicsProcessing = false;
   std::vector<TrackingParameters> trackParams;
-
+  if (mMode == TrackingMode::Unset) {
+    mMode = (TrackingMode)(o2::its::TrackerParamConfig::Instance().trackingMode);
+    LOGP(info, "Tracking mode not set, trying to fetch it from configurable params to: {}", asString(mMode));
+  }
   if (mMode == TrackingMode::Async) {
-
     trackParams.resize(3);
     for (auto& param : trackParams) {
       param.ZBins = 64;
@@ -49,7 +52,6 @@ void ITSTrackingInterface::initialise()
     trackParams[2].CellDeltaTanLambdaSigma *= 4.;
     trackParams[2].MinTrackLength = 4;
     LOG(info) << "Initializing tracker in async. phase reconstruction with " << trackParams.size() << " passes";
-
   } else if (mMode == TrackingMode::Sync) {
     trackParams.resize(1);
     trackParams[0].ZBins = 64;
@@ -195,10 +197,7 @@ void ITSTrackingInterface::run(framework::ProcessingContext& pc)
         }
       }
       if (processingMask[iRof] && !selROF) { // passed selection in clusters and not in vertex multiplicity
-        LOG(debug) << fmt::format("ROF {} rejected by the vertex multiplicity selection [{},{}]",
-                                  iRof,
-                                  multEstConf.cutMultVtxLow,
-                                  multEstConf.cutMultVtxHigh);
+        LOGP(info, "ROF {} rejected by the vertex multiplicity selection [{},{}]", iRof, multEstConf.cutMultVtxLow, multEstConf.cutMultVtxHigh);
         processingMask[iRof] = selROF;
         cutVertexMult++;
       }
@@ -325,7 +324,7 @@ void ITSTrackingInterface::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
     return;
   }
   if (matcher == ConcreteDataMatcher("GLO", "MEANVERTEX", 0)) {
-    LOGP(info, "mean vertex acquired");
+    LOGP(info, "Mean vertex acquired");
     setMeanVertex((const o2::dataformats::MeanVertexObject*)obj);
     return;
   }

Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx

@@ -67,6 +67,7 @@ float Vertexer::clustersToVerticesHybrid(std::function<void(std::string s)> logg
 void Vertexer::getGlobalConfiguration()
 {
   auto& vc = o2::its::VertexerParamConfig::Instance();
+  vc.printKeyValues(true, true);
   auto& grc = o2::its::GpuRecoParamConfig::Instance();
 
   VertexingParameters verPar;
@@ -90,8 +91,7 @@ void Vertexer::getGlobalConfiguration()
   verPar.PhiBins = vc.PhiBins;
 
   TimeFrameGPUParameters tfGPUpar;
-  tfGPUpar.maxGPUMemoryGB = grc.maxGPUMemoryGB;
-  tfGPUpar.maxVerticesCapacity = grc.maxVerticesCapacity;
+  // tfGPUpar.nROFsPerChunk = grc.nROFsPerChunk;
 
   mTraits->updateVertexingParameters(verPar, tfGPUpar);
 }

Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h

@@ -23,13 +23,11 @@
 
 #include "ITStracking/TrackingInterface.h"
 
-#include "GPUO2Interface.h"
-#include "GPUReconstruction.h"
-#include "GPUChainITS.h"
-#include "CommonUtils/StringUtils.h"
-#include "TStopwatch.h"
+#include "GPUDataTypes.h"
 #include "DetectorsBase/GRPGeomHelper.h"
 
+#include "TStopwatch.h"
+
 namespace o2
 {
 namespace its
@@ -41,9 +39,9 @@ class TrackerDPL : public framework::Task
   TrackerDPL(std::shared_ptr<o2::base::GRPGeomRequest> gr,
              bool isMC,
              int trgType,
-             const TrackingMode& trMode,
+             const TrackingMode& trMode = TrackingMode::Unset,
              const bool overrBeamEst = false,
-             o2::gpu::GPUDataTypes::DeviceType dType = o2::gpu::GPUDataTypes::DeviceType::CPU);
+             gpu::GPUDataTypes::DeviceType dType = gpu::GPUDataTypes::DeviceType::CPU);
   ~TrackerDPL() override = default;
   void init(framework::InitContext& ic) final;
   void run(framework::ProcessingContext& pc) final;
@@ -60,7 +58,8 @@ class TrackerDPL : public framework::Task
   TStopwatch mTimer;
 };
 
-framework::DataProcessorSpec getTrackerSpec(bool useMC, bool useGeom, int useTrig, const std::string& trModeS, const bool overrBeamEst, o2::gpu::GPUDataTypes::DeviceType dType);
+using o2::its::TrackingMode;
+framework::DataProcessorSpec getTrackerSpec(bool useMC, bool useGeom, int useTrig, const std::string& trMode, const bool overrBeamEst = false, gpu::GPUDataTypes::DeviceType dType = gpu::GPUDataTypes::DeviceType::CPU);
 
 } // namespace its
 } // namespace o2

Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx

@@ -31,8 +31,6 @@ namespace o2
 {
 namespace its
 {
-// using ITSTrackReconstruction::TrackingMode;
-
 namespace reco_workflow
 {
 
@@ -65,7 +63,6 @@ framework::WorkflowSpec getWorkflow(bool useMC,
         cfg.runITSTracking = true;
         cfg.itsTriggerType = useTrig;
         cfg.itsOverrBeamEst = overrideBeamPosition;
-        cfg.itsTrackingMode = trmode == "sync" ? (int)TrackingMode::Sync : (trmode == "async" ? (int)TrackingMode::Async : (int)TrackingMode::Cosmics);
 
         Inputs ggInputs;
         auto ggRequest = std::make_shared<o2::base::GRPGeomRequest>(false, true, false, true, true,

Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx

@@ -30,8 +30,9 @@ TrackerDPL::TrackerDPL(std::shared_ptr<o2::base::GRPGeomRequest> gr,
                        const bool overrBeamEst,
                        o2::gpu::GPUDataTypes::DeviceType dType) : mGGCCDBRequest(gr),
                                                                   mRecChain{o2::gpu::GPUReconstruction::CreateInstance(dType, true)},
-                                                                  mITSTrackingInterface{isMC, trgType, trMode, overrBeamEst}
+                                                                  mITSTrackingInterface{isMC, trgType, overrBeamEst}
 {
+  mITSTrackingInterface.setTrackingMode(trMode);
 }
 
 void TrackerDPL::init(InitContext& ic)

GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h

@@ -127,7 +127,6 @@ class GPURecoWorkflowSpec : public o2::framework::Task
     int lumiScaleType = 0; // 0=off, 1=CTP, 2=TPC scalers
     bool outputErrorQA = false;
     bool runITSTracking = false;
-    int itsTrackingMode = 0; // 0=sync, 1=async, 2=cosmics
     bool itsOverrBeamEst = false;
     bool tpcTriggerHandling = false;
   };

GPU/Workflow/src/GPUWorkflowITS.cxx

@@ -37,7 +37,6 @@ void GPURecoWorkflowSpec::initFunctionITS(o2::framework::InitContext& ic)
   o2::its::TrackerTraits* trkTraits = nullptr;
   mITSTrackingInterface = std::make_unique<o2::its::ITSTrackingInterface>(mSpecConfig.processMC,
                                                                           mSpecConfig.itsTriggerType,
-                                                                          static_cast<o2::its::TrackingMode>(mSpecConfig.itsTrackingMode),
                                                                           mSpecConfig.itsOverrBeamEst);
   mGPUReco->GetITSTraits(trkTraits, vtxTraits, mITSTimeFrame);
   mITSTrackingInterface->setTraitsFromProvider(vtxTraits, trkTraits, mITSTimeFrame);

GPU/Workflow/src/gpu-reco-workflow.cxx

@@ -107,6 +107,7 @@ enum struct ioType { Digits,
                      SendClustersPerSector,
                      ITSClusters,
                      ITSTracks,
+                     MeanVertex,
                      TPCTriggers };
 
 static const std::unordered_map<std::string, ioType> InputMap{
@@ -117,7 +118,9 @@ static const std::unordered_map<std::string, ioType> InputMap{
   {"compressed-clusters-root", ioType::CompClustROOT},
   {"compressed-clusters-ctf", ioType::CompClustCTF},
   {"trd-tracklets", ioType::TRDTracklets},
-  {"its-clusters", ioType::ITSClusters}};
+  {"its-clusters", ioType::ITSClusters},
+  {"its-mean-vertex", ioType::MeanVertex},
+};
 
 static const std::unordered_map<std::string, ioType> OutputMap{
   {"clusters", ioType::Clusters},
@@ -183,6 +186,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   cfg.enableDoublePipeline = cfgc.options().get<bool>("enableDoublePipeline");
   cfg.tpcDeadMapSources = cfgc.options().get<int>("tpc-deadMap-sources");
   cfg.runITSTracking = isEnabled(outputTypes, ioType::ITSTracks);
+  cfg.itsOverrBeamEst = isEnabled(inputTypes, ioType::MeanVertex);
 
   Inputs ggInputs;
   auto ggRequest = std::make_shared<o2::base::GRPGeomRequest>(false, true, false, true, true, o2::base::GRPGeomRequest::Aligned, ggInputs, true);