Skip to content

Commit

Permalink
Make compilation flags for duplicate cleaning and triplet pLSs into r…
Browse files Browse the repository at this point in the history 
…untime flags (TrackLooper PR#408 + cmssw PR#31)
  • Loading branch information
@VourMa
VourMa committed Jun 7, 2024
1 parent 314e95d commit 4747b41
Show file tree
Hide file tree
Showing 14 changed files with 89 additions and 84 deletions.
14 changes: 10 additions & 4 deletions RecoTracker/LST/plugins/alpaka/LSTProducer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
lstPhase2OTHitsInputToken_{
consumes<LSTPhase2OTHitsInput>(config.getParameter<edm::InputTag>("phase2OTHitsInput"))},
lstESToken_{esConsumes()},
verbose_(config.getParameter<int>("verbose")),
verbose_(config.getParameter<bool>("verbose")),
nopLSDupClean_(config.getParameter<bool>("nopLSDupClean")),
tcpLSTriplets_(config.getParameter<bool>("tcpLSTriplets")),
lstOutputToken_{produces()} {}

void acquire(device::Event const& event, device::EventSetup const& setup) override {
Expand Down Expand Up @@ -61,7 +63,9 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
phase2OTHits.detId(),
phase2OTHits.x(),
phase2OTHits.y(),
phase2OTHits.z());
phase2OTHits.z(),
nopLSDupClean_,
tcpLSTriplets_);
}

void produce(device::Event& event, device::EventSetup const&) override {
Expand All @@ -76,15 +80,17 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
edm::ParameterSetDescription desc;
desc.add<edm::InputTag>("pixelSeedInput", edm::InputTag{"lstPixelSeedInputProducer"});
desc.add<edm::InputTag>("phase2OTHitsInput", edm::InputTag{"lstPhase2OTHitsInputProducer"});
desc.add<int>("verbose", 0);
desc.add<bool>("verbose", false);
desc.add<bool>("nopLSDupClean", false);
desc.add<bool>("tcpLSTriplets", false);
descriptions.addWithDefaultLabel(desc);
}

private:
edm::EDGetTokenT<LSTPixelSeedInput> lstPixelSeedInputToken_;
edm::EDGetTokenT<LSTPhase2OTHitsInput> lstPhase2OTHitsInputToken_;
device::ESGetToken<SDL::LSTESDeviceData<SDL::Dev>, TrackerRecoGeometryRecord> lstESToken_;
const int verbose_;
const bool verbose_, nopLSDupClean_, tcpLSTriplets_;
edm::EDPutTokenT<LSTOutput> lstOutputToken_;

SDL::LST<SDL::Acc> lst_;
Expand Down
4 changes: 3 additions & 1 deletion RecoTracker/LSTCore/interface/alpaka/LST.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ namespace SDL {
const std::vector<unsigned int> ph2_detId,
const std::vector<float> ph2_x,
const std::vector<float> ph2_y,
const std::vector<float> ph2_z);
const std::vector<float> ph2_z,
bool no_pls_dupclean,
bool tc_pls_triplets);
std::vector<std::vector<unsigned int>> hits() { return out_tc_hitIdxs_; }
std::vector<unsigned int> len() { return out_tc_len_; }
std::vector<int> seedIdx() { return out_tc_seedIdx_; }
Expand Down
49 changes: 25 additions & 24 deletions RecoTracker/LSTCore/src/alpaka/Event.dev.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ void SDL::Event<SDL::Acc>::createTriplets() {
}
}

void SDL::Event<SDL::Acc>::createTrackCandidates() {
void SDL::Event<SDL::Acc>::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
if (trackCandidatesInGPU == nullptr) {
trackCandidatesInGPU = new SDL::trackCandidates();
trackCandidatesBuffers = new SDL::trackCandidatesBuffer<Dev>(
Expand Down Expand Up @@ -709,18 +709,18 @@ void SDL::Event<SDL::Acc>::createTrackCandidates() {

alpaka::enqueue(queue, addT5asTrackCandidateInGPUTask);

#ifndef NOPLSDUPCLEAN
Vec const threadsPerBlockCheckHitspLS = createVec(1, 16, 16);
Vec const blocksPerGridCheckHitspLS = createVec(1, MAX_BLOCKS * 4, MAX_BLOCKS / 4);
WorkDiv const checkHitspLS_workDiv =
createWorkDiv(blocksPerGridCheckHitspLS, threadsPerBlockCheckHitspLS, elementsPerThread);
if (!no_pls_dupclean) {
Vec const threadsPerBlockCheckHitspLS = createVec(1, 16, 16);
Vec const blocksPerGridCheckHitspLS = createVec(1, MAX_BLOCKS * 4, MAX_BLOCKS / 4);
WorkDiv const checkHitspLS_workDiv =
createWorkDiv(blocksPerGridCheckHitspLS, threadsPerBlockCheckHitspLS, elementsPerThread);

SDL::checkHitspLS checkHitspLS_kernel;
auto const checkHitspLSTask(alpaka::createTaskKernel<Acc>(
checkHitspLS_workDiv, checkHitspLS_kernel, *modulesBuffers_->data(), *segmentsInGPU, true));
SDL::checkHitspLS checkHitspLS_kernel;
auto const checkHitspLSTask(alpaka::createTaskKernel<Acc>(
checkHitspLS_workDiv, checkHitspLS_kernel, *modulesBuffers_->data(), *segmentsInGPU, true));

alpaka::enqueue(queue, checkHitspLSTask);
#endif
alpaka::enqueue(queue, checkHitspLSTask);
}

Vec const threadsPerBlock_crossCleanpLS = createVec(1, 16, 32);
Vec const blocksPerGrid_crossCleanpLS = createVec(1, 4, 20);
Expand Down Expand Up @@ -751,7 +751,8 @@ void SDL::Event<SDL::Acc>::createTrackCandidates() {
addpLSasTrackCandidateInGPU_kernel,
nLowerModules_,
*trackCandidatesInGPU,
*segmentsInGPU));
*segmentsInGPU,
tc_pls_triplets));

alpaka::enqueue(queue, addpLSasTrackCandidateInGPUTask);

Expand Down Expand Up @@ -990,20 +991,20 @@ void SDL::Event<SDL::Acc>::createQuintuplets() {
}
}

void SDL::Event<SDL::Acc>::pixelLineSegmentCleaning() {
#ifndef NOPLSDUPCLEAN
Vec const threadsPerBlockCheckHitspLS = createVec(1, 16, 16);
Vec const blocksPerGridCheckHitspLS = createVec(1, MAX_BLOCKS * 4, MAX_BLOCKS / 4);
WorkDiv const checkHitspLS_workDiv =
createWorkDiv(blocksPerGridCheckHitspLS, threadsPerBlockCheckHitspLS, elementsPerThread);
void SDL::Event<SDL::Acc>::pixelLineSegmentCleaning(bool no_pls_dupclean) {
if (!no_pls_dupclean) {
Vec const threadsPerBlockCheckHitspLS = createVec(1, 16, 16);
Vec const blocksPerGridCheckHitspLS = createVec(1, MAX_BLOCKS * 4, MAX_BLOCKS / 4);
WorkDiv const checkHitspLS_workDiv =
createWorkDiv(blocksPerGridCheckHitspLS, threadsPerBlockCheckHitspLS, elementsPerThread);

SDL::checkHitspLS checkHitspLS_kernel;
auto const checkHitspLSTask(alpaka::createTaskKernel<Acc>(
checkHitspLS_workDiv, checkHitspLS_kernel, *modulesBuffers_->data(), *segmentsInGPU, false));
SDL::checkHitspLS checkHitspLS_kernel;
auto const checkHitspLSTask(alpaka::createTaskKernel<Acc>(
checkHitspLS_workDiv, checkHitspLS_kernel, *modulesBuffers_->data(), *segmentsInGPU, false));

alpaka::enqueue(queue, checkHitspLSTask);
alpaka::wait(queue);
#endif
alpaka::enqueue(queue, checkHitspLSTask);
alpaka::wait(queue);
}
}

void SDL::Event<SDL::Acc>::createPixelQuintuplets() {
Expand Down
4 changes: 2 additions & 2 deletions RecoTracker/LSTCore/src/alpaka/Event.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,12 @@ namespace SDL {
void createTriplets();
void createPixelTracklets();
void createPixelTrackletsWithMap();
void createTrackCandidates();
void createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets);
void createExtendedTracks();
void createQuintuplets();
void createPixelTriplets();
void createPixelQuintuplets();
void pixelLineSegmentCleaning();
void pixelLineSegmentCleaning(bool no_pls_dupclean);

unsigned int getNumberOfHits();
unsigned int getNumberOfHitsByLayer(unsigned int layer);
Expand Down
8 changes: 5 additions & 3 deletions RecoTracker/LSTCore/src/alpaka/LST.dev.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ void SDL::LST<SDL::Acc>::run(SDL::QueueAcc& queue,
const std::vector<unsigned int> ph2_detId,
const std::vector<float> ph2_x,
const std::vector<float> ph2_y,
const std::vector<float> ph2_z) {
const std::vector<float> ph2_z,
bool no_pls_dupclean,
bool tc_pls_triplets) {
auto event = SDL::Event<Acc>(verbose, queue, deviceESData);
prepareInput(see_px,
see_py,
Expand Down Expand Up @@ -135,7 +137,7 @@ void SDL::LST<SDL::Acc>::run(SDL::QueueAcc& queue,
printf("# of Quintuplets produced endcap layer 5: %d\n", event.getNumberOfQuintupletsByLayerEndcap(4));
}

event.pixelLineSegmentCleaning();
event.pixelLineSegmentCleaning(no_pls_dupclean);

event.createPixelQuintuplets();
if (verbose)
Expand All @@ -145,7 +147,7 @@ void SDL::LST<SDL::Acc>::run(SDL::QueueAcc& queue,
if (verbose)
printf("# of Pixel T3s produced: %d\n", event.getNumberOfPixelTriplets());

event.createTrackCandidates();
event.createTrackCandidates(no_pls_dupclean, tc_pls_triplets);
if (verbose) {
printf("# of TrackCandidates produced: %d\n", event.getNumberOfTrackCandidates());
printf(" # of Pixel TrackCandidates produced: %d\n", event.getNumberOfPixelTrackCandidates());
Expand Down
9 changes: 3 additions & 6 deletions RecoTracker/LSTCore/src/alpaka/TrackCandidate.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,18 +491,15 @@ namespace SDL {
ALPAKA_FN_ACC void operator()(TAcc const& acc,
uint16_t nLowerModules,
struct SDL::trackCandidates trackCandidatesInGPU,
struct SDL::segments segmentsInGPU) const {
struct SDL::segments segmentsInGPU,
bool tc_pls_triplets) const {
auto const globalThreadIdx = alpaka::getIdx<alpaka::Grid, alpaka::Threads>(acc);
auto const gridThreadExtent = alpaka::getWorkDiv<alpaka::Grid, alpaka::Threads>(acc);

unsigned int nPixels = segmentsInGPU.nSegments[nLowerModules];
for (unsigned int pixelArrayIndex = globalThreadIdx[2]; pixelArrayIndex < nPixels;
pixelArrayIndex += gridThreadExtent[2]) {
#ifdef TC_PLS_TRIPLETS
if (segmentsInGPU.isDup[pixelArrayIndex])
#else
if ((!segmentsInGPU.isQuad[pixelArrayIndex]) || (segmentsInGPU.isDup[pixelArrayIndex]))
#endif
if ((tc_pls_triplets ? 0 : !segmentsInGPU.isQuad[pixelArrayIndex]) || (segmentsInGPU.isDup[pixelArrayIndex]))
continue;

unsigned int trackCandidateIdx =
Expand Down
6 changes: 3 additions & 3 deletions RecoTracker/LSTCore/standalone/SDL/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,13 @@ CUTVALUEFLAG =
CUTVALUEFLAG_FLAGS = -DCUT_VALUE_DEBUG

%_cpu.o: ../../src/alpaka/%.dev.cc
$(COMPILE_CMD_CPU) $(CXXFLAGS_CPU) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_CPU) $(T5CUTFLAGS) $(NOPLSDUPCLEANFLAG) $(TCPLSTRIPLETSFLAG) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_CPU) $< -o $@
$(COMPILE_CMD_CPU) $(CXXFLAGS_CPU) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_CPU) $(T5CUTFLAGS) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_CPU) $< -o $@

%_cuda.o: ../../src/alpaka/%.dev.cc
$(COMPILE_CMD_CUDA) $(CXXFLAGS_CUDA) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_CUDA) $(T5CUTFLAGS) $(NOPLSDUPCLEANFLAG) $(TCPLSTRIPLETSFLAG) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_CUDA) $< -o $@
$(COMPILE_CMD_CUDA) $(CXXFLAGS_CUDA) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_CUDA) $(T5CUTFLAGS) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_CUDA) $< -o $@

%_rocm.o: ../../src/alpaka/%.dev.cc
$(COMPILE_CMD_ROCM) $(CXXFLAGS_ROCM) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_ROCM) $(T5CUTFLAGS) $(NOPLSDUPCLEANFLAG) $(TCPLSTRIPLETSFLAG) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_ROCM) $< -o $@
$(COMPILE_CMD_ROCM) $(CXXFLAGS_ROCM) $(ROOTINCLUDE) $(PRINTFLAG) $(CACHEFLAG) $(CUTVALUEFLAG) $(LSTWARNINGSFLAG) $(CMSSW_WERRORS_ROCM) $(T5CUTFLAGS) $(PTCUTFLAG) $(DUPLICATES) $(ALPAKAINCLUDE) $(ALPAKABACKEND_ROCM) $< -o $@

$(LIB_CPU): $(CCOBJECTS_CPU) $(LSTOBJECTS_CPU)
$(LD_CPU) $(SOFLAGS_CPU) $^ -o $@
Expand Down
23 changes: 18 additions & 5 deletions RecoTracker/LSTCore/standalone/bin/sdl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,18 +52,21 @@ int main(int argc, char **argv) {
"N,nmatch", "N match for MTV-like matching", cxxopts::value<int>()->default_value("9"))(
"n,nevents", "N events to loop over", cxxopts::value<int>()->default_value("-1"))(
"x,event_index", "specific event index to process", cxxopts::value<int>()->default_value("-1"))(
"g,pdg_id", "The simhit pdgId match option (default = 0)", cxxopts::value<int>()->default_value("0"))(
"g,pdg_id", "The simhit pdgId match option", cxxopts::value<int>()->default_value("0"))(
"v,verbose",
"Verbose mode (0: no print, 1: only final timing, 2: object multiplitcity",
cxxopts::value<int>()->default_value("0"))(
"w,write_ntuple", "Write Ntuple", cxxopts::value<int>()->default_value("1"))(
"s,streams", "Set number of streams (default=1)", cxxopts::value<int>()->default_value("1"))(
"s,streams", "Set number of streams", cxxopts::value<int>()->default_value("1"))(
"d,debug", "Run debug job. i.e. overrides output option to 'debug.root' and 'recreate's the file.")(
"l,lower_level", "write lower level objects ntuple results")("G,gnn_ntuple", "write gnn input variable ntuple")(
"j,nsplit_jobs", "Enable splitting jobs by N blocks (--job_index must be set)", cxxopts::value<int>())(
"I,job_index",
"job_index of split jobs (--nsplit_jobs must be set. index starts from 0. i.e. 0, 1, 2, 3, etc...)",
cxxopts::value<int>())("h,help", "Print help");
cxxopts::value<int>())(
"3,tc_pls_triplets" , "Allow triplet pLSs in TC collection")(
"2,no_pls_dupclean" , "Disable pLS duplicate cleaning (both steps)")(
"h,help", "Print help");

auto result = options.parse(argc, argv);

Expand Down Expand Up @@ -242,6 +245,14 @@ int main(int argc, char **argv) {
ana.gnn_ntuple = false;
}

//_______________________________________________________________________________
// --tc_pls_triplets
ana.tc_pls_triplets = result["tc_pls_triplets"].as<bool>();

//_______________________________________________________________________________
// --no_pls_dupclean
ana.no_pls_dupclean = result["no_pls_dupclean"].as<bool>();

// Printing out the option settings overview
std::cout << "=========================================================" << std::endl;
std::cout << " Running for Acc = " << alpaka::getAccName<SDL::Acc>() << std::endl;
Expand All @@ -258,6 +269,8 @@ int main(int argc, char **argv) {
std::cout << " ana.streams: " << ana.streams << std::endl;
std::cout << " ana.verbose: " << ana.verbose << std::endl;
std::cout << " ana.nmatch_threshold: " << ana.nmatch_threshold << std::endl;
std::cout << " ana.tc_pls_triplets: " << ana.tc_pls_triplets << std::endl;
std::cout << " ana.no_pls_dupclean: " << ana.no_pls_dupclean << std::endl;
std::cout << "=========================================================" << std::endl;

// Create the TChain that holds the TTree's of the baby ntuples
Expand Down Expand Up @@ -430,10 +443,10 @@ void run_sdl() {
timing_LS = runSegment(events.at(omp_get_thread_num()));
timing_T3 = runT3(events.at(omp_get_thread_num()));
timing_T5 = runQuintuplet(events.at(omp_get_thread_num()));
timing_pLS = runPixelLineSegment(events.at(omp_get_thread_num()));
timing_pLS = runPixelLineSegment(events.at(omp_get_thread_num()),ana.no_pls_dupclean);
timing_pT5 = runPixelQuintuplet(events.at(omp_get_thread_num()));
timing_pT3 = runpT3(events.at(omp_get_thread_num()));
timing_TC = runTrackCandidate(events.at(omp_get_thread_num()));
timing_TC = runTrackCandidate(events.at(omp_get_thread_num()),ana.no_pls_dupclean,ana.tc_pls_triplets);

if (ana.verbose == 4) {
#pragma omp critical
Expand Down
4 changes: 0 additions & 4 deletions RecoTracker/LSTCore/standalone/bin/sdl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@
#include "rooutil.h"
#include "cxxopts.h"

#include "Event.h" // SDL::Event
#include "Module.h" // SDL::Module
#include "EndcapGeometry.h" // SDL::EndcapGeometr
#include "ModuleConnectionMap.h" // SDL::ModuleConnectionMap
#include "Event.h"
#include "LST.h"

Expand Down
Loading

0 comments on commit 4747b41

Please sign in to comment.