diff --git a/Makefile b/Makefile index 3b91ca81..8a64dd8f 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,10 @@ OBJECTS=$(OBJECTS_CPU) $(OBJECTS_CUDA) $(OBJECTS_ROCM) CXX = g++ CXXFLAGS = -g -O2 -Wall -fPIC -Wshadow -Woverloaded-virtual -Wno-unused-function -fno-var-tracking -std=c++17 -INCLUDEFLAGS= -ISDL -I$(shell pwd) -Icode -Icode/core -I${ALPAKA_ROOT}/include -I/${BOOST_ROOT}/include $(shell rooutil-config --include) -I$(shell root-config --incdir) +INCLUDEFLAGS= -ISDL -I$(shell pwd) -Icode -Icode/core -I${ALPAKA_ROOT}/include -I/${BOOST_ROOT}/include $(shell rooutil-config --include) -I$(shell root-config --incdir) -I${CMSSW_BASE}/src +ifdef CMSSW_RELEASE_BASE +INCLUDEFLAGS:= ${INCLUDEFLAGS} -I${CMSSW_RELEASE_BASE}/src +endif LDFLAGS = -g -O2 $(SDLLIB) -L${TRACKLOOPERDIR}/SDL $(shell rooutil-config --libs) $(shell root-config --libs) LDFLAGS_CUDA= -L${CUDA_HOME}/lib64 -lcudart LDFLAGS_ROCM= -L${ROCM_ROOT}/lib -lamdhip64 diff --git a/SDL/Constants.h b/SDL/Constants.h index 60f255c0..da4dd3fb 100644 --- a/SDL/Constants.h +++ b/SDL/Constants.h @@ -3,6 +3,8 @@ #include +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" + #ifdef CACHE_ALLOC #include "HeterogeneousCore/AlpakaInterface/interface/CachedBufAlloc.h" #endif @@ -23,33 +25,19 @@ namespace SDL { typedef float FPX; #endif - using Idx = std::size_t; - using Dim = alpaka::DimInt<3u>; - using Dim1d = alpaka::DimInt<1u>; - using Vec = alpaka::Vec; - using Vec1d = alpaka::Vec; -#if defined(ALPAKA_ACC_GPU_CUDA_ENABLED) || defined(ALPAKA_ACC_GPU_HIP_ENABLED) - using QueueProperty = alpaka::NonBlocking; -#else - using QueueProperty = alpaka::Blocking; -#endif - using WorkDiv = alpaka::WorkDivMembers; + using Idx = alpaka_common::Idx; + using Dim = alpaka_common::Dim3D; + using Dim1d = alpaka_common::Dim1D; + using Vec = alpaka_common::Vec3D; + using Vec1d = alpaka_common::Vec1D; + using WorkDiv = alpaka_common::WorkDiv3D; - Vec const elementsPerThread(Vec::all(static_cast(1))); + using Acc = ALPAKA_ACCELERATOR_NAMESPACE::Acc3D; + using Dev = ALPAKA_ACCELERATOR_NAMESPACE::Device; + using DevHost = ALPAKA_ACCELERATOR_NAMESPACE::DevHost; + using QueueAcc = ALPAKA_ACCELERATOR_NAMESPACE::Queue; -// - AccGpuCudaRt -// - AccCpuThreads -// - AccCpuSerial -#ifdef ALPAKA_ACC_GPU_CUDA_ENABLED - using Acc = alpaka::AccGpuCudaRt; -#elif ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED - using Acc = alpaka::AccCpuThreads; -#elif ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED - using Acc = alpaka::AccCpuSerial; -#elif ALPAKA_ACC_GPU_HIP_ENABLED - using Acc = alpaka::AccGpuHipRt; -#endif - using Dev = alpaka::Dev; + Vec const elementsPerThread(Vec::all(static_cast(1))); // Needed for files that are compiled by g++ to not throw an error. // uint4 is defined only for CUDA, so we will have to revisit this soon when running on other backends. @@ -62,15 +50,6 @@ namespace SDL { }; #endif - auto const platformAcc = alpaka::Platform{}; - auto const platformHost = alpaka::Platform{}; - auto const devHost = alpaka::getDevByIdx(platformHost, 0u); -#if defined ALPAKA_ACC_GPU_CUDA_ENABLED || defined ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED || \ - defined ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED || defined ALPAKA_ACC_GPU_HIP_ENABLED - auto const devAcc = alpaka::getDevByIdx(platformAcc, 0u); - using QueueAcc = alpaka::Queue; -#endif - // Buffer type for allocations where auto type can't be used. template using Buf = alpaka::Buf; diff --git a/SDL/EndcapGeometry.cc b/SDL/EndcapGeometry.cc index e9c1967d..27acc8a5 100644 --- a/SDL/EndcapGeometry.cc +++ b/SDL/EndcapGeometry.cc @@ -1,16 +1,19 @@ #include "EndcapGeometry.h" -SDL::EndcapGeometry::EndcapGeometry(unsigned int sizef) - : geoMapDetId_buf(allocBufWrapper(devAcc, sizef)), - geoMapPhi_buf(allocBufWrapper(devAcc, sizef)) {} - -SDL::EndcapGeometry::EndcapGeometry(std::string filename, unsigned int sizef) - : geoMapDetId_buf(allocBufWrapper(devAcc, sizef)), - geoMapPhi_buf(allocBufWrapper(devAcc, sizef)) { - load(filename); +SDL::EndcapGeometry::EndcapGeometry(SDL::Dev const& devAccIn, unsigned int sizef) + : geoMapDetId_buf(allocBufWrapper(devAccIn, sizef)), + geoMapPhi_buf(allocBufWrapper(devAccIn, sizef)) {} + +SDL::EndcapGeometry::EndcapGeometry(SDL::Dev const& devAccIn, + SDL::QueueAcc& queue, + std::string filename, + unsigned int sizef) + : geoMapDetId_buf(allocBufWrapper(devAccIn, sizef)), + geoMapPhi_buf(allocBufWrapper(devAccIn, sizef)) { + load(queue, filename); } -void SDL::EndcapGeometry::load(std::string filename) { +void SDL::EndcapGeometry::load(SDL::QueueAcc& queue, std::string filename) { dxdy_slope_.clear(); centroid_phis_.clear(); @@ -30,13 +33,11 @@ void SDL::EndcapGeometry::load(std::string filename) { } } - fillGeoMapArraysExplicit(); + fillGeoMapArraysExplicit(queue); } -void SDL::EndcapGeometry::fillGeoMapArraysExplicit() { - QueueAcc queue(devAcc); - - int phi_size = centroid_phis_.size(); +void SDL::EndcapGeometry::fillGeoMapArraysExplicit(SDL::QueueAcc& queue) { + unsigned int phi_size = centroid_phis_.size(); // Temporary check for endcap initialization. if (phi_size != endcap_size) { @@ -47,6 +48,7 @@ void SDL::EndcapGeometry::fillGeoMapArraysExplicit() { } // Allocate buffers on host + SDL::DevHost const& devHost = cms::alpakatools::host(); auto mapPhi_host_buf = allocBufWrapper(devHost, phi_size); auto mapDetId_host_buf = allocBufWrapper(devHost, phi_size); @@ -66,8 +68,8 @@ void SDL::EndcapGeometry::fillGeoMapArraysExplicit() { nEndCapMap = counter; // Copy data from host to device buffers - alpaka::memcpy(queue, geoMapPhi_buf, mapPhi_host_buf, phi_size); - alpaka::memcpy(queue, geoMapDetId_buf, mapDetId_host_buf, phi_size); + alpaka::memcpy(queue, geoMapPhi_buf, mapPhi_host_buf); + alpaka::memcpy(queue, geoMapDetId_buf, mapDetId_host_buf); alpaka::wait(queue); } diff --git a/SDL/EndcapGeometry.h b/SDL/EndcapGeometry.h index eb9e4806..df9d7ef7 100644 --- a/SDL/EndcapGeometry.h +++ b/SDL/EndcapGeometry.h @@ -11,6 +11,8 @@ #include "Constants.h" +#include "HeterogeneousCore/AlpakaInterface/interface/host.h" + namespace SDL { template class EndcapGeometry {}; @@ -26,14 +28,13 @@ namespace SDL { unsigned int nEndCapMap; - EndcapGeometry(unsigned int sizef = endcap_size); - EndcapGeometry(std::string filename, unsigned int sizef = endcap_size); + EndcapGeometry(Dev const& devAccIn, unsigned int sizef = endcap_size); + EndcapGeometry(Dev const& devAccIn, QueueAcc& queue, std::string filename, unsigned int sizef = endcap_size); ~EndcapGeometry() = default; - void load(std::string); + void load(QueueAcc& queue, std::string); - void fillGeoMapArraysExplicit(); - void CreateGeoMapArraysExplicit(); + void fillGeoMapArraysExplicit(QueueAcc& queue); float getdxdy_slope(unsigned int detid); }; } // namespace SDL diff --git a/SDL/Event.cc b/SDL/Event.cc index b998dc5e..6b45aada 100644 --- a/SDL/Event.cc +++ b/SDL/Event.cc @@ -43,9 +43,6 @@ void SDL::Event::init(bool verbose) { } } -// Standalone constructor that has each event object create its own queue. -SDL::Event::Event(bool verbose) : queue(alpaka::getDevByIdx(platformAcc, 0u)) { init(verbose); } - void SDL::Event::resetEvent() { //reset the arrays for (int i = 0; i < 6; i++) { @@ -152,9 +149,9 @@ void SDL::Event::resetEvent() { } } -void SDL::Event::initModules(const MapPLStoLayer& pLStoLayer, const char* moduleMetaDataFilePath) { - QueueAcc queue(devAcc); - +void SDL::Event::initModules(QueueAcc& queue, + const MapPLStoLayer& pLStoLayer, + const char* moduleMetaDataFilePath) { // nModules gets filled here loadModulesFromFile(Globals::modulesBuffers, Globals::nModules, @@ -199,7 +196,7 @@ void SDL::Event::addHitToEvent(std::vector x, alpaka::memcpy(queue, hitsBuffers->zs_buf, z, nHits); alpaka::memcpy(queue, hitsBuffers->detid_buf, detId, nHits); alpaka::memcpy(queue, hitsBuffers->idxs_buf, idxInNtuple, nHits); - alpaka::memcpy(queue, hitsBuffers->nHits_buf, nHits_view, 1); + alpaka::memcpy(queue, hitsBuffers->nHits_buf, nHits_view); alpaka::wait(queue); Vec const threadsPerBlock1 = createVec(1, 1, 256); @@ -445,7 +442,7 @@ void SDL::Event::createMiniDoublets() { auto nTotalMDs_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTotalMDs_buf, rangesBuffers->device_nTotalMDs_buf, 1); + alpaka::memcpy(queue, nTotalMDs_buf, rangesBuffers->device_nTotalMDs_buf); alpaka::wait(queue); unsigned int nTotalMDs = *alpaka::getPtrNative(nTotalMDs_buf); @@ -560,7 +557,7 @@ void SDL::Event::createTriplets() { // TODO: Why are we pulling this back down only to put it back on the device in a new struct? auto maxTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, maxTriplets_buf, rangesBuffers->device_nTotalTrips_buf, 1); + alpaka::memcpy(queue, maxTriplets_buf, rangesBuffers->device_nTotalTrips_buf); alpaka::wait(queue); tripletsInGPU = new SDL::triplets(); @@ -568,7 +565,7 @@ void SDL::Event::createTriplets() { *alpaka::getPtrNative(maxTriplets_buf), Globals::nLowerModules, devAcc, queue); tripletsInGPU->setData(*tripletsBuffers); - alpaka::memcpy(queue, tripletsBuffers->nMemoryLocations_buf, maxTriplets_buf, 1); + alpaka::memcpy(queue, tripletsBuffers->nMemoryLocations_buf, maxTriplets_buf); alpaka::wait(queue); } @@ -663,7 +660,7 @@ void SDL::Event::createTrackCandidates() { // Pull nEligibleT5Modules from the device. auto nEligibleModules_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nEligibleModules_buf, rangesBuffers->nEligibleT5Modules_buf, 1); + alpaka::memcpy(queue, nEligibleModules_buf, rangesBuffers->nEligibleT5Modules_buf); alpaka::wait(queue); uint16_t nEligibleModules = *alpaka::getPtrNative(nEligibleModules_buf); @@ -796,10 +793,10 @@ void SDL::Event::createTrackCandidates() { auto nTrackCanpT3Host_buf = allocBufWrapper(devHost, 1, queue); auto nTrackCanpLSHost_buf = allocBufWrapper(devHost, 1, queue); auto nTrackCanT5Host_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCanpT5Host_buf, trackCandidatesBuffers->nTrackCandidatespT5_buf, 1); - alpaka::memcpy(queue, nTrackCanpT3Host_buf, trackCandidatesBuffers->nTrackCandidatespT3_buf, 1); - alpaka::memcpy(queue, nTrackCanpLSHost_buf, trackCandidatesBuffers->nTrackCandidatespLS_buf, 1); - alpaka::memcpy(queue, nTrackCanT5Host_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf, 1); + alpaka::memcpy(queue, nTrackCanpT5Host_buf, trackCandidatesBuffers->nTrackCandidatespT5_buf); + alpaka::memcpy(queue, nTrackCanpT3Host_buf, trackCandidatesBuffers->nTrackCandidatespT3_buf); + alpaka::memcpy(queue, nTrackCanpLSHost_buf, trackCandidatesBuffers->nTrackCandidatespLS_buf); + alpaka::memcpy(queue, nTrackCanT5Host_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf); alpaka::wait(queue); int nTrackCandidatespT5 = *alpaka::getPtrNative(nTrackCanpT5Host_buf); @@ -836,8 +833,8 @@ void SDL::Event::createPixelTriplets() { auto superbins_buf = allocBufWrapper(devHost, N_MAX_PIXEL_SEGMENTS_PER_MODULE, queue); auto pixelTypes_buf = allocBufWrapper(devHost, N_MAX_PIXEL_SEGMENTS_PER_MODULE, queue); - alpaka::memcpy(queue, superbins_buf, segmentsBuffers->superbin_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, pixelTypes_buf, segmentsBuffers->pixelType_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); + alpaka::memcpy(queue, superbins_buf, segmentsBuffers->superbin_buf); + alpaka::memcpy(queue, pixelTypes_buf, segmentsBuffers->pixelType_buf); alpaka::wait(queue); auto connectedPixelSize_host_buf = allocBufWrapper(devHost, nInnerSegments, queue); @@ -916,7 +913,7 @@ void SDL::Event::createPixelTriplets() { #ifdef Warnings auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1u); + alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf); alpaka::wait(queue); std::cout << "number of pixel triplets = " << *alpaka::getPtrNative(nPixelTriplets_buf) << std::endl; @@ -957,8 +954,8 @@ void SDL::Event::createQuintuplets() { auto nEligibleT5Modules_buf = allocBufWrapper(devHost, 1, queue); auto nTotalQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nEligibleT5Modules_buf, rangesBuffers->nEligibleT5Modules_buf, 1); - alpaka::memcpy(queue, nTotalQuintuplets_buf, rangesBuffers->device_nTotalQuints_buf, 1); + alpaka::memcpy(queue, nEligibleT5Modules_buf, rangesBuffers->nEligibleT5Modules_buf); + alpaka::memcpy(queue, nTotalQuintuplets_buf, rangesBuffers->device_nTotalQuints_buf); alpaka::wait(queue); uint16_t nEligibleT5Modules = *alpaka::getPtrNative(nEligibleT5Modules_buf); @@ -970,7 +967,7 @@ void SDL::Event::createQuintuplets() { new SDL::quintupletsBuffer(nTotalQuintuplets, Globals::nLowerModules, devAcc, queue); quintupletsInGPU->setData(*quintupletsBuffers); - alpaka::memcpy(queue, quintupletsBuffers->nMemoryLocations_buf, nTotalQuintuplets_buf, 1); + alpaka::memcpy(queue, quintupletsBuffers->nMemoryLocations_buf, nTotalQuintuplets_buf); alpaka::wait(queue); } @@ -1070,8 +1067,8 @@ void SDL::Event::createPixelQuintuplets() { auto superbins_buf = allocBufWrapper(devHost, N_MAX_PIXEL_SEGMENTS_PER_MODULE, queue); auto pixelTypes_buf = allocBufWrapper(devHost, N_MAX_PIXEL_SEGMENTS_PER_MODULE, queue); - alpaka::memcpy(queue, superbins_buf, segmentsBuffers->superbin_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, pixelTypes_buf, segmentsBuffers->pixelType_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); + alpaka::memcpy(queue, superbins_buf, segmentsBuffers->superbin_buf); + alpaka::memcpy(queue, pixelTypes_buf, segmentsBuffers->pixelType_buf); alpaka::wait(queue); auto connectedPixelSize_host_buf = allocBufWrapper(devHost, nInnerSegments, queue); @@ -1179,7 +1176,7 @@ void SDL::Event::createPixelQuintuplets() { #ifdef Warnings auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1u); + alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf); alpaka::wait(queue); std::cout << "number of pixel quintuplets = " << *alpaka::getPtrNative(nPixelQuintuplets_buf) << std::endl; @@ -1199,7 +1196,7 @@ void SDL::Event::addMiniDoubletsToEventExplicit() { queue, module_layers_buf, Globals::modulesBuffersES->layers_buf, Globals::nLowerModules); auto module_hitRanges_buf = allocBufWrapper(devHost, Globals::nLowerModules * 2, queue); - alpaka::memcpy(queue, module_hitRanges_buf, hitsBuffers->hitRanges_buf, Globals::nLowerModules * 2); + alpaka::memcpy(queue, module_hitRanges_buf, hitsBuffers->hitRanges_buf, Globals::nLowerModules * 2u); alpaka::wait(queue); @@ -1250,7 +1247,7 @@ void SDL::Event::addSegmentsToEventExplicit() { void SDL::Event::addQuintupletsToEventExplicit() { auto nQuintupletsCPU_buf = allocBufWrapper(devHost, Globals::nLowerModules, queue); - alpaka::memcpy(queue, nQuintupletsCPU_buf, quintupletsBuffers->nQuintuplets_buf, Globals::nLowerModules); + alpaka::memcpy(queue, nQuintupletsCPU_buf, quintupletsBuffers->nQuintuplets_buf); auto module_subdets_buf = allocBufWrapper(devHost, Globals::nModules, queue); alpaka::memcpy( @@ -1261,10 +1258,7 @@ void SDL::Event::addQuintupletsToEventExplicit() { queue, module_layers_buf, Globals::modulesBuffersES->layers_buf, Globals::nLowerModules); auto module_quintupletModuleIndices_buf = allocBufWrapper(devHost, Globals::nLowerModules, queue); - alpaka::memcpy(queue, - module_quintupletModuleIndices_buf, - rangesBuffers->quintupletModuleIndices_buf, - Globals::nLowerModules); + alpaka::memcpy(queue, module_quintupletModuleIndices_buf, rangesBuffers->quintupletModuleIndices_buf); alpaka::wait(queue); @@ -1286,7 +1280,7 @@ void SDL::Event::addQuintupletsToEventExplicit() { void SDL::Event::addTripletsToEventExplicit() { auto nTripletsCPU_buf = allocBufWrapper(devHost, Globals::nLowerModules, queue); - alpaka::memcpy(queue, nTripletsCPU_buf, tripletsBuffers->nTriplets_buf, Globals::nLowerModules); + alpaka::memcpy(queue, nTripletsCPU_buf, tripletsBuffers->nTriplets_buf); auto module_subdets_buf = allocBufWrapper(devHost, Globals::nLowerModules, queue); alpaka::memcpy( @@ -1423,7 +1417,7 @@ unsigned int SDL::Event::getNumberOfTripletsByLayerEndcap(unsigned int int SDL::Event::getNumberOfPixelTriplets() { auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1); + alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf); alpaka::wait(queue); int nPixelTriplets = *alpaka::getPtrNative(nPixelTriplets_buf); @@ -1434,7 +1428,7 @@ int SDL::Event::getNumberOfPixelTriplets() { int SDL::Event::getNumberOfPixelQuintuplets() { auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1); + alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf); alpaka::wait(queue); int nPixelQuintuplets = *alpaka::getPtrNative(nPixelQuintuplets_buf); @@ -1472,7 +1466,7 @@ unsigned int SDL::Event::getNumberOfQuintupletsByLayerEndcap(unsigned int SDL::Event::getNumberOfTrackCandidates() { auto nTrackCandidates_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidates_buf, trackCandidatesBuffers->nTrackCandidates_buf, 1); + alpaka::memcpy(queue, nTrackCandidates_buf, trackCandidatesBuffers->nTrackCandidates_buf); alpaka::wait(queue); int nTrackCandidates = *alpaka::getPtrNative(nTrackCandidates_buf); @@ -1483,7 +1477,7 @@ int SDL::Event::getNumberOfTrackCandidates() { int SDL::Event::getNumberOfPT5TrackCandidates() { auto nTrackCandidatesPT5_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidatesPT5_buf, trackCandidatesBuffers->nTrackCandidatespT5_buf, 1); + alpaka::memcpy(queue, nTrackCandidatesPT5_buf, trackCandidatesBuffers->nTrackCandidatespT5_buf); alpaka::wait(queue); int nTrackCandidatesPT5 = *alpaka::getPtrNative(nTrackCandidatesPT5_buf); @@ -1494,7 +1488,7 @@ int SDL::Event::getNumberOfPT5TrackCandidates() { int SDL::Event::getNumberOfPT3TrackCandidates() { auto nTrackCandidatesPT3_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidatesPT3_buf, trackCandidatesBuffers->nTrackCandidatespT3_buf, 1); + alpaka::memcpy(queue, nTrackCandidatesPT3_buf, trackCandidatesBuffers->nTrackCandidatespT3_buf); alpaka::wait(queue); int nTrackCandidatesPT3 = *alpaka::getPtrNative(nTrackCandidatesPT3_buf); @@ -1505,7 +1499,7 @@ int SDL::Event::getNumberOfPT3TrackCandidates() { int SDL::Event::getNumberOfPLSTrackCandidates() { auto nTrackCandidatesPLS_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidatesPLS_buf, trackCandidatesBuffers->nTrackCandidatespLS_buf, 1); + alpaka::memcpy(queue, nTrackCandidatesPLS_buf, trackCandidatesBuffers->nTrackCandidatespLS_buf); alpaka::wait(queue); unsigned int nTrackCandidatesPLS = *alpaka::getPtrNative(nTrackCandidatesPLS_buf); @@ -1517,8 +1511,8 @@ int SDL::Event::getNumberOfPixelTrackCandidates() { auto nTrackCandidates_buf = allocBufWrapper(devHost, 1, queue); auto nTrackCandidatesT5_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidates_buf, trackCandidatesBuffers->nTrackCandidates_buf, 1); - alpaka::memcpy(queue, nTrackCandidatesT5_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf, 1); + alpaka::memcpy(queue, nTrackCandidates_buf, trackCandidatesBuffers->nTrackCandidates_buf); + alpaka::memcpy(queue, nTrackCandidatesT5_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf); alpaka::wait(queue); int nTrackCandidates = *alpaka::getPtrNative(nTrackCandidates_buf); @@ -1530,7 +1524,7 @@ int SDL::Event::getNumberOfPixelTrackCandidates() { int SDL::Event::getNumberOfT5TrackCandidates() { auto nTrackCandidatesT5_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCandidatesT5_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf, 1); + alpaka::memcpy(queue, nTrackCandidatesT5_buf, trackCandidatesBuffers->nTrackCandidatesT5_buf); alpaka::wait(queue); int nTrackCandidatesT5 = *alpaka::getPtrNative(nTrackCandidatesT5_buf); @@ -1542,7 +1536,7 @@ SDL::hitsBuffer* SDL::Event::getHits() //std::shared_ { if (hitsInCPU == nullptr) { auto nHits_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nHits_buf, hitsBuffers->nHits_buf, 1); + alpaka::memcpy(queue, nHits_buf, hitsBuffers->nHits_buf); alpaka::wait(queue); unsigned int nHits = *alpaka::getPtrNative(nHits_buf); @@ -1564,7 +1558,7 @@ SDL::hitsBuffer* SDL::Event::getHits() //std::shared_ SDL::hitsBuffer* SDL::Event::getHitsInCMSSW() { if (hitsInCPU == nullptr) { auto nHits_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nHits_buf, hitsBuffers->nHits_buf, 1); + alpaka::memcpy(queue, nHits_buf, hitsBuffers->nHits_buf); alpaka::wait(queue); unsigned int nHits = *alpaka::getPtrNative(nHits_buf); @@ -1584,23 +1578,11 @@ SDL::objectRangesBuffer* SDL::Event::getRanges() { Globals::nModules, Globals::nLowerModules, devHost, queue); rangesInCPU->setData(*rangesInCPU); - alpaka::memcpy(queue, rangesInCPU->hitRanges_buf, rangesBuffers->hitRanges_buf, 2 * Globals::nModules); - alpaka::memcpy(queue, - rangesInCPU->quintupletModuleIndices_buf, - rangesBuffers->quintupletModuleIndices_buf, - Globals::nLowerModules); - alpaka::memcpy(queue, - rangesInCPU->miniDoubletModuleIndices_buf, - rangesBuffers->miniDoubletModuleIndices_buf, - Globals::nLowerModules + 1); - alpaka::memcpy(queue, - rangesInCPU->segmentModuleIndices_buf, - rangesBuffers->segmentModuleIndices_buf, - Globals::nLowerModules + 1); - alpaka::memcpy(queue, - rangesInCPU->tripletModuleIndices_buf, - rangesBuffers->tripletModuleIndices_buf, - Globals::nLowerModules); + alpaka::memcpy(queue, rangesInCPU->hitRanges_buf, rangesBuffers->hitRanges_buf); + alpaka::memcpy(queue, rangesInCPU->quintupletModuleIndices_buf, rangesBuffers->quintupletModuleIndices_buf); + alpaka::memcpy(queue, rangesInCPU->miniDoubletModuleIndices_buf, rangesBuffers->miniDoubletModuleIndices_buf); + alpaka::memcpy(queue, rangesInCPU->segmentModuleIndices_buf, rangesBuffers->segmentModuleIndices_buf); + alpaka::memcpy(queue, rangesInCPU->tripletModuleIndices_buf, rangesBuffers->tripletModuleIndices_buf); alpaka::wait(queue); } return rangesInCPU; @@ -1610,7 +1592,7 @@ SDL::miniDoubletsBuffer* SDL::Event::getMiniDoublets() if (mdsInCPU == nullptr) { // Get nMemoryLocations parameter to initialize host based mdsInCPU auto nMemHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nMemHost_buf, miniDoubletsBuffers->nMemoryLocations_buf, 1); + alpaka::memcpy(queue, nMemHost_buf, miniDoubletsBuffers->nMemoryLocations_buf); alpaka::wait(queue); unsigned int nMemHost = *alpaka::getPtrNative(nMemHost_buf); @@ -1621,11 +1603,8 @@ SDL::miniDoubletsBuffer* SDL::Event::getMiniDoublets() alpaka::memcpy(queue, mdsInCPU->anchorHitIndices_buf, miniDoubletsBuffers->anchorHitIndices_buf, nMemHost); alpaka::memcpy(queue, mdsInCPU->outerHitIndices_buf, miniDoubletsBuffers->outerHitIndices_buf, nMemHost); alpaka::memcpy(queue, mdsInCPU->dphichanges_buf, miniDoubletsBuffers->dphichanges_buf, nMemHost); - alpaka::memcpy(queue, mdsInCPU->nMDs_buf, miniDoubletsBuffers->nMDs_buf, (Globals::nLowerModules + 1)); - alpaka::memcpy(queue, - mdsInCPU->totOccupancyMDs_buf, - miniDoubletsBuffers->totOccupancyMDs_buf, - (Globals::nLowerModules + 1)); + alpaka::memcpy(queue, mdsInCPU->nMDs_buf, miniDoubletsBuffers->nMDs_buf); + alpaka::memcpy(queue, mdsInCPU->totOccupancyMDs_buf, miniDoubletsBuffers->totOccupancyMDs_buf); alpaka::wait(queue); } return mdsInCPU; @@ -1635,7 +1614,7 @@ SDL::segmentsBuffer* SDL::Event::getSegments() { if (segmentsInCPU == nullptr) { // Get nMemoryLocations parameter to initialize host based segmentsInCPU auto nMemHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nMemHost_buf, segmentsBuffers->nMemoryLocations_buf, 1); + alpaka::memcpy(queue, nMemHost_buf, segmentsBuffers->nMemoryLocations_buf); alpaka::wait(queue); unsigned int nMemHost = *alpaka::getPtrNative(nMemHost_buf); @@ -1644,9 +1623,8 @@ SDL::segmentsBuffer* SDL::Event::getSegments() { segmentsInCPU->setData(*segmentsInCPU); *alpaka::getPtrNative(segmentsInCPU->nMemoryLocations_buf) = nMemHost; - alpaka::memcpy( - queue, segmentsInCPU->nSegments_buf, segmentsBuffers->nSegments_buf, (Globals::nLowerModules + 1)); - alpaka::memcpy(queue, segmentsInCPU->mdIndices_buf, segmentsBuffers->mdIndices_buf, 2 * nMemHost); + alpaka::memcpy(queue, segmentsInCPU->nSegments_buf, segmentsBuffers->nSegments_buf); + alpaka::memcpy(queue, segmentsInCPU->mdIndices_buf, segmentsBuffers->mdIndices_buf, 2u * nMemHost); alpaka::memcpy(queue, segmentsInCPU->innerMiniDoubletAnchorHitIndices_buf, segmentsBuffers->innerMiniDoubletAnchorHitIndices_buf, @@ -1655,17 +1633,14 @@ SDL::segmentsBuffer* SDL::Event::getSegments() { segmentsInCPU->outerMiniDoubletAnchorHitIndices_buf, segmentsBuffers->outerMiniDoubletAnchorHitIndices_buf, nMemHost); - alpaka::memcpy(queue, - segmentsInCPU->totOccupancySegments_buf, - segmentsBuffers->totOccupancySegments_buf, - (Globals::nLowerModules + 1)); - alpaka::memcpy(queue, segmentsInCPU->ptIn_buf, segmentsBuffers->ptIn_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->eta_buf, segmentsBuffers->eta_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->phi_buf, segmentsBuffers->phi_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->seedIdx_buf, segmentsBuffers->seedIdx_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->isDup_buf, segmentsBuffers->isDup_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->isQuad_buf, segmentsBuffers->isQuad_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); - alpaka::memcpy(queue, segmentsInCPU->score_buf, segmentsBuffers->score_buf, N_MAX_PIXEL_SEGMENTS_PER_MODULE); + alpaka::memcpy(queue, segmentsInCPU->totOccupancySegments_buf, segmentsBuffers->totOccupancySegments_buf); + alpaka::memcpy(queue, segmentsInCPU->ptIn_buf, segmentsBuffers->ptIn_buf); + alpaka::memcpy(queue, segmentsInCPU->eta_buf, segmentsBuffers->eta_buf); + alpaka::memcpy(queue, segmentsInCPU->phi_buf, segmentsBuffers->phi_buf); + alpaka::memcpy(queue, segmentsInCPU->seedIdx_buf, segmentsBuffers->seedIdx_buf); + alpaka::memcpy(queue, segmentsInCPU->isDup_buf, segmentsBuffers->isDup_buf); + alpaka::memcpy(queue, segmentsInCPU->isQuad_buf, segmentsBuffers->isQuad_buf); + alpaka::memcpy(queue, segmentsInCPU->score_buf, segmentsBuffers->score_buf); alpaka::wait(queue); } return segmentsInCPU; @@ -1675,7 +1650,7 @@ SDL::tripletsBuffer* SDL::Event::getTriplets() { if (tripletsInCPU == nullptr) { // Get nMemoryLocations parameter to initialize host based tripletsInCPU auto nMemHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nMemHost_buf, tripletsBuffers->nMemoryLocations_buf, 1); + alpaka::memcpy(queue, nMemHost_buf, tripletsBuffers->nMemoryLocations_buf); alpaka::wait(queue); unsigned int nMemHost = *alpaka::getPtrNative(nMemHost_buf); @@ -1703,12 +1678,8 @@ SDL::tripletsBuffer* SDL::Event::getTriplets() { alpaka::memcpy(queue, tripletsInCPU->betaIn_buf, tripletsBuffers->betaIn_buf, nMemHost); alpaka::memcpy(queue, tripletsInCPU->betaOut_buf, tripletsBuffers->betaOut_buf, nMemHost); alpaka::memcpy(queue, tripletsInCPU->pt_beta_buf, tripletsBuffers->pt_beta_buf, nMemHost); - alpaka::memcpy( - queue, tripletsInCPU->nTriplets_buf, tripletsBuffers->nTriplets_buf, Globals::nLowerModules); - alpaka::memcpy(queue, - tripletsInCPU->totOccupancyTriplets_buf, - tripletsBuffers->totOccupancyTriplets_buf, - Globals::nLowerModules); + alpaka::memcpy(queue, tripletsInCPU->nTriplets_buf, tripletsBuffers->nTriplets_buf); + alpaka::memcpy(queue, tripletsInCPU->totOccupancyTriplets_buf, tripletsBuffers->totOccupancyTriplets_buf); alpaka::wait(queue); } return tripletsInCPU; @@ -1718,7 +1689,7 @@ SDL::quintupletsBuffer* SDL::Event::getQuintuplets() { if (quintupletsInCPU == nullptr) { // Get nMemoryLocations parameter to initialize host based quintupletsInCPU auto nMemHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nMemHost_buf, quintupletsBuffers->nMemoryLocations_buf, 1); + alpaka::memcpy(queue, nMemHost_buf, quintupletsBuffers->nMemoryLocations_buf); alpaka::wait(queue); unsigned int nMemHost = *alpaka::getPtrNative(nMemHost_buf); @@ -1727,14 +1698,9 @@ SDL::quintupletsBuffer* SDL::Event::getQuintuplets() { quintupletsInCPU->setData(*quintupletsInCPU); *alpaka::getPtrNative(quintupletsInCPU->nMemoryLocations_buf) = nMemHost; - alpaka::memcpy(queue, - quintupletsInCPU->nQuintuplets_buf, - quintupletsBuffers->nQuintuplets_buf, - Globals::nLowerModules); - alpaka::memcpy(queue, - quintupletsInCPU->totOccupancyQuintuplets_buf, - quintupletsBuffers->totOccupancyQuintuplets_buf, - Globals::nLowerModules); + alpaka::memcpy(queue, quintupletsInCPU->nQuintuplets_buf, quintupletsBuffers->nQuintuplets_buf); + alpaka::memcpy( + queue, quintupletsInCPU->totOccupancyQuintuplets_buf, quintupletsBuffers->totOccupancyQuintuplets_buf); alpaka::memcpy(queue, quintupletsInCPU->tripletIndices_buf, quintupletsBuffers->tripletIndices_buf, 2 * nMemHost); alpaka::memcpy( queue, quintupletsInCPU->lowerModuleIndices_buf, quintupletsBuffers->lowerModuleIndices_buf, 5 * nMemHost); @@ -1758,18 +1724,16 @@ SDL::pixelTripletsBuffer* SDL::Event::getPixelTriplets if (pixelTripletsInCPU == nullptr) { // Get nPixelTriplets parameter to initialize host based quintupletsInCPU auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1); + alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf); alpaka::wait(queue); - int nPixelTriplets = *alpaka::getPtrNative(nPixelTriplets_buf); + unsigned int nPixelTriplets = *alpaka::getPtrNative(nPixelTriplets_buf); pixelTripletsInCPU = new SDL::pixelTripletsBuffer(nPixelTriplets, devHost, queue); pixelTripletsInCPU->setData(*pixelTripletsInCPU); *alpaka::getPtrNative(pixelTripletsInCPU->nPixelTriplets_buf) = nPixelTriplets; - alpaka::memcpy(queue, - pixelTripletsInCPU->totOccupancyPixelTriplets_buf, - pixelTripletsBuffers->totOccupancyPixelTriplets_buf, - 1); + alpaka::memcpy( + queue, pixelTripletsInCPU->totOccupancyPixelTriplets_buf, pixelTripletsBuffers->totOccupancyPixelTriplets_buf); alpaka::memcpy(queue, pixelTripletsInCPU->rzChiSquared_buf, pixelTripletsBuffers->rzChiSquared_buf, nPixelTriplets); alpaka::memcpy( queue, pixelTripletsInCPU->rPhiChiSquared_buf, pixelTripletsBuffers->rPhiChiSquared_buf, nPixelTriplets); @@ -1799,18 +1763,17 @@ SDL::pixelQuintupletsBuffer* SDL::Event::getPixelQuint if (pixelQuintupletsInCPU == nullptr) { // Get nPixelQuintuplets parameter to initialize host based quintupletsInCPU auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1); + alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf); alpaka::wait(queue); - int nPixelQuintuplets = *alpaka::getPtrNative(nPixelQuintuplets_buf); + unsigned int nPixelQuintuplets = *alpaka::getPtrNative(nPixelQuintuplets_buf); pixelQuintupletsInCPU = new SDL::pixelQuintupletsBuffer(nPixelQuintuplets, devHost, queue); pixelQuintupletsInCPU->setData(*pixelQuintupletsInCPU); *alpaka::getPtrNative(pixelQuintupletsInCPU->nPixelQuintuplets_buf) = nPixelQuintuplets; alpaka::memcpy(queue, pixelQuintupletsInCPU->totOccupancyPixelQuintuplets_buf, - pixelQuintupletsBuffers->totOccupancyPixelQuintuplets_buf, - 1); + pixelQuintupletsBuffers->totOccupancyPixelQuintuplets_buf); alpaka::memcpy( queue, pixelQuintupletsInCPU->rzChiSquared_buf, pixelQuintupletsBuffers->rzChiSquared_buf, nPixelQuintuplets); alpaka::memcpy(queue, @@ -1836,10 +1799,10 @@ SDL::trackCandidatesBuffer* SDL::Event::getTrackCandid if (trackCandidatesInCPU == nullptr) { // Get nTrackCanHost parameter to initialize host based trackCandidatesInCPU auto nTrackCanHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCanHost_buf, trackCandidatesBuffers->nTrackCandidates_buf, 1); + alpaka::memcpy(queue, nTrackCanHost_buf, trackCandidatesBuffers->nTrackCandidates_buf); alpaka::wait(queue); - int nTrackCanHost = *alpaka::getPtrNative(nTrackCanHost_buf); + unsigned int nTrackCanHost = *alpaka::getPtrNative(nTrackCanHost_buf); trackCandidatesInCPU = new SDL::trackCandidatesBuffer( N_MAX_NONPIXEL_TRACK_CANDIDATES + N_MAX_PIXEL_TRACK_CANDIDATES, devHost, queue); trackCandidatesInCPU->setData(*trackCandidatesInCPU); @@ -1870,10 +1833,10 @@ SDL::trackCandidatesBuffer* SDL::Event::getTrackCandid if (trackCandidatesInCPU == nullptr) { // Get nTrackCanHost parameter to initialize host based trackCandidatesInCPU auto nTrackCanHost_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nTrackCanHost_buf, trackCandidatesBuffers->nTrackCandidates_buf, 1); + alpaka::memcpy(queue, nTrackCanHost_buf, trackCandidatesBuffers->nTrackCandidates_buf); alpaka::wait(queue); - int nTrackCanHost = *alpaka::getPtrNative(nTrackCanHost_buf); + unsigned int nTrackCanHost = *alpaka::getPtrNative(nTrackCanHost_buf); trackCandidatesInCPU = new SDL::trackCandidatesBuffer( N_MAX_NONPIXEL_TRACK_CANDIDATES + N_MAX_PIXEL_TRACK_CANDIDATES, devHost, queue); trackCandidatesInCPU->setData(*trackCandidatesInCPU); diff --git a/SDL/Event.h b/SDL/Event.h index c5f5b66c..fd812b8a 100644 --- a/SDL/Event.h +++ b/SDL/Event.h @@ -13,6 +13,8 @@ #include "TrackCandidate.h" #include "Constants.h" +#include "HeterogeneousCore/AlpakaInterface/interface/host.h" + namespace SDL { template class Event {}; @@ -21,6 +23,8 @@ namespace SDL { class Event { private: QueueAcc queue; + Dev devAcc; + DevHost devHost; bool addObjects; std::array n_hits_by_layer_barrel_; @@ -75,11 +79,9 @@ namespace SDL { int8_t* pixelTypeCPU; public: - // Standalone constructor that has each event object create its own queue. - Event(bool verbose); // Constructor used for CMSSW integration. Uses an external queue. template - Event(bool verbose, const TQueue& q) : queue(q) { + Event(bool verbose, TQueue const& q) : queue(q), devAcc(alpaka::getDev(q)), devHost(cms::alpakatools::host()) { init(verbose); } void resetEvent(); @@ -177,7 +179,8 @@ namespace SDL { modulesBuffer* getModules(bool isFull = false); //read from file and init - static void initModules(const MapPLStoLayer& pLStoLayer, + static void initModules(QueueAcc& queue, + const MapPLStoLayer& pLStoLayer, const char* moduleMetaDataFilePath = "data/OT800_IT615_pt0.8/sensor_centroids.txt"); }; diff --git a/SDL/Globals.cc b/SDL/Globals.cc index a3431175..ab015d5b 100644 --- a/SDL/Globals.cc +++ b/SDL/Globals.cc @@ -6,7 +6,7 @@ std::shared_ptr SDL::Globals::pixelMapping = nullptr; uint16_t SDL::Globals::nModules; uint16_t SDL::Globals::nLowerModules; -SDL::EndcapGeometry* SDL::Globals::endcapGeometry = new SDL::EndcapGeometry(); +SDL::EndcapGeometry* SDL::Globals::endcapGeometry = nullptr; SDL::TiltedGeometry SDL::Globals::tiltedGeometry; SDL::ModuleConnectionMap SDL::Globals::moduleConnectionMap; diff --git a/SDL/LST.cc b/SDL/LST.cc index 73a754f2..6ed062aa 100644 --- a/SDL/LST.cc +++ b/SDL/LST.cc @@ -17,13 +17,16 @@ namespace { return fullpath.string(); } - void loadMaps(SDL::MapPLStoLayer& pLStoLayer) { + void loadMaps(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, SDL::MapPLStoLayer& pLStoLayer) { // Module orientation information (DrDz or phi angles) auto endcap_geom = get_absolute_path_after_check_file_exists(trackLooperDir() + "/data/OT800_IT615_pt0.8/endcap_orientation.txt"); auto tilted_geom = get_absolute_path_after_check_file_exists( trackLooperDir() + "/data/OT800_IT615_pt0.8/tilted_barrel_orientation.txt"); - SDL::Globals::endcapGeometry->load(endcap_geom); // centroid values added to the map + if (SDL::Globals::endcapGeometry == nullptr) { + SDL::Globals::endcapGeometry = + new SDL::EndcapGeometry(devAccIn, queue, endcap_geom); // centroid values added to the map + } SDL::Globals::tiltedGeometry.load(tilted_geom); // Module connection map (for line segment building) @@ -53,14 +56,15 @@ namespace { } // namespace -void SDL::LST::loadAndFillES(alpaka::QueueCpuBlocking& queue, struct modulesBuffer* modules) { +void SDL::LST::loadAndFillES(SDL::QueueAcc& queue, struct modulesBuffer* modules) { SDL::MapPLStoLayer pLStoLayer; - ::loadMaps(pLStoLayer); + SDL::Dev const& devAccIn = alpaka::getDev(queue); + ::loadMaps(devAccIn, queue, pLStoLayer); auto path = get_absolute_path_after_check_file_exists(trackLooperDir() + "/data/OT800_IT615_pt0.8/sensor_centroids.txt"); if (SDL::Globals::modulesBuffers == nullptr) { - SDL::Globals::modulesBuffers = new SDL::modulesBuffer(SDL::devAcc); + SDL::Globals::modulesBuffers = new SDL::modulesBuffer(devAccIn); } if (SDL::Globals::pixelMapping == nullptr) { SDL::Globals::pixelMapping = std::make_shared(); diff --git a/SDL/LST.h b/SDL/LST.h index b8a2da41..4e467ebc 100644 --- a/SDL/LST.h +++ b/SDL/LST.h @@ -24,7 +24,7 @@ namespace SDL { public: LST() = default; - static void loadAndFillES(alpaka::QueueCpuBlocking& queue, struct modulesBuffer* modules); + static void loadAndFillES(SDL::QueueAcc& queue, struct modulesBuffer* modules); void run(SDL::QueueAcc& queue, const SDL::modulesBuffer* modules, diff --git a/SDL/ModuleMethods.h b/SDL/ModuleMethods.h index e9cdd81d..2a777ba5 100644 --- a/SDL/ModuleMethods.h +++ b/SDL/ModuleMethods.h @@ -12,6 +12,8 @@ #include "PixelMap.h" #include "Globals.h" +#include "HeterogeneousCore/AlpakaInterface/interface/host.h" + namespace SDL { struct ModuleMetaData { std::map detIdToIndex; @@ -34,9 +36,9 @@ namespace SDL { std::vector connectedModuleDetIds_pos; std::vector connectedModuleDetIds_neg; - int totalSizes = 0; - int totalSizes_pos = 0; - int totalSizes_neg = 0; + unsigned int totalSizes = 0; + unsigned int totalSizes_pos = 0; + unsigned int totalSizes_neg = 0; for (unsigned int isuperbin = 0; isuperbin < size_superbins; isuperbin++) { int sizes = 0; for (auto const& mCM_pLS : pLStoLayer[0]) { @@ -75,7 +77,7 @@ namespace SDL { totalSizes_neg += sizes_neg; } - int connectedPix_size = totalSizes + totalSizes_pos + totalSizes_neg; + unsigned int connectedPix_size = totalSizes + totalSizes_pos + totalSizes_neg; // Temporary check for module initialization. if (pix_tot != connectedPix_size) { @@ -85,20 +87,21 @@ namespace SDL { throw std::runtime_error("Mismatched sizes"); } + DevHost const& devHost = cms::alpakatools::host(); auto connectedPixels_buf = allocBufWrapper(devHost, connectedPix_size); unsigned int* connectedPixels = alpaka::getPtrNative(connectedPixels_buf); - for (int icondet = 0; icondet < totalSizes; icondet++) { + for (unsigned int icondet = 0; icondet < totalSizes; icondet++) { connectedPixels[icondet] = mmd.detIdToIndex[connectedModuleDetIds[icondet]]; } - for (int icondet = 0; icondet < totalSizes_pos; icondet++) { + for (unsigned int icondet = 0; icondet < totalSizes_pos; icondet++) { connectedPixels[icondet + totalSizes] = mmd.detIdToIndex[connectedModuleDetIds_pos[icondet]]; } - for (int icondet = 0; icondet < totalSizes_neg; icondet++) { + for (unsigned int icondet = 0; icondet < totalSizes_neg; icondet++) { connectedPixels[icondet + totalSizes + totalSizes_pos] = mmd.detIdToIndex[connectedModuleDetIds_neg[icondet]]; } - alpaka::memcpy(queue, modulesBuf->connectedPixels_buf, connectedPixels_buf, connectedPix_size); + alpaka::memcpy(queue, modulesBuf->connectedPixels_buf, connectedPixels_buf); alpaka::wait(queue); }; @@ -107,6 +110,7 @@ namespace SDL { unsigned int nMod, TQueue queue, struct ModuleMetaData& mmd) { + DevHost const& devHost = cms::alpakatools::host(); auto moduleMap_buf = allocBufWrapper(devHost, nMod * MAX_CONNECTED_MODULES); uint16_t* moduleMap = alpaka::getPtrNative(moduleMap_buf); @@ -123,8 +127,8 @@ namespace SDL { } } - alpaka::memcpy(queue, modulesBuf->moduleMap_buf, moduleMap_buf, nMod * MAX_CONNECTED_MODULES); - alpaka::memcpy(queue, modulesBuf->nConnectedModules_buf, nConnectedModules_buf, nMod); + alpaka::memcpy(queue, modulesBuf->moduleMap_buf, moduleMap_buf); + alpaka::memcpy(queue, modulesBuf->nConnectedModules_buf, nConnectedModules_buf); alpaka::wait(queue); }; @@ -133,6 +137,7 @@ namespace SDL { unsigned int nMod, TQueue queue, struct ModuleMetaData& mmd) { + DevHost const& devHost = cms::alpakatools::host(); auto mapIdx_buf = allocBufWrapper(devHost, nMod); uint16_t* mapIdx = alpaka::getPtrNative(mapIdx_buf); @@ -148,8 +153,8 @@ namespace SDL { counter++; } - alpaka::memcpy(queue, modulesBuf->mapIdx_buf, mapIdx_buf, nMod); - alpaka::memcpy(queue, modulesBuf->mapdetId_buf, mapdetId_buf, nMod); + alpaka::memcpy(queue, modulesBuf->mapIdx_buf, mapIdx_buf); + alpaka::memcpy(queue, modulesBuf->mapdetId_buf, mapdetId_buf); alpaka::wait(queue); }; @@ -235,6 +240,7 @@ namespace SDL { throw std::runtime_error("Mismatched sizes"); } + DevHost const& devHost = cms::alpakatools::host(); auto detIds_buf = allocBufWrapper(devHost, nModules); auto layers_buf = allocBufWrapper(devHost, nModules); auto rings_buf = allocBufWrapper(devHost, nModules); diff --git a/bin/sdl.cc b/bin/sdl.cc index fcd516fe..3adb17a2 100644 --- a/bin/sdl.cc +++ b/bin/sdl.cc @@ -297,11 +297,13 @@ int main(int argc, char** argv) //___________________________________________________________________________________________________________________________________________________________________________________________ void run_sdl() { + SDL::Dev devAcc = alpaka::getDevByIdx(ALPAKA_ACCELERATOR_NAMESPACE::Platform{}, 0u); + SDL::QueueAcc queue(devAcc); // Load various maps used in the SDL reconstruction TStopwatch full_timer; full_timer.Start(); - loadMaps(); + loadMaps(devAcc, queue); float timeForMapLoading = full_timer.RealTime()*1000; if (ana.do_write_ntuple) @@ -383,7 +385,7 @@ void run_sdl() std::vector*> events; for (int s = 0; s < ana.streams; s++) { - SDL::Event *event = new SDL::Event(ana.verbose>=2); + SDL::Event *event = new SDL::Event(ana.verbose>=2, queue); events.push_back(event); } float timeForEventCreation = full_timer.RealTime()*1000; diff --git a/code/core/trkCore.cc b/code/core/trkCore.cc index fd64dfc6..50ed2193 100644 --- a/code/core/trkCore.cc +++ b/code/core/trkCore.cc @@ -2,7 +2,7 @@ #include "Globals.h" //___________________________________________________________________________________________________________________________________________________________________________________________ -void loadMaps() +void loadMaps(SDL::Dev& devAccIn, SDL::QueueAcc& queue) { // From the environment variable figure out the main tracklooper absolute path TString TrackLooperDir = gSystem->Getenv("TRACKLOOPERDIR"); @@ -21,7 +21,9 @@ void loadMaps() std::cout << "pLS map: " << pLSMapDir << std::endl; std::cout << "centroid: " << centroid << std::endl; - SDL::Globals::endcapGeometry->load(endcap_geom.Data()); // centroid values added to the map + if (SDL::Globals::endcapGeometry == nullptr) { + SDL::Globals::endcapGeometry = new SDL::EndcapGeometry(devAccIn, queue, endcap_geom.Data()); // centroid values added to the map + } SDL::Globals::tiltedGeometry.load(tilted_geom.Data()); SDL::Globals::moduleConnectionMap.load(mappath.Data()); @@ -41,12 +43,12 @@ void loadMaps() // WARNING: initModules must come after above load commands!! keep it at the last line here! if (SDL::Globals::modulesBuffers == nullptr) { - SDL::Globals::modulesBuffers = new SDL::modulesBuffer(SDL::devAcc); + SDL::Globals::modulesBuffers = new SDL::modulesBuffer(devAccIn); } if (SDL::Globals::pixelMapping == nullptr) { SDL::Globals::pixelMapping = std::make_shared(); } - SDL::Event::initModules(pLStoLayer, centroid.Data()); + SDL::Event::initModules(queue, pLStoLayer, centroid.Data()); } //___________________________________________________________________________________________________________________________________________________________________________________________ diff --git a/code/core/trkCore.h b/code/core/trkCore.h index f39083fc..d236f22a 100644 --- a/code/core/trkCore.h +++ b/code/core/trkCore.h @@ -13,7 +13,7 @@ // --------------------- ======================== --------------------- -void loadMaps(); +void loadMaps(SDL::Dev& devAccIn, SDL::QueueAcc& queue); bool goodEvent(); float runMiniDoublet(SDL::Event* event, int evt); float runSegment(SDL::Event* event);