Remove device initialization from library

SDL/Constants.h

@@ -67,7 +67,7 @@ namespace SDL {
   auto const devHost = alpaka::getDevByIdx(platformHost, 0u);
 #if defined ALPAKA_ACC_GPU_CUDA_ENABLED || defined ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED || \
     defined ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED
-  auto const devAcc = alpaka::getDevByIdx(platformAcc, 0u);
+  // auto const devAcc = alpaka::getDevByIdx(platformAcc, 0u);
   using QueueAcc = alpaka::Queue<Acc, QueueProperty>;
 #endif
 

SDL/EndcapGeometry.cc

@@ -1,16 +1,17 @@
 #include "EndcapGeometry.h"
 
-SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(unsigned int sizef)
-    : geoMapDetId_buf(allocBufWrapper<unsigned int>(devAcc, sizef)),
-      geoMapPhi_buf(allocBufWrapper<float>(devAcc, sizef)) {}
-
-SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(std::string filename, unsigned int sizef)
-    : geoMapDetId_buf(allocBufWrapper<unsigned int>(devAcc, sizef)),
-      geoMapPhi_buf(allocBufWrapper<float>(devAcc, sizef)) {
-  load(filename);
+SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(SDL::Dev const& devAccIn, unsigned int sizef)
+    : geoMapDetId_buf(allocBufWrapper<unsigned int>(devAccIn, sizef)),
+      geoMapPhi_buf(allocBufWrapper<float>(devAccIn, sizef)) {}
+
+SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, std::string filename, unsigned int sizef)
+    : geoMapDetId_buf(allocBufWrapper<unsigned int>(devAccIn, sizef)),
+      geoMapPhi_buf(allocBufWrapper<float>(devAccIn, sizef)) {
+  load(queue, filename);
 }
 
-void SDL::EndcapGeometry<SDL::Dev>::load(std::string filename) {
+template <typename TQueue>
+void SDL::EndcapGeometry<SDL::Dev>::load(TQueue& queue, std::string filename) {
   dxdy_slope_.clear();
   centroid_phis_.clear();
 
@@ -30,11 +31,11 @@
     }
   }
 
-  fillGeoMapArraysExplicit();
+  fillGeoMapArraysExplicit(queue);
 }
 
-void SDL::EndcapGeometry<SDL::Dev>::fillGeoMapArraysExplicit() {
-  QueueAcc queue(devAcc);
+template <typename TQueue>
+void SDL::EndcapGeometry<SDL::Dev>::fillGeoMapArraysExplicit(TQueue& queue) {
 
   int phi_size = centroid_phis_.size();
 

SDL/EndcapGeometry.h

@@ -1,4 +1,4 @@
 #ifndef EndcapGeometry_h
 #define EndcapGeometry_h
 
 #include <map>
@@ -26,14 +26,15 @@
 
     unsigned int nEndCapMap;
 
-    EndcapGeometry(unsigned int sizef = endcap_size);
-    EndcapGeometry(std::string filename, unsigned int sizef = endcap_size);
+    EndcapGeometry(SDL::Dev const& devAccIn, unsigned int sizef = endcap_size);
+    EndcapGeometry(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, std::string filename, unsigned int sizef = endcap_size);
     ~EndcapGeometry() = default;
 
-    void load(std::string);
+    template <typename TQueue>
+    void load(TQueue& queue, std::string);
 
-    void fillGeoMapArraysExplicit();
-    void CreateGeoMapArraysExplicit();
+    template <typename TQueue>
+    void fillGeoMapArraysExplicit(TQueue& queue);
     float getdxdy_slope(unsigned int detid);
   };
 }  // namespace SDL

SDL/Event.cc

@@ -1,4 +1,4 @@
 #include "Event.h"
 #include "Globals.h"
 
 void SDL::Event<SDL::Acc>::init(bool verbose) {
@@ -44,7 +44,7 @@
 }
 
 // Standalone constructor that has each event object create its own queue.
-SDL::Event<SDL::Acc>::Event(bool verbose) : queue(alpaka::getDevByIdx(platformAcc, 0u)) { init(verbose); }
+SDL::Event<SDL::Acc>::Event(bool verbose) : devAcc(alpaka::getDevByIdx(platformAcc, 0u)), queue(alpaka::getDevByIdx(platformAcc, 0u)) { init(verbose); }
 
 void SDL::Event<SDL::Acc>::resetEvent() {
   //reset the arrays
@@ -152,8 +152,7 @@
   }
 }
 
-void SDL::Event<SDL::Acc>::initModules(const MapPLStoLayer& pLStoLayer, const char* moduleMetaDataFilePath) {
-  QueueAcc queue(devAcc);
+void SDL::Event<SDL::Acc>::initModules(QueueAcc& queue, const MapPLStoLayer& pLStoLayer, const char* moduleMetaDataFilePath) {
 
   // nModules gets filled here
   loadModulesFromFile(Globals<SDL::Dev>::modulesBuffers,

SDL/Event.h

@@ -20,6 +20,7 @@ namespace SDL {
   template <>
   class Event<SDL::Acc> {
   private:
+    Dev devAcc;
     QueueAcc queue;
     bool addObjects;
 
@@ -75,11 +76,11 @@ namespace SDL {
     int8_t* pixelTypeCPU;
 
   public:
-    // Standalone constructor that has each event object create its own queue.
+    // Standalone constructor that has each event object create its own device and queue.
     Event(bool verbose);
-    // Constructor used for CMSSW integration. Uses an external queue.
-    template <typename TQueue>
-    Event(bool verbose, const TQueue& q) : queue(q) {
+    // Constructor used for CMSSW integration. Uses an external device and queue.
+    template <typename TDevAcc, typename TQueue>
+    Event(bool verbose, TDevAcc const& devAccIn, TQueue const& q) : devAcc(devAccIn), queue(q) {
       init(verbose);
     }
     void resetEvent();
@@ -177,7 +178,8 @@ namespace SDL {
     modulesBuffer<alpaka::DevCpu>* getModules(bool isFull = false);
 
     //read from file and init
-    static void initModules(const MapPLStoLayer& pLStoLayer,
+    static void initModules(QueueAcc& queue,
+                            const MapPLStoLayer& pLStoLayer,
                             const char* moduleMetaDataFilePath = "data/OT800_IT615_pt0.8/sensor_centroids.txt");
   };
 

SDL/Globals.cc

@@ -6,7 +6,7 @@ std::shared_ptr<SDL::pixelMap> SDL::Globals<SDL::Dev>::pixelMapping = nullptr;
 uint16_t SDL::Globals<SDL::Dev>::nModules;
 uint16_t SDL::Globals<SDL::Dev>::nLowerModules;
 
-SDL::EndcapGeometry<SDL::Dev>* SDL::Globals<SDL::Dev>::endcapGeometry = new SDL::EndcapGeometry<SDL::Dev>();
+SDL::EndcapGeometry<SDL::Dev>* SDL::Globals<SDL::Dev>::endcapGeometry = nullptr;
 
 SDL::TiltedGeometry<SDL::Dev> SDL::Globals<SDL::Dev>::tiltedGeometry;
 SDL::ModuleConnectionMap<SDL::Dev> SDL::Globals<SDL::Dev>::moduleConnectionMap;

SDL/LST.cc

@@ -1,4 +1,4 @@
 #include "LST.h"
 #include "Event.h"
 #include "Globals.h"
 
@@ -17,13 +17,15 @@
     return fullpath.string();
   }
 
-  void loadMaps(SDL::MapPLStoLayer& pLStoLayer) {
+  void loadMaps(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, SDL::MapPLStoLayer& pLStoLayer) {
     // Module orientation information (DrDz or phi angles)
     auto endcap_geom =
         get_absolute_path_after_check_file_exists(trackLooperDir() + "/data/OT800_IT615_pt0.8/endcap_orientation.txt");
     auto tilted_geom = get_absolute_path_after_check_file_exists(
         trackLooperDir() + "/data/OT800_IT615_pt0.8/tilted_barrel_orientation.txt");
-    SDL::Globals<SDL::Dev>::endcapGeometry->load(endcap_geom);  // centroid values added to the map
+    if (SDL::Globals<SDL::Dev>::endcapGeometry == nullptr) {
+      SDL::Globals<SDL::Dev>::endcapGeometry = new SDL::EndcapGeometry<SDL::Dev>(devAccIn, queue, endcap_geom); // centroid values added to the map
+    }
     SDL::Globals<SDL::Dev>::tiltedGeometry.load(tilted_geom);
 
     // Module connection map (for line segment building)
@@ -53,14 +55,14 @@
 
 }  // namespace
 
-void SDL::LST<SDL::Acc>::loadAndFillES(alpaka::QueueCpuBlocking& queue, struct modulesBuffer<alpaka::DevCpu>* modules) {
+void SDL::LST<SDL::Acc>::loadAndFillES(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, struct modulesBuffer<alpaka::DevCpu>* modules) {
   SDL::MapPLStoLayer pLStoLayer;
-  ::loadMaps(pLStoLayer);
+  ::loadMaps(devAccIn, queue, pLStoLayer);
 
   auto path =
       get_absolute_path_after_check_file_exists(trackLooperDir() + "/data/OT800_IT615_pt0.8/sensor_centroids.txt");
   if (SDL::Globals<SDL::Dev>::modulesBuffers == nullptr) {
-    SDL::Globals<SDL::Dev>::modulesBuffers = new SDL::modulesBuffer<SDL::Dev>(SDL::devAcc);
+    SDL::Globals<SDL::Dev>::modulesBuffers = new SDL::modulesBuffer<SDL::Dev>(devAccIn);
   }
   if (SDL::Globals<SDL::Dev>::pixelMapping == nullptr) {
     SDL::Globals<SDL::Dev>::pixelMapping = std::make_shared<SDL::pixelMap>();
@@ -74,7 +76,8 @@
                            pLStoLayer);
 }
 
-void SDL::LST<SDL::Acc>::run(SDL::QueueAcc& queue,
+void SDL::LST<SDL::Acc>::run(SDL::Dev& devAccIn,
+                             SDL::QueueAcc& queue,
                              const SDL::modulesBuffer<SDL::Dev>* modules,
                              bool verbose,
                              const std::vector<float> see_px,
@@ -97,7 +100,7 @@
                              const std::vector<float> ph2_y,
                              const std::vector<float> ph2_z) {
   SDL::Globals<SDL::Dev>::modulesBuffersES = modules;
-  auto event = SDL::Event<Acc>(verbose, queue);
+  auto event = SDL::Event<Acc>(verbose, devAccIn, queue);
   prepareInput(see_px,
                see_py,
                see_pz,

SDL/LST.h

@@ -1,4 +1,4 @@
 #ifndef LST_H
 #define LST_H
 
 #include "Constants.h"
@@ -24,9 +24,10 @@
   public:
     LST() = default;
 
-    static void loadAndFillES(alpaka::QueueCpuBlocking& queue, struct modulesBuffer<alpaka::DevCpu>* modules);
+    static void loadAndFillES(SDL::Dev const& devAccIn, SDL::QueueAcc& queue, struct modulesBuffer<alpaka::DevCpu>* modules);
 
-    void run(SDL::QueueAcc& queue,
+    void run(SDL::Dev& devAccIn,
+             SDL::QueueAcc& queue,
              const SDL::modulesBuffer<SDL::Dev>* modules,
              bool verbose,
              const std::vector<float> see_px,

bin/sdl.cc

@@ -297,11 +297,13 @@ int main(int argc, char** argv)
 //___________________________________________________________________________________________________________________________________________________________________________________________
 void run_sdl()
 {
+    SDL::Dev devAcc = alpaka::getDevByIdx(SDL::platformAcc, 0u);
+    SDL::QueueAcc queue(devAcc);
 
     // Load various maps used in the SDL reconstruction
     TStopwatch full_timer;
     full_timer.Start();
-    loadMaps();
+    loadMaps(devAcc, queue);
     float timeForMapLoading = full_timer.RealTime()*1000;
 
     if (ana.do_write_ntuple)
@@ -383,7 +385,7 @@ void run_sdl()
     std::vector<SDL::Event<SDL::Acc>*> events;
     for (int s = 0; s < ana.streams; s++)
     {
-        SDL::Event<SDL::Acc> *event = new SDL::Event<SDL::Acc>(ana.verbose>=2);
+        SDL::Event<SDL::Acc> *event = new SDL::Event<SDL::Acc>(ana.verbose>=2, devAcc, queue);
         events.push_back(event);
     }
     float timeForEventCreation = full_timer.RealTime()*1000;

code/core/trkCore.cc

@@ -2,7 +2,7 @@
 #include "Globals.h"
 
 //___________________________________________________________________________________________________________________________________________________________________________________________
-void loadMaps()
+void loadMaps(SDL::Dev& devAccIn, SDL::QueueAcc& queue)
 {
     // From the environment variable figure out the main tracklooper absolute path
     TString TrackLooperDir = gSystem->Getenv("TRACKLOOPERDIR");
@@ -21,7 +21,9 @@ void loadMaps()
     std::cout << "pLS map: " << pLSMapDir << std::endl;
     std::cout << "centroid: " << centroid << std::endl;
 
-    SDL::Globals<SDL::Dev>::endcapGeometry->load(endcap_geom.Data()); // centroid values added to the map
+    if (SDL::Globals<SDL::Dev>::endcapGeometry == nullptr) {
+      SDL::Globals<SDL::Dev>::endcapGeometry = new SDL::EndcapGeometry<SDL::Dev>(devAccIn, queue, endcap_geom.Data()); // centroid values added to the map
+    }
     SDL::Globals<SDL::Dev>::tiltedGeometry.load(tilted_geom.Data());
     SDL::Globals<SDL::Dev>::moduleConnectionMap.load(mappath.Data());
 
@@ -41,12 +43,12 @@ void loadMaps()
 
     // WARNING: initModules must come after above load commands!! keep it at the last line here!
     if (SDL::Globals<SDL::Dev>::modulesBuffers == nullptr) {
-      SDL::Globals<SDL::Dev>::modulesBuffers = new SDL::modulesBuffer<SDL::Dev>(SDL::devAcc);
+      SDL::Globals<SDL::Dev>::modulesBuffers = new SDL::modulesBuffer<SDL::Dev>(devAccIn);
     }
     if (SDL::Globals<SDL::Dev>::pixelMapping == nullptr) {
       SDL::Globals<SDL::Dev>::pixelMapping = std::make_shared<SDL::pixelMap>();
     }
-    SDL::Event<SDL::Acc>::initModules(pLStoLayer, centroid.Data());
+    SDL::Event<SDL::Acc>::initModules(queue, pLStoLayer, centroid.Data());
 }
 
 //___________________________________________________________________________________________________________________________________________________________________________________________

code/core/trkCore.h

@@ -13,7 +13,7 @@
 
 // --------------------- ======================== ---------------------
 
-void loadMaps();
+void loadMaps(SDL::Dev& devAccIn, SDL::QueueAcc& queue);
 bool goodEvent();
 float runMiniDoublet(SDL::Event<SDL::Acc>* event, int evt);
 float runSegment(SDL::Event<SDL::Acc>* event);