diff --git a/README.md b/README.md index 9d1e0f6c..95f7b741 100644 --- a/README.md +++ b/README.md @@ -129,14 +129,14 @@ cd .. ```bash mkdir workingFolder # Create the folder you will be working in cd workingFolder -cmsrel CMSSW_13_3_0_pre3 -cd CMSSW_13_3_0_pre3/src +cmsrel CMSSW_14_1_0_pre0 +cd CMSSW_14_1_0_pre0/src cmsenv git cms-init git remote add SegLink git@github.com:SegmentLinking/cmssw.git -git fetch SegLink CMSSW_13_3_0_pre3_LST_X +git fetch SegLink CMSSW_14_1_0_pre0_LST_X git cms-addpkg RecoTracker Configuration -git checkout CMSSW_13_3_0_pre3_LST_X +git checkout CMSSW_14_1_0_pre0_LST_X #To include both the CPU library and GPU library into CMSSW, create 3 xml files (headers file has no library). #Before writing the following xml file, check that libsdl_cpu.so and libsdl_gpu.so can be found under the ../../../TrackLooper/SDL/ folder. cat <lst_headers.xml @@ -247,7 +247,7 @@ The setup scripts included in this repository assume that the [CernVM File Syste The necessary dependencies are CUDA, ROOT, the Boost libraries, Alpaka, and some CMSSW headers. CUDA, ROOT, and Boost, are fairly standard libraries and are available from multiple package managers. For the remaining necessary headers you will need to clone the [Alpaka](https://github.com/alpaka-group/alpaka) and [CMSSW](https://github.com/cms-sw/cmssw) repositories. The Alpaka repository is reasonably sized, but the CMSSW one extremely large, especially considering that we only need a tiny fraction of its files to build LST. We can get only the Alpaka interface headers from CMSSW by running the following commands. ``` bash -git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch CMSSW_13_3_X https://github.com/cms-sw/cmssw.git +git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch CMSSW_14_1_X https://github.com/cms-sw/cmssw.git cd cmssw git sparse-checkout add HeterogeneousCore/AlpakaInterface git checkout @@ -257,16 +257,16 @@ Then all that is left to do is set some environment variables. We give an exampl ```bash # These two lines are only needed to set the right version of gcc and nvcc. They are not needed for standard installations. -export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/external/el8_amd64_gcc11/bin:$PATH -export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/biglib/el8_amd64_gcc11:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/lib/el8_amd64_gcc11:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/external/el8_amd64_gcc11/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/lib:$LD_LIBRARY_PATH +export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/external/el8_amd64_gcc12/bin:$PATH +export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/biglib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/lib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/external/el8_amd64_gcc12/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib:$LD_LIBRARY_PATH # These are the lines that you need to manually change for a CVMFS-less setup. # In this example we use cvmfs paths since that is where the dependencies are in lnx7188/cgpu1, but they can point to local directories. -export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/boost/1.80.0-536665e33076f709097addc8619ee0a0 -export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/alpaka/develop-20230621-5659bacc780954326761375d025a3208 -export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/cuda/11.8.0-9f0af0f4206be7b705fe550319c49a11 -export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/lcg/root/6.26.11-700833a012ccdeb623021803b605d653 -export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3 +export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/boost/1.80.0-9d4e0dc8cf1f8fc8a67cf53ae917955e +export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/alpaka/1.1.0-0e0b978d445f7af747cf00064c146356 +export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/cuda/12.2.1-bdf3fff69eaec65abe18a7569592cab6 +export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/lcg/root/6.30.05-d0417d1d3c55ded4cc9aa75628166a77 +export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0 # These lines are needed to account for some extra environment variables that are exported in the setup script. export LD_LIBRARY_PATH=$PWD/SDL/cuda:$PWD/SDL/cpu:$PWD:$LD_LIBRARY_PATH diff --git a/SDL/Constants.h b/SDL/Constants.h index 60829b1b..0a10ad2e 100644 --- a/SDL/Constants.h +++ b/SDL/Constants.h @@ -62,10 +62,12 @@ namespace SDL { }; #endif - auto const devHost = alpaka::getDevByIdx(0u); + auto const platformAcc = alpaka::Platform{}; + auto const platformHost = alpaka::Platform{}; + auto const devHost = alpaka::getDevByIdx(platformHost, 0u); #if defined ALPAKA_ACC_GPU_CUDA_ENABLED || defined ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED || \ defined ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED - auto const devAcc = alpaka::getDevByIdx(0u); + auto const devAcc = alpaka::getDevByIdx(platformAcc, 0u); using QueueAcc = alpaka::Queue; #endif diff --git a/SDL/Event.cc b/SDL/Event.cc index a43fc383..8731fe28 100644 --- a/SDL/Event.cc +++ b/SDL/Event.cc @@ -51,7 +51,7 @@ void SDL::Event::init(bool verbose) { } // Standalone constructor that has each event object create its own queue. -SDL::Event::Event(bool verbose) : queue(alpaka::getDevByIdx(0u)) { init(verbose); } +SDL::Event::Event(bool verbose) : queue(alpaka::getDevByIdx(platformAcc, 0u)) { init(verbose); } void SDL::Event::resetEvent() { //reset the arrays @@ -908,9 +908,9 @@ void SDL::Event::createPixelTriplets() { alpaka::wait(queue); #ifdef Warnings - auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); + auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1); + alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1u); alpaka::wait(queue); std::cout << "number of pixel triplets = " << *alpaka::getPtrNative(nPixelTriplets_buf) << std::endl; @@ -1168,9 +1168,9 @@ void SDL::Event::createPixelQuintuplets() { alpaka::wait(queue); #ifdef Warnings - auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); + auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1); + alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1u); alpaka::wait(queue); std::cout << "number of pixel quintuplets = " << *alpaka::getPtrNative(nPixelQuintuplets_buf) << std::endl; diff --git a/SDL/Makefile b/SDL/Makefile index 9df51a4d..369ef7af 100644 --- a/SDL/Makefile +++ b/SDL/Makefile @@ -27,8 +27,8 @@ LIBS=$(LIB_CUDA) $(LIB_CPU) # # Different architectures to optimize for -GENCODE_SM70 := -gencode arch=compute_70,code=sm_70 -GENCODE_SM89 := -gencode arch=compute_89,code=sm_89 +GENCODE_SM70 := -gencode arch=compute_70,code=[sm_70,compute_70] +GENCODE_SM89 := -gencode arch=compute_89,code=[sm_89,compute_89] CXX = g++ CXXFLAGS_CPU = -march=native -mtune=native -Ofast -fno-reciprocal-math -fopenmp-simd -g -Wall -Wshadow -Woverloaded-virtual -fPIC -fopenmp -I.. diff --git a/SDL/PixelTriplet.h b/SDL/PixelTriplet.h index 57c2799d..cba81807 100644 --- a/SDL/PixelTriplet.h +++ b/SDL/PixelTriplet.h @@ -1078,7 +1078,7 @@ namespace SDL { if (tripletLowerModuleIndex >= *modulesInGPU.nLowerModules) { printf("tripletLowerModuleIndex %d >= modulesInGPU.nLowerModules %d \n", tripletLowerModuleIndex, - modulesInGPU.nLowerModules); + *modulesInGPU.nLowerModules); continue; //sanity check } #endif diff --git a/setup.sh b/setup.sh index 77d0e1f3..6bc292ca 100644 --- a/setup.sh +++ b/setup.sh @@ -6,8 +6,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh -export SCRAM_ARCH=el8_amd64_gcc11 -export CMSSW_VERSION=CMSSW_13_3_0_pre3 +export SCRAM_ARCH=el8_amd64_gcc12 +export CMSSW_VERSION=CMSSW_14_1_0_pre0 source /cvmfs/cms.cern.ch/cmsset_default.sh cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src diff --git a/setup_hpg.sh b/setup_hpg.sh index d92b3802..2ac19737 100644 --- a/setup_hpg.sh +++ b/setup_hpg.sh @@ -11,8 +11,8 @@ module load cuda/11.4.3 git DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh -export SCRAM_ARCH=el8_amd64_gcc11 -export CMSSW_VERSION=CMSSW_13_3_0_pre3 +export SCRAM_ARCH=el8_amd64_gcc12 +export CMSSW_VERSION=CMSSW_14_1_0_pre0 export CUDA_HOME=${HPC_CUDA_DIR} source /cvmfs/cms.cern.ch/cmsset_default.sh