diff --git a/.github/actions/install-repo/action.yml b/.github/actions/install-repo/action.yml new file mode 100644 index 00000000..b7995864 --- /dev/null +++ b/.github/actions/install-repo/action.yml @@ -0,0 +1,47 @@ +name: Install-Repo + +inputs: + repo-name: + required: true + repo-path: + required: true + repo-ref: + required: true + options: + required: true + cache: + required: true + submodules: + default: '' + +runs: + using: "composite" + steps: + + - name: Check Cache + if: ${{ inputs.cache == 'true'}} + uses: actions/cache@v3 + id: check-cache + with: + key: build-${{ inputs.repo-name }} + path: ${{ runner.temp }}/build-${{ inputs.repo-name }} + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo-path }} + submodules: ${{ inputs.submodules }} + ref: ${{ inputs.repo-ref }} + path: ${{ inputs.repo-name }} + + - name: Configure CMake + if: ${{ !steps.check-cache.outputs.cache-hit }} + shell: bash + run: cmake -S $GITHUB_WORKSPACE/${{ inputs.repo-name }} -B ${{ runner.temp }}/build-${{ inputs.repo-name }} + -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-${{ inputs.repo-name }}/install + ${{ inputs.options }} + + - name: Build Cmake + if: ${{ !steps.check-cache.outputs.cache-hit }} + shell: bash + run: cmake --build ${{ runner.temp }}/build-${{ inputs.repo-name }} -j8 --target install \ No newline at end of file diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 6c9ef06e..a6a028a4 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -32,215 +32,105 @@ jobs: - name: Install NetCDF-Fortran run: sudo apt-get install libnetcdff-dev - # Build Kokkos + - uses: actions/checkout@v4 - - name: Cache Kokkos Build - uses: actions/cache@v3 - id: build-kokkos - with: - key: build-kokkos - path: ${{ runner.temp }}/build-kokkos - - - name: Kokkos Checkout repo - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 - with: - repository: kokkos/kokkos - ref: 4.1.00 - path: kokkos - - - name: Kokkos Create Directory - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-kokkos - - - name: Kokkos Configure CMake - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/kokkos -B ${{ runner.temp }}/build-kokkos - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-kokkos/install - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DKokkos_ENABLE_SERIAL=ON - -DKokkos_ENABLE_OPENMP=off - -DKokkos_ENABLE_CUDA=off - -DKokkos_ENABLE_CUDA_LAMBDA=off - -DKokkos_ENABLE_DEBUG=on - - - name: Kokkos Build - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-kokkos -j8 --target install - - # Build EnGPar - - - name: Cache Engpar Build - uses: actions/cache@v3 - id: build-engpar - with: - key: build-engpar - path: ${{ runner.temp }}/build-engpar - - - name: EnGPar Checkout repo - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 - with: - repository: SCOREC/EnGPar - path: engpar - - - name: EnGPar Create Directory - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-engpar - - - name: EnGPar Configure CMake - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/engpar -B ${{ runner.temp }}/build-engpar - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-engpar/install - -DCMAKE_C_COMPILER="mpicc" - -DCMAKE_CXX_COMPILER="mpicxx" - -DCMAKE_CXX_FLAGS="-std=c++11" - -DENABLE_PARMETIS=OFF - -DENABLE_PUMI=OFF - -DIS_TESTING=OFF - - - name: EnGPar Build - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-engpar -j8 --target install - - # Build Omega_h - - - name: Cache Omega_h Build - uses: actions/cache@v3 - id: build-omega_h - with: - key: build-omega_h - path: ${{ runner.temp }}/build-omega_h - - - name: Omega_h Checkout repo - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + + - name: build kokkos + uses: ./.github/actions/install-repo with: - repository: SCOREC/omega_h - ref: scorec-v10.8.0 - path: omega_h - - - name: Omega_h Create Directory - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-omega_h - - - name: Omega_h Configure CMake - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/omega_h -B ${{ runner.temp }}/build-omega_h - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-omega_h/install - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - -DBUILD_SHARED_LIBS=OFF - -DOmega_h_USE_Kokkos=ON - -DOmega_h_USE_CUDA=off - -DOmega_h_USE_MPI=on - -DCMAKE_BUILD_TYPE=Release - -DBUILD_TESTING=on - -DCMAKE_CXX_COMPILER="mpicxx" - -DCMAKE_C_COMPILER="mpicc" - - - name: Omega_h Build - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-omega_h -j8 --target install - - # Build Cabana - - - name: Cache Cabana Build - uses: actions/cache@v3 - id: build-cabana + repo-name: 'kokkos' + repo-path: 'kokkos/kokkos' + repo-ref: '4.2.00' + cache: true + options: '-DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_OPENMP=off + -DKokkos_ENABLE_CUDA=off + -DKokkos_ENABLE_CUDA_LAMBDA=off + -DKokkos_ENABLE_DEBUG=on' + + + - name: build Engpar + uses: ./.github/actions/install-repo with: - key: build-cabana - path: ${{ runner.temp }}/build-cabana - - - name: Cabana Checkout repo - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + repo-name: 'Engpar' + repo-path: 'SCOREC/EnGPar' + repo-ref: '' + cache: true + options: '-DCMAKE_C_COMPILER=mpicc + -DCMAKE_CXX_COMPILER=mpicxx + -DCMAKE_CXX_FLAGS="-std=c++11" + -DENABLE_PARMETIS=OFF + -DENABLE_PUMI=OFF + -DIS_TESTING=OFF' + + + - name: build omega_h + uses: ./.github/actions/install-repo with: - repository: ECP-copa/cabana - ref: 0.6.1 - path: cabana - - - name: Cabana Create Directory - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-cabana - - - name: Cabana Configure CMake - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/cabana -B ${{ runner.temp }}/build-cabana - -DCMAKE_BUILD_TYPE="Release" - -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-cabana/install - - - name: Cabana Build - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-cabana -j8 --target install - - # Build PUMI-PIC - - - name: Cache PUMI-PIC Build - uses: actions/cache@v3 - id: build-pumi-pic + repo-name: 'omega_h' + repo-path: 'SCOREC/omega_h' + repo-ref: 'scorec-v10.8.4' + cache: true + options: '-DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=OFF + -DOmega_h_USE_Kokkos=ON + -DOmega_h_USE_CUDA=off + -DOmega_h_USE_MPI=on + -DMPIEXEC_EXECUTABLE=srun + -DBUILD_TESTING=off + -DCMAKE_C_COMPILER=mpicc + -DCMAKE_CXX_COMPILER=mpicxx + -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + + - name: build cabana + uses: ./.github/actions/install-repo with: - key: build-pumi-pic - path: ${{ runner.temp }}/build-pumi-pic - - - name: PUMI-PIC Checkout repo - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + repo-name: 'cabana' + repo-path: 'ECP-copa/cabana' + repo-ref: '0.6.1' + cache: true + options: '-DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + + - name: build pumi-pic + uses: ./.github/actions/install-repo with: - submodules: recursive - repository: SCOREC/pumi-pic - ref: 2.0.3 - path: pumi-pic - - - name: PUMI-PIC Create Directory - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-pumi-pic - - - name: PUMI-PIC Configure CMake - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/pumi-pic -B ${{ runner.temp }}/build-pumi-pic - -DCMAKE_CXX_COMPILER=mpicxx - -DIS_TESTING=ON - -DPS_IS_TESTING=ON - -DCMAKE_BUILD_TYPE="Release" - -DTEST_DATA_DIR=$GITHUB_WORKSPACE/pumi-pic/pumipic-data - -DOmega_h_PREFIX=${{ runner.temp }}/build-omega_h/install - -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install - -DEnGPar_PREFIX=${{ runner.temp }}/build-engpar/install - -DCabana_PREFIX=${{ runner.temp }}/build-cabana/install - -DENABLE_CABANA=on - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-pumi-pic/install - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - - - name: PUMI-PIC Build - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-pumi-pic -j8 --target install - - # Build PolyMPO - - - name: PolyMPO Checkout repo - uses: actions/checkout@v3 + repo-name: 'pumi-pic' + repo-path: 'SCOREC/pumi-pic' + repo-ref: '2.1.2' + submodules: 'recursive' + cache: true + options: '-DCMAKE_CXX_COMPILER=mpicxx + -DCMAKE_BUILD_TYPE=Release + -DIS_TESTING=OFF + -DPS_IS_TESTING=OFF + -DTEST_DATA_DIR=$GITHUB_WORKSPACE/pumi-pic/pumipic-data + -DOmega_h_PREFIX=${{ runner.temp }}/build-omega_h/install + -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install + -DEnGPar_PREFIX=${{ runner.temp }}/build-Engpar/install + -DCabana_PREFIX=${{ runner.temp }}/build-cabana/install + -DENABLE_CABANA=on + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + + - name: build polyMPO + uses: ./.github/actions/install-repo with: - submodules: recursive - repository: SCOREC/polyMPO - path: polyMPO - - - name: PolyMPO Create Directory - run: cmake -E make_directory ${{ runner.temp }}/build-polyMPO - - - name: PolyMPO Configure CMake - run: cmake -S $GITHUB_WORKSPACE/polyMPO -B ${{ runner.temp }}/build-polyMPO - -DCMAKE_BUILD_TYPE="Debug" - -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib64/cmake/Kokkos - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-pumi-pic/install - -DIS_TESTING=on - -DCMAKE_CXX_COMPILER=mpicxx - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-polyMPO/install - - - name: PolyMPO Build - run: cmake --build ${{ runner.temp }}/build-polyMPO -j8 --target install + repo-name: 'polyMPO' + repo-path: 'SCOREC/polyMPO' + repo-ref: '' + cache: false + submodules: 'recursive' + options: '-DCMAKE_CXX_COMPILER=mpicxx + -DCMAKE_BUILD_TYPE=Debug + -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib64/cmake/Kokkos + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-pumi-pic/install + -DIS_TESTING=on' - name: PolyMPO Test run: ctest --test-dir ${{ runner.temp }}/build-polyMPO diff --git a/.github/workflows/updateCmakeVersion.yml b/.github/workflows/updateCmakeVersion.yml new file mode 100644 index 00000000..d7a9da9b --- /dev/null +++ b/.github/workflows/updateCmakeVersion.yml @@ -0,0 +1,31 @@ +name: Update CMAKE Version +on: + pull_request: + +jobs: + checkVersion: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash + + steps: + + - name: Check out master + uses: actions/checkout@v4 + with: + path: master + ref: ${{ github.base_ref }} + + - name: Check out current + uses: actions/checkout@v4 + with: + path: branch + + - name: CMake Version Has Updated + run: | + masterVersion=$(grep -oP 'project\(\s*[\w-]+\s+VERSION\s+\K[0-9]+\.[0-9]+\.[0-9]+' master/CMakeLists.txt) + branchVersion=$(grep -oP 'project\(\s*[\w-]+\s+VERSION\s+\K[0-9]+\.[0-9]+\.[0-9]+' branch/CMakeLists.txt) + [ "$masterVersion" != "$branchVersion" ] + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 080a4186..094d7a6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.19) -project(polyMPO VERSION 0.1.0 LANGUAGES CXX Fortran) +project(polyMPO VERSION 0.2.0 LANGUAGES CXX Fortran) include(cmake/bob.cmake) bob_begin_package() diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index d5d06503..8a0038ac 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -211,7 +211,7 @@ void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, auto mpAppID = p_MPs->getData(); auto elmIDoffset = p_MPs->getElmIDoffset(); - kkIntViewHostU arrayHost(elmIDs,numMPs); + polyMPO::MPsViewHostU arrayHost(elmIDs,numMPs); polyMPO::IntView mpCurElmIDCopy("mpCurElmIDNewValue",numMPs); auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ @@ -230,255 +230,105 @@ void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFla } -void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpPositionsIn){ +template +void setMPData(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const T* mpDataIn){ Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(nComps == polyMPO::mpSliceToNumEntries()); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); + + kkViewHostU mpDataIn_h(mpDataIn,nComps,numMPs); - if (p_MPs->rebuildOngoing()) { - p_MPs->setRebuildMPSlice(mpPositionsIn_h); + if (mpSlice == polyMPO::MPF_Cur_Pos_XYZ && p_MPs->rebuildOngoing()) { + p_MPs->setRebuildMPSlice(mpDataIn_h); return; } - auto mpPositions = p_MPs->getData(); + auto mpData = p_MPs->getData(); auto mpAppID = p_MPs->getData(); - Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); - Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); - auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + Kokkos::View mpData_d("mpData_d",nComps,numMPs); + Kokkos::deep_copy(mpData_d, mpDataIn_h); + + auto setData = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ - mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); - mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); - mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); + for (int i=0; iparallel_for(setPos, "setMPPositions"); - pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); + p_MPs->parallel_for(setData, "setMPData"); + pumipic::RecordTime("PolyMPO_setMPData", timer.seconds()); } -void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpPositionsHost){ +template +void getMPData(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + T* mpDataOut){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(nComps == polyMPO::mpSliceToNumEntries()); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - auto mpPositions = p_MPs->getData(); + auto mpData = p_MPs->getData(); auto mpAppID = p_MPs->getData(); - Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + Kokkos::View mpDataCopy("mpDataCopy",nComps,numMPs); + auto getData = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ - mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); - mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); - mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); + for (int i=0; iparallel_for(getPos, "getMPPositions"); - kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpPositionsCopy); + p_MPs->parallel_for(getData, "getMPData"); + kkViewHostU arrayHost(mpDataOut,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpDataCopy); + pumipic::RecordTime("PolyMPO_getMPData", timer.seconds()); } -void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpRotLatLonIn){ +void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn){ + setMPData(p_mpmesh, nComps, numMPs, mpPositionsIn); +} +void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpPositionsHost){ + getMPData(p_mpmesh, nComps, numMPs, mpPositionsHost); +} +void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn){ static int callCount = 0; PMT_ALWAYS_ASSERT(callCount == 0); - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); - Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); - Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); - auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); - mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setPos, "setMPRotLatLon"); + setMPData(p_mpmesh, nComps, numMPs, mpRotLatLonIn); callCount++; } - -void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpRotLatLonHost){ - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); - mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); - } - }; - p_MPs->parallel_for(getPos, "getMPRotLatLon"); - kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); +void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpRotLatLonHost){ + getMPData(p_mpmesh, nComps, numMPs, mpRotLatLonHost); } - void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpMassIn) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpMass = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpMassIn_h(mpMassIn,nComps,numMPs); - Kokkos::View mpMassIn_d("mpMassDevice",nComps,numMPs); - Kokkos::deep_copy(mpMassIn_d, mpMassIn_h); - auto setMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpMass(mp,0) = mpMassIn_d(0, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPMass, "setMPMass"); - pumipic::RecordTime("PolyMPO_setMPMass", timer.seconds()); + setMPData(p_mpmesh, nComps, numMPs, mpMassIn); } - void polympo_getMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpMassHost) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpMass = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpMassCopy("mpMassCopy",nComps,numMPs); - auto getMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpMassCopy(0,mpAppID(mp)) = mpMass(mp,0); - } - }; - p_MPs->parallel_for(getMPMass, "getMPMass"); - kkDbl2dViewHostU arrayHost(mpMassHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpMassCopy); - pumipic::RecordTime("PolyMPO_getMPMass", timer.seconds()); + getMPData(p_mpmesh, nComps, numMPs, mpMassHost); } - void polympo_setMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpVelIn) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpVel = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpVelIn_h(mpVelIn,nComps,numMPs); - Kokkos::View mpVelIn_d("mpVelDevice",vec2d_nEntries,numMPs); - Kokkos::deep_copy(mpVelIn_d, mpVelIn_h); - auto setMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpVel(mp,0) = mpVelIn_d(0, mpAppID(mp)); - mpVel(mp,1) = mpVelIn_d(1, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPVel, "setMPVel"); - pumipic::RecordTime("PolyMPO_setMPVel", timer.seconds()); + setMPData(p_mpmesh, nComps, numMPs, mpVelIn); } - void polympo_getMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpVelHost) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpVel = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpVelCopy("mpVelCopy",vec2d_nEntries,numMPs); - auto getMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpVelCopy(0,mpAppID(mp)) = mpVel(mp,0); - mpVelCopy(1,mpAppID(mp)) = mpVel(mp,1); - } - }; - p_MPs->parallel_for(getMPVel, "getMPVel"); - kkDbl2dViewHostU arrayHost(mpVelHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpVelCopy); - pumipic::RecordTime("PolyMPO_getMPVel", timer.seconds()); + getMPData(p_mpmesh, nComps, numMPs, mpVelHost); } - -//TODO: implement these void polympo_setMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStrainRateIn){ - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 6); //TODO: mp_sym_mat3d_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpStrainRate = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpStrainRateIn_h(mpStrainRateIn,nComps,numMPs); - Kokkos::View mpStrainRateIn_d("mpStrainRateDevice",nComps,numMPs); - Kokkos::deep_copy(mpStrainRateIn_d, mpStrainRateIn_h); - auto setMPStrainRate = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpStrainRate(mp,0) = mpStrainRateIn_d(0, mpAppID(mp)); - mpStrainRate(mp,1) = mpStrainRateIn_d(1, mpAppID(mp)); - mpStrainRate(mp,2) = mpStrainRateIn_d(2, mpAppID(mp)); - mpStrainRate(mp,3) = mpStrainRateIn_d(3, mpAppID(mp)); - mpStrainRate(mp,4) = mpStrainRateIn_d(4, mpAppID(mp)); - mpStrainRate(mp,5) = mpStrainRateIn_d(5, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPStrainRate, "setMPStrainRate"); + setMPData(p_mpmesh, nComps, numMPs, mpStrainRateIn); } void polympo_getMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStrainRateHost){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStrainRateHost; + getMPData(p_mpmesh, nComps, numMPs, mpStrainRateHost); } void polympo_setMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStressIn){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStressIn; + setMPData(p_mpmesh, nComps, numMPs, mpStressIn); } void polympo_getMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStressHost){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStressHost; + getMPData(p_mpmesh, nComps, numMPs, mpStressHost); } void polympo_startMeshFill_f(MPMesh_ptr p_mpmesh){ @@ -630,303 +480,140 @@ int polympo_getMeshFElmType_f() { return polyMPO::MeshFType_ElmBased; } -void polympo_setMeshVtxCoords_f(MPMesh_ptr p_mpmesh, const int nVertices, const double* xArray, const double* yArray, const double* zArray){ +template +void setMeshField1D(MPMesh_ptr p_mpmesh, const int size, Args... meshArraysIn){ Kokkos::Timer timer; - //chech validity checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + using T = std::common_type_t; + T combinedIn[] = {meshArraysIn...}; - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; igetMeshField(); + auto meshField_h = Kokkos::create_mirror_view(Kokkos::HostSpace(), meshField); + + int numArrays = sizeof...(meshArraysIn); + for(int i=0; i +void getMeshField1D(MPMesh_ptr p_mpmesh, const int size, Args... meshArraysOut){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the device to host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), - coordsArray); - for(int i=0; i; + T combinedOut[] = {meshArraysOut...}; + + auto meshField = p_mesh->getMeshField(); + auto meshField_h = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), meshField); + + int numArrays = sizeof...(meshArraysOut); + for(int i=0; i +void setMeshField2D(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const T* arrayIn){ Kokkos::Timer timer; - //chech validity checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkViewHostU hostView(arrayIn,nComps,nVertices); + Kokkos::View deviceView("meshDeviceView",nComps,nVertices); + Kokkos::deep_copy(deviceView, hostView); + + auto vtxField = p_mesh->getMeshField(); //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; i +void getMeshField2D(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, T* arrayOut){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkViewHostU hostViewOut(arrayOut,nComps,nVertices); + Kokkos::View deviceView("meshDeviceView",nComps,nVertices); - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the device to host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), - coordsArray); - for(int i=0; ip_mesh; + auto vtxField = p_mesh->getMeshField(); //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - //copy the host array to the device - auto elmCenter = p_mesh->getMeshField(); - auto h_elmCenter = Kokkos::create_mirror_view(elmCenter); - for(int i=0; i(p_mpmesh, nVertices, xArray, yArray, zArray); +} +void polympo_getMeshVtxCoords_f(MPMesh_ptr p_mpmesh, const int nVertices, double* xArray, double* yArray, double* zArray){ + getMeshField1D(p_mpmesh, nVertices, xArray, yArray, zArray); +} +void polympo_setMeshVtxRotLat_f(MPMesh_ptr p_mpmesh, const int nVertices, const double* latitude){ + setMeshField1D(p_mpmesh, nVertices, latitude); +} +void polympo_getMeshVtxRotLat_f(MPMesh_ptr p_mpmesh, const int nVertices, double* latitude){ + getMeshField1D(p_mpmesh, nVertices, latitude); +} +void polympo_setMeshElmCenter_f(MPMesh_ptr p_mpmesh, const int nCells, const double* xArray, const double* yArray, const double* zArray){ + setMeshField1D(p_mpmesh, nCells, xArray, yArray, zArray); +} void polympo_getMeshElmCenter_f(MPMesh_ptr p_mpmesh, const int nCells, double* xArray, double* yArray, double* zArray){ - //chech validity - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - - //copy the device to host - auto elmCenter = p_mesh->getMeshField(); - auto h_elmCenter = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), elmCenter); - for(int i=0; i(p_mpmesh, nCells, xArray, yArray, zArray); } - void polympo_setMeshVtxVel_f(MPMesh_ptr p_mpmesh, const int nVertices, const double* uVelIn, const double* vVelIn){ - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; i(p_mpmesh, nVertices, uVelIn, vVelIn); } - void polympo_getMeshVtxVel_f(MPMesh_ptr p_mpmesh, const int nVertices, double* uVelOut, double* vVelOut){ - Kokkos::Timer timer; - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; i(p_mpmesh, nVertices, uVelOut, vVelOut); } - void polympo_setMeshVtxMass_f(MPMesh_ptr p_mpmesh, const int nVertices, const double* vtxMass){ - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; i(p_mpmesh, nVertices, vtxMass); } - void polympo_getMeshVtxMass_f(MPMesh_ptr p_mpmesh, const int nVertices, double* vtxMass){ - Kokkos::Timer timer; - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; i(p_mpmesh, nVertices, vtxMass); } - void polympo_setMeshElmMass_f(MPMesh_ptr p_mpmesh, const int nCells, const double* elmMass){ - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - kkViewHostU arrayHost(elmMass, nCells); - for(int i=0; i(p_mpmesh, nCells, elmMass); } - void polympo_getMeshElmMass_f(MPMesh_ptr p_mpmesh, const int nCells, double* elmMass){ - Kokkos::Timer timer; - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == nCells); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; i(p_mpmesh, nCells, elmMass); } - void polympo_setMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkViewHostU arrayHost(array,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the host array to the device - Kokkos::deep_copy(vtxField,arrayHost); + setMeshField2D(p_mpmesh, nComps, nVertices, array); } - void polympo_getMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkVec2dViewHostU arrayHost(array,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the device array to the host - Kokkos::deep_copy(arrayHost, vtxField); + getMeshField2D(p_mpmesh, nComps, nVertices, array); } - void polympo_setMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { - Kokkos::Timer timer; - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); - Kokkos::deep_copy(array_d, arrayHost); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the host array to the device - Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - vtxField(iVtx,0) = array_d(0,iVtx); - vtxField(iVtx,1) = array_d(1,iVtx); - }); - pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfDispIncr", timer.seconds()); + setMeshField2D(p_mpmesh, nComps, nVertices, array); } - void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkDbl2dViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the device array to the host - Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - array_d(0,iVtx) = vtxField(iVtx,0); - array_d(1,iVtx) = vtxField(iVtx,1); - }); - Kokkos::deep_copy(arrayHost, array_d); + getMeshField2D(p_mpmesh, nComps, nVertices, array); } void polympo_push_f(MPMesh_ptr p_mpmesh){ diff --git a/src/pmpo_createTestMPMesh.cpp b/src/pmpo_createTestMPMesh.cpp index 3a2ae6cb..44b56a82 100644 --- a/src/pmpo_createTestMPMesh.cpp +++ b/src/pmpo_createTestMPMesh.cpp @@ -173,8 +173,8 @@ MaterialPoints* initTestMPs(Mesh* mesh, int testMPOption){ iMP += numMPsPerElement(i); },numMPs); - MPSView positions("MPpositions",numMPs); - MPSView latLonPositions("MPRotLatLonPositions",numMPs); + MPsView positions("MPpositions",numMPs); + MPsView latLonPositions("MPRotLatLonPositions",numMPs); if(geomType == geom_planar_surf){ Kokkos::parallel_for("intializeMPsPositionPlanar", numMPs, KOKKOS_LAMBDA(const int iMP){ int ielm = MPToElement(iMP); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 34857b93..556d292d 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -24,12 +24,6 @@ using kkViewHostU = Kokkos::View< Kokkos::DefaultHostExecutionSpace, Kokkos::MemoryTraits>; -typedef kkViewHostU kkDblViewHostU;//TODO:put it to mesh.hpp -typedef kkViewHostU kkVec2dViewHostU;//TODO:put it to mesh.hpp -typedef kkViewHostU kkDbl2dViewHostU;//TODO:put it somewhere else (maybe) -typedef kkViewHostU kkInt2dViewHostU;//TODO:put it somewhere else (maybe) -typedef kkViewHostU kkIntViewHostU;//TODO:put it somewhere else (maybe) - template auto create_mirror_view_and_copy(DataT array, const int size){ kkViewHostU temp_host(array, size); diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index c20fe004..86a9cdd0 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -19,7 +19,7 @@ pumipic::MemberTypeViews createInternalMemberViews(int numMPs, View mp2elm, View return mpInfo; } -PS* createDPS(int numElms, int numMPs, MPSView positions, IntView mpsPerElm, IntView mp2elm) { +PS* createDPS(int numElms, int numMPs, MPsView positions, IntView mpsPerElm, IntView mp2elm) { PS::kkGidView elmGids("elementGlobalIds", numElms); //TODO - initialize this to [0..numElms) auto mpInfo = ps::createMemberViews(numMPs); auto mpPositions = ps::getMemberView(mpInfo); @@ -51,7 +51,7 @@ PS* createDPS(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntVie } //End anonymous namespace -MaterialPoints::MaterialPoints(int numElms, int numMPs, MPSView positions, IntView mpsPerElm, IntView mp2elm) { +MaterialPoints::MaterialPoints(int numElms, int numMPs, MPsView positions, IntView mpsPerElm, IntView mp2elm) { MPs = createDPS(numElms, numMPs, positions, mpsPerElm, mp2elm); maxAppID = numMPs; //this ctor does not support inactive MPs operating_mode = MP_RELEASE; diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 86999eb5..2c007660 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -77,7 +77,10 @@ static constexpr int mpSliceToNumEntries() { } template -using MPSView = Kokkos::View::type*>; +using MPsViewHostU = kkViewHostU::type*>; + +template +using MPsView = Kokkos::View::type*>; const static std::vector> mpSliceSwap = {{MPF_Cur_Elm_ID, MPF_Tgt_Elm_ID}, @@ -129,7 +132,7 @@ class MaterialPoints { public: MaterialPoints() : MPs(nullptr) {}; - MaterialPoints(int numElms, int numMPs, MPSView positions, IntView mpsPerElm, IntView mp2elm); + MaterialPoints(int numElms, int numMPs, MPsView positions, IntView mpsPerElm, IntView mp2elm); MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID); ~MaterialPoints(); diff --git a/src/pmpo_mesh.cpp b/src/pmpo_mesh.cpp index cdf9b286..8d44393a 100644 --- a/src/pmpo_mesh.cpp +++ b/src/pmpo_mesh.cpp @@ -65,7 +65,7 @@ namespace polyMPO{ // Lat [iVtx,0] = dispIncrY [iVtx,1] /R // Lon [iVtx,1] = dispIncrX [iVtx,0] /(R*cos(lat)) rotLatLonIncr(iVtx, 0) = dispIncr(iVtx, 1)/sphereRadius; - rotLatLonIncr(iVtx, 1) = dispIncr(iVtx, 0)/(sphereRadius * std::cos(lat(iVtx))); + rotLatLonIncr(iVtx, 1) = dispIncr(iVtx, 0)/(sphereRadius * std::cos(lat(iVtx, 0))); }); pumipic::RecordTime("PolyMPO_computeRotLatLonIncr", timer.seconds()); } diff --git a/src/pmpo_mesh.hpp b/src/pmpo_mesh.hpp index 3b91dc39..98138d15 100644 --- a/src/pmpo_mesh.hpp +++ b/src/pmpo_mesh.hpp @@ -36,7 +36,7 @@ enum MeshFieldType{ template struct meshFieldToType; template <> struct meshFieldToType < MeshF_VtxCoords > { using type = Kokkos::View; }; -template <> struct meshFieldToType < MeshF_VtxRotLat > { using type = DoubleView; }; +template <> struct meshFieldToType < MeshF_VtxRotLat > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_ElmCenterXYZ > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_Vel > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_VtxMass > { using type = Kokkos::View; }; diff --git a/test/mpUnitTest.cpp b/test/mpUnitTest.cpp index 86821a55..1d24a271 100644 --- a/test/mpUnitTest.cpp +++ b/test/mpUnitTest.cpp @@ -45,7 +45,7 @@ int main(int argc, char** argv) { auto mpToElement_d = copyToDevice(mpToElement, "mpToElement"); auto mpPerElement_d = copyToDevice(mpPerElement, "mpPerElement"); - auto mpPositions = polyMPO::MPSView("positions",numMPs); + auto mpPositions = polyMPO::MPsView("positions",numMPs); Kokkos::parallel_for("intializeMPsPosition", numMPs, KOKKOS_LAMBDA(const int i){ const auto x = mpToElement_d(i)*1.0; const auto y = mpToElement_d(i)*-1.0; diff --git a/test/use_at_your_own_risk/Makefile.testFortranInit b/test/use_at_your_own_risk/Makefile.testFortranInit deleted file mode 100644 index 839c9ed6..00000000 --- a/test/use_at_your_own_risk/Makefile.testFortranInit +++ /dev/null @@ -1,98 +0,0 @@ -############################################################################ -# The following is an minimal example of the compile and link commands -# needed on Perlmutter. The Perlmutter build of the polyMPO stack followed -# the build instructions in README.md - **no other configuration was tested**. -# -# Using this Makefile on a different system is not advised as it will require -# editing the paths in the first section **and** modifying the compile and link -# arguments. This is not intended to be portable as there is no logic to -# detect/discover compilers and other system software. -# -# This file was generated in July/Aug 2023. -############################################################################ - - -########################### -## edit the following paths -########################### -FC=gfortran - -CUDA_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/ - -MPI_DIR=/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/ -MPI_LIBS=${MPI_DIR}/lib/libmpifort_gnu_91.so ${MPI_DIR}/lib/libmpi_gnu_91.so - -devRoot=/global/homes/y/yygong/develop - -KOKKOS_DIR=${devRoot}/build-kokkos-login21-cuda/install/ -KOKKOS_LIB_DIR=${KOKKOS_DIR}/lib64 - -OMEGAH_DIR=${devRoot}/build-omegah-login21-cuda/install/ - -ENGPAR_DIR=${devRoot}/build-engpar-login21-cuda/install/ - -PUMIPIC_DIR=${devRoot}/build-pumipic-login21-cuda/install/ - -CABANA_DIR=${devRoot}/build-cabana-login21-cuda/install/ - -POLYMPO_DIR=${devRoot}/buildPolyMPO-GPU/install - -###################################################################################### -## Dont't change the following unless you know exactly what you are doing... -## The following was generated from the CMake generated compile and link commands in -## a CUDA enabled build of polyMPO -###################################################################################### -POLYMPO_LIBS=\ --L${CUDA_DIR}/targets/x86_64-linux/lib/stubs \ --L${CUDA_DIR}/targets/x86_64-linux/lib \ --Wl,-rpath,${CUDA_DIR}/lib64: \ -${POLYMPO_DIR}/lib/libpolympo_fortran.a \ -${POLYMPO_DIR}/lib/libpolyMPO-core.a \ --lm \ -${PUMIPIC_DIR}/lib/libpumipic-core.a \ -${PUMIPIC_DIR}/lib/libparticleStructs.a \ -${PUMIPIC_DIR}/lib/libsupport.a \ -${OMEGAH_DIR}/lib/libomega_h.a \ -${KOKKOS_LIB_DIR}/libkokkoscontainers.a \ -${KOKKOS_LIB_DIR}/libkokkoscore.a \ --lcuda \ --lcudart \ --ldl \ --lz \ -${ENGPAR_DIR}/lib/libengpar.a \ -${ENGPAR_DIR}/lib/libdiffusive.a \ -${ENGPAR_DIR}/lib/libmultilevel.a \ -${ENGPAR_DIR}/lib/libengpar_metrics.a \ -${ENGPAR_DIR}/lib/libcoloring.a \ -${ENGPAR_DIR}/lib/libagi.a \ -${ENGPAR_DIR}/lib/libengpar_support.a \ -${ENGPAR_DIR}/lib/libpcu.a \ -${MPI_LIBS} \ --lcudadevrt \ --lcudart_static \ --lrt \ --lpthread \ --ldl \ --lstdc++ - -POLYMPO_COMPILE_FLAGS=\ --DFP64 \ --DKOKKOS_ENABLED \ --DLOCAL_ID_FTYPE=C_INT32_T \ --DLOCAL_ID_TYPE=int32_t \ --DPP_ENABLE_CAB \ --DPP_USE_CUDA \ --I${POLYMPO_DIR}/include \ --I${PUMIPIC_DIR}/include \ --I${KOKKOS_DIR}/include \ --I${CUDA_DIR}/include \ --I${CABANA_DIR}/include \ --I${MPI_DIR}/include \ --I${OMEGAH_DIR}/include \ --I${ENGPAR_DIR}/include - -testFortranInit: testFortranInit.o - ${FC} testFortranInit.o -o testFortranInit ${POLYMPO_LIBS} - -testFortranInit.o: testFortranInit.f90 - ${FC} ${POLYMPO_COMPILE_FLAGS} -c testFortranInit.f90 -o testFortranInit.o diff --git a/test/use_at_your_own_risk/MakefileCPU.testFortranInit b/test/use_at_your_own_risk/MakefileCPU.testFortranInit deleted file mode 100644 index 87c46afc..00000000 --- a/test/use_at_your_own_risk/MakefileCPU.testFortranInit +++ /dev/null @@ -1,99 +0,0 @@ -############################################################################ -# The following is an minimal example of the compile and link commands -# needed on Perlmutter. The Perlmutter build of the polyMPO stack followed -# the build instructions in README.md - **no other configuration was tested**. -# -# Using this Makefile on a different system is not advised as it will require -# editing the paths in the first section **and** modifying the compile and link -# arguments. This is not intended to be portable as there is no logic to -# detect/discover compilers and other system software. -# -# This file was generated in Nov 16 2023. -############################################################################ - - -########################### -## edit the following paths -########################### -FC=/opt/cray/pe/gcc/11.2.0/bin/gfortran - -NETCDF_DIR=/opt/cray/pe/netcdf/4.9.0.3/gnu/9.1/ -HDF5_DIR=/opt/cray/pe/hdf5/1.12.2.7/gnu/9.1/ -MPI_DIR=/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/ - -devRoot=/global/homes/c/castia5/cpu-pumi-pic - -KOKKOS_DIR=${devRoot}/build-kokkos-login21-cpu/install/ -KOKKOS_LIB_DIR=${KOKKOS_DIR}/lib64 - -OMEGAH_DIR=${devRoot}/build-omegah-login21-cpu/install/ - -ENGPAR_DIR=${devRoot}/build-engpar-login21-cpu/install/ - -PUMIPIC_DIR=${devRoot}/build-pumipic-login21-cpu/install/ - -CABANA_DIR=${devRoot}/build-cabana-login21-cpu/install/ - -POLYMPO_DIR=${devRoot}/buildPolyMPO-CPU/install - -###################################################################################### -## Dont't change the following unless you know exactly what you are doing... -## The following was generated from the CMake generated compile and link commands in -## a CUDA disabled build of polyMPO -###################################################################################### -POLYMPO_LIBS=\ --L${MPI_DIR}/lib \ --Wl,-rpath,${NETCDF_DIR}/lib:${MPI_DIR}/lib: \ -${POLYMPO_DIR}/lib/libpolympo_fortran.a \ -${POLYMPO_DIR}/lib/libpolyMPO-core.a \ --lm \ -${PUMIPIC_DIR}/lib/libpumipic-core.a \ -${PUMIPIC_DIR}/lib/libparticleStructs.a \ -${PUMIPIC_DIR}/lib/libsupport.a \ -${OMEGAH_DIR}/lib64/libomega_h.a \ -${KOKKOS_LIB_DIR}/libkokkoscontainers.a \ -${KOKKOS_LIB_DIR}/libkokkoscore.a \ --ldl \ -${KOKKOS_LIB_DIR}/libkokkossimd.a \ -/usr/lib64/libz.so \ -${ENGPAR_DIR}/lib/libengpar.a \ -${ENGPAR_DIR}/lib/libdiffusive.a \ -${ENGPAR_DIR}/lib/libmultilevel.a \ -${ENGPAR_DIR}/lib/libengpar_metrics.a \ -${ENGPAR_DIR}/lib/libcoloring.a \ -${ENGPAR_DIR}/lib/libagi.a \ -${ENGPAR_DIR}/lib/libengpar_support.a \ -${ENGPAR_DIR}/lib/libpcu.a \ -${NETCDF_DIR}/lib/libnetcdf.so \ -${POLYMPO_DIR}/lib/libpolympo_fortran.a \ -${MPI_DIR}/lib/libmpifort_gnu_91.so \ -${MPI_DIR}/lib/libmpi_gnu_91.so \ -${NETCDF_DIR}/lib/libnetcdff.so \ --lmpi_gnu_91 \ --lstdc++ - -POLYMPO_COMPILE_FLAGS=\ --DFP64 \ --DKOKKOS_ENABLED \ --DLOCAL_ID_FTYPE=C_INT32_T \ --DLOCAL_ID_TYPE=int32_t \ --DPP_ENABLE_CAB \ --I${devRoot}/polyMPO/src \ --I${devRoot}/polyMPO/test \ --I${devRoot}/buildPolyMPO-CPU/src \ --I${PUMIPIC_DIR}/include \ --I${KOKKOS_DIR}/include \ --I${CABANA_DIR}/include \ --I${OMEGAH_DIR}/include \ --I/usr/include \ --I${ENGPAR_DIR}/include \ --I${MPI_DIR}/include \ --I${NETCDF_DIR}/include \ --I${HDF5_DIR}/include \ --g -J${devRoot}/buildPolyMPO-CPU/mod_files - -testFortranInit: testFortranInit.o - ${FC} testFortranInit.o -o testFortranInit ${POLYMPO_LIBS} - -testFortranInit.o: testFortranInit.f90 - ${FC} ${POLYMPO_COMPILE_FLAGS} -c testFortranInit.f90 -o testFortranInit.o diff --git a/test/use_at_your_own_risk/MakefilePerlmutterCPU.testFortranInit b/test/use_at_your_own_risk/MakefilePerlmutterCPU.testFortranInit deleted file mode 100644 index a515165c..00000000 --- a/test/use_at_your_own_risk/MakefilePerlmutterCPU.testFortranInit +++ /dev/null @@ -1,107 +0,0 @@ -############################################################################ -# The following is an minimal example of the compile and link commands -# needed on Perlmutter. The Perlmutter build of the polyMPO stack followed -# the build instructions in README.md - **no other configuration was tested**. -# -# Using this Makefile on a different system is not advised as it will require -# editing the paths in the first section **and** modifying the compile and link -# arguments. This is not intended to be portable as there is no logic to -# detect/discover compilers and other system software. -# -# This file was generated in Nov 16 2023. -############################################################################ - - -########################### -## edit the following paths -########################### -FC=/usr/bin/gfortran - -NETCDF_DIR=/opt/cray/pe/netcdf/4.9.0.9/gnu/12.3/ -HDF5_DIR=/opt/cray/pe/hdf5/1.12.2.9/gnu/12.3/ -MPI_DIR=/opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3/ -CUDA_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/12.2/targets/x86_64-linux/lib - -devRoot=/global/homes/c/castia5/cpu-pumi-pic - -KOKKOS_DIR=${devRoot}/build-kokkos-perlmutter-cpu/install/ -KOKKOS_LIB_DIR=${KOKKOS_DIR}/lib64 - -OMEGAH_DIR=${devRoot}/build-omegah-perlmutter-cpu/install/ - -ENGPAR_DIR=${devRoot}/build-engpar-perlmutter-cpu/install/ - -PUMIPIC_DIR=${devRoot}/build-pumipic-perlmutter-cpu/install/ - -CABANA_DIR=${devRoot}/build-cabana-perlmutter-cpu/install/ - -POLYMPO_DIR=${devRoot}/buildPolyMPO-CPU/install - -###################################################################################### -## Dont't change the following unless you know exactly what you are doing... -## The following was generated from the CMake generated compile and link commands in -## a CUDA disabled build of polyMPO -###################################################################################### -POLYMPO_LIBS=\ --L${MPI_DIR}/lib \ --L${CUDA_DIR}/stubs \ --L${CUDA_DIR} \ --Wl,-rpath,${NETCDF_DIR}/lib:${MPI_DIR}/lib: \ -${POLYMPO_DIR}/lib/libpolympo_fortran.a \ -${POLYMPO_DIR}/lib/libpolyMPO-core.a \ --lm \ -${PUMIPIC_DIR}/lib/libpumipic-core.a \ -${PUMIPIC_DIR}/lib/libparticleStructs.a \ -${PUMIPIC_DIR}/lib/libsupport.a \ -${OMEGAH_DIR}/lib64/libomega_h.a \ -${KOKKOS_LIB_DIR}/libkokkoscontainers.a \ -${KOKKOS_LIB_DIR}/libkokkoscore.a \ --ldl \ -${KOKKOS_LIB_DIR}/libkokkossimd.a \ -/usr/lib64/libz.so \ -${ENGPAR_DIR}/lib/libengpar.a \ -${ENGPAR_DIR}/lib/libdiffusive.a \ -${ENGPAR_DIR}/lib/libmultilevel.a \ -${ENGPAR_DIR}/lib/libengpar_metrics.a \ -${ENGPAR_DIR}/lib/libcoloring.a \ -${ENGPAR_DIR}/lib/libagi.a \ -${ENGPAR_DIR}/lib/libengpar_support.a \ -${ENGPAR_DIR}/lib/libpcu.a \ -${NETCDF_DIR}/lib/libnetcdf.so \ -${POLYMPO_DIR}/lib/libpolympo_fortran.a \ -${MPI_DIR}/lib/libmpifort_gnu_123.so \ -${MPI_DIR}/lib/libmpi_gnu_123.so \ -${NETCDF_DIR}/lib/libnetcdff.so \ --lmpi_gnu_123 \ --lcudadevrt \ --lcudart_static \ --lrt \ --lpthread \ --ldl \ --lstdc++ - -POLYMPO_COMPILE_FLAGS=\ --DFP64 \ --DKOKKOS_ENABLED \ --DLOCAL_ID_FTYPE=C_INT32_T \ --DLOCAL_ID_TYPE=int32_t \ --DPP_ENABLE_CAB \ --I${devRoot}/polyMPO/src \ --I${devRoot}/polyMPO/test \ --I${devRoot}/buildPolyMPO-CPU/src \ --I${PUMIPIC_DIR}/include \ --I${KOKKOS_DIR}/include \ --I${CABANA_DIR}/include \ --I${OMEGAH_DIR}/include \ --I/usr/include \ --I${ENGPAR_DIR}/include \ --I${MPI_DIR}/include \ --I${NETCDF_DIR}/include \ --I${HDF5_DIR}/include \ --g -J${devRoot}/buildPolyMPO-CPU/mod_files - -testFortranInit: testFortranInit.o - ${FC} testFortranInit.o -o testFortranInit ${POLYMPO_LIBS} - -testFortranInit.o: testFortranInit.f90 - ${FC} ${POLYMPO_COMPILE_FLAGS} -c testFortranInit.f90 -o testFortranInit.o diff --git a/test/use_at_your_own_risk/README.md b/test/use_at_your_own_risk/README.md deleted file mode 100644 index 8da17fe8..00000000 --- a/test/use_at_your_own_risk/README.md +++ /dev/null @@ -1,11 +0,0 @@ -The Makefiles in this directoery were generated manually by extracting the -compile and link line produced by CMake (`make VERBOSE=1`) for the GPU build. -It was tested by compiling one of our fortran tests that calls polyMPO APIs. - -A big draw back of this approach, which has already bitten us once, is that if -we add new libraries or headers the Makefiles have to be recreated/updated -manually. - -It is a crude approach, but since MPAS/MPM uses a Makefile based system and -isn't using CMake, autotools, pkgconfig etc. we don't have a significantly -better choice.